Various bug fixes for new forms.
This commit is contained in:
Landon Wark
@@ -374,11 +374,15 @@ class Reagent(BaseClass):
|
||||
except (TypeError, AttributeError) as e:
|
||||
place_holder = date.today()
|
||||
logger.debug(f"We got a type error setting {self.lot} expiry: {e}. setting to today for testing")
|
||||
if self.expiry.year == 1970:
|
||||
place_holder = "NA"
|
||||
else:
|
||||
place_holder = place_holder.strftime("%Y-%m-%d")
|
||||
return dict(
|
||||
name=self.name,
|
||||
type=rtype,
|
||||
lot=self.lot,
|
||||
expiry=place_holder.strftime("%Y-%m-%d")
|
||||
expiry=place_holder
|
||||
)
|
||||
|
||||
def update_last_used(self, kit:KitType) -> Report:
|
||||
@@ -410,6 +414,7 @@ class Reagent(BaseClass):
|
||||
@classmethod
|
||||
@setup_lookup
|
||||
def query(cls,
|
||||
id:int|None=None,
|
||||
reagent_type:str|ReagentType|None=None,
|
||||
lot_number:str|None=None,
|
||||
name:str|None=None,
|
||||
@@ -428,6 +433,12 @@ class Reagent(BaseClass):
|
||||
models.Reagent | List[models.Reagent]: reagent or list of reagents matching filter.
|
||||
"""
|
||||
query: Query = cls.__database_session__.query(cls)
|
||||
match id:
|
||||
case int():
|
||||
query = query.filter(cls.id==id)
|
||||
limit = 1
|
||||
case _:
|
||||
pass
|
||||
match reagent_type:
|
||||
case str():
|
||||
# logger.debug(f"Looking up reagents by reagent type str: {reagent_type}")
|
||||
@@ -535,6 +546,7 @@ class SubmissionType(BaseClass):
|
||||
id = Column(INTEGER, primary_key=True) #: primary key
|
||||
name = Column(String(128), unique=True) #: name of submission type
|
||||
info_map = Column(JSON) #: Where basic information is found in the excel workbook corresponding to this type.
|
||||
defaults = Column(JSON) #: Basic information about this submission type
|
||||
instances = relationship("BasicSubmission", backref="submission_type") #: Concrete instances of this type.
|
||||
template_file = Column(BLOB) #: Blank form for this type stored as binary.
|
||||
processes = relationship("Process", back_populates="submission_types", secondary=submissiontypes_processes) #: Relation to equipment processes used for this type.
|
||||
@@ -653,6 +665,10 @@ class SubmissionType(BaseClass):
|
||||
raise TypeError(f"Type {type(equipment_role)} is not allowed")
|
||||
return list(set([item for items in relevant for item in items if item != None ]))
|
||||
|
||||
def get_submission_class(self):
|
||||
from .submissions import BasicSubmission
|
||||
return BasicSubmission.find_polymorphic_subclass(polymorphic_identity=self.name)
|
||||
|
||||
@classmethod
|
||||
@setup_lookup
|
||||
def query(cls,
|
||||
|
||||
@@ -65,8 +65,8 @@ class Organization(BaseClass):
|
||||
pass
|
||||
match name:
|
||||
case str():
|
||||
# logger.debug(f"Looking up organization with name: {name}")
|
||||
query = query.filter(cls.name==name)
|
||||
# logger.debug(f"Looking up organization with name starting with: {name}")
|
||||
query = query.filter(cls.name.startswith(name))
|
||||
limit = 1
|
||||
case _:
|
||||
pass
|
||||
|
||||
@@ -3,12 +3,12 @@ Models for the main submission types.
|
||||
'''
|
||||
from __future__ import annotations
|
||||
from getpass import getuser
|
||||
import logging, uuid, tempfile, re, yaml, base64
|
||||
import logging, uuid, tempfile, re, yaml, base64, sys
|
||||
from zipfile import ZipFile
|
||||
from tempfile import TemporaryDirectory
|
||||
from reportlab.graphics.barcode import createBarcodeImageInMemory
|
||||
from reportlab.graphics.shapes import Drawing
|
||||
from reportlab.lib.units import mm
|
||||
# from reportlab.graphics.barcode import createBarcodeImageInMemory
|
||||
# from reportlab.graphics.shapes import Drawing
|
||||
# from reportlab.lib.units import mm
|
||||
from operator import attrgetter, itemgetter
|
||||
from pprint import pformat
|
||||
from . import BaseClass, Reagent, SubmissionType, KitType, Organization
|
||||
@@ -18,9 +18,6 @@ from sqlalchemy import Column, String, TIMESTAMP, INTEGER, ForeignKey, JSON, FLO
|
||||
from sqlalchemy.orm import relationship, validates, Query
|
||||
from sqlalchemy.orm.attributes import flag_modified
|
||||
from sqlalchemy.ext.associationproxy import association_proxy
|
||||
# from sqlalchemy.ext.declarative import declared_attr
|
||||
# from sqlalchemy_json import NestedMutableJson
|
||||
# from sqlalchemy.ext.mutable import MutableDict
|
||||
from sqlalchemy.exc import OperationalError as AlcOperationalError, IntegrityError as AlcIntegrityError, StatementError
|
||||
from sqlite3 import OperationalError as SQLOperationalError, IntegrityError as SQLIntegrityError
|
||||
import pandas as pd
|
||||
@@ -59,9 +56,10 @@ class BasicSubmission(BaseClass):
|
||||
reagents_id = Column(String, ForeignKey("_reagent.id", ondelete="SET NULL", name="fk_BS_reagents_id")) #: id of used reagents
|
||||
extraction_info = Column(JSON) #: unstructured output from the extraction table logger.
|
||||
run_cost = Column(FLOAT(2)) #: total cost of running the plate. Set from constant and mutable kit costs at time of creation.
|
||||
uploaded_by = Column(String(32)) #: user name of person who submitted the submission to the database.
|
||||
signed_by = Column(String(32)) #: user name of person who submitted the submission to the database.
|
||||
comment = Column(JSON) #: user notes
|
||||
submission_category = Column(String(64)) #: ["Research", "Diagnostic", "Surveillance", "Validation"], else defaults to submission_type_name
|
||||
cost_centre = Column(String(64)) #: Permanent storage of used cost centre in case organization field changed in the future.
|
||||
|
||||
submission_sample_associations = relationship(
|
||||
"SubmissionSampleAssociation",
|
||||
@@ -103,12 +101,59 @@ class BasicSubmission(BaseClass):
|
||||
return f"{submission_type}Submission({self.rsl_plate_num})"
|
||||
|
||||
@classmethod
|
||||
def jsons(cls):
|
||||
def jsons(cls) -> List[str]:
|
||||
output = [item.name for item in cls.__table__.columns if isinstance(item.type, JSON)]
|
||||
if issubclass(cls, BasicSubmission) and not cls.__name__ == "BasicSubmission":
|
||||
output += BasicSubmission.jsons()
|
||||
return output
|
||||
|
||||
@classmethod
|
||||
def get_default_info(cls, *args):
|
||||
# Create defaults for all submission_types
|
||||
# print(args)
|
||||
recover = ['filepath', 'samples', 'csv', 'comment', 'equipment']
|
||||
dicto = dict(
|
||||
details_ignore = ['excluded', 'reagents', 'samples',
|
||||
'extraction_info', 'comment', 'barcode',
|
||||
'platemap', 'export_map', 'equipment'],
|
||||
form_recover = recover,
|
||||
form_ignore = ['reagents', 'ctx', 'id', 'cost', 'extraction_info', 'signed_by'] + recover,
|
||||
parser_ignore = ['samples', 'signed_by'] + cls.jsons(),
|
||||
excel_ignore = []
|
||||
)
|
||||
# Grab subtype specific info.
|
||||
st = cls.get_submission_type()
|
||||
if st is None:
|
||||
logger.error("No default info for BasicSubmission.")
|
||||
return dicto
|
||||
else:
|
||||
dicto['submission_type'] = st.name
|
||||
output = {}
|
||||
for k,v in dicto.items():
|
||||
if len(args) > 0 and k not in args:
|
||||
logger.debug(f"Don't want {k}")
|
||||
continue
|
||||
else:
|
||||
output[k] = v
|
||||
for k,v in st.defaults.items():
|
||||
if len(args) > 0 and k not in args:
|
||||
logger.debug(f"Don't want {k}")
|
||||
continue
|
||||
else:
|
||||
match v:
|
||||
case list():
|
||||
output[k] += v
|
||||
case _:
|
||||
output[k] = v
|
||||
if len(args) == 1:
|
||||
return output[args[0]]
|
||||
return output
|
||||
|
||||
@classmethod
|
||||
def get_submission_type(cls):
|
||||
name = cls.__mapper_args__['polymorphic_identity']
|
||||
return SubmissionType.query(name=name)
|
||||
|
||||
def to_dict(self, full_data:bool=False, backup:bool=False, report:bool=False) -> dict:
|
||||
"""
|
||||
Constructs dictionary used in submissions summary
|
||||
@@ -168,6 +213,11 @@ class BasicSubmission(BaseClass):
|
||||
logger.debug(f"Attempting reagents.")
|
||||
try:
|
||||
reagents = [item.to_sub_dict(extraction_kit=self.extraction_kit) for item in self.submission_reagent_associations]
|
||||
for k in self.extraction_kit.construct_xl_map_for_use(self.submission_type):
|
||||
if k == 'info':
|
||||
continue
|
||||
if not any([item['type']==k for item in reagents]):
|
||||
reagents.append(dict(type=k, name="Not Applicable", lot="NA", expiry=date(year=1970, month=1, day=1), missing=True))
|
||||
except Exception as e:
|
||||
logger.error(f"We got an error retrieving reagents: {e}")
|
||||
reagents = None
|
||||
@@ -181,10 +231,12 @@ class BasicSubmission(BaseClass):
|
||||
except Exception as e:
|
||||
logger.error(f"Error setting equipment: {e}")
|
||||
equipment = None
|
||||
cost_centre = self.cost_centre
|
||||
else:
|
||||
reagents = None
|
||||
samples = None
|
||||
equipment = None
|
||||
cost_centre = None
|
||||
# logger.debug("Getting comments")
|
||||
try:
|
||||
comments = self.comment
|
||||
@@ -198,6 +250,8 @@ class BasicSubmission(BaseClass):
|
||||
output["extraction_info"] = ext_info
|
||||
output["comment"] = comments
|
||||
output["equipment"] = equipment
|
||||
output["Cost Centre"] = cost_centre
|
||||
output["Signed By"] = self.signed_by
|
||||
return output
|
||||
|
||||
def calculate_column_count(self) -> int:
|
||||
@@ -293,18 +347,18 @@ class BasicSubmission(BaseClass):
|
||||
"""
|
||||
return [item.role for item in self.submission_equipment_associations]
|
||||
|
||||
def make_plate_barcode(self, width:int=100, height:int=25) -> Drawing:
|
||||
"""
|
||||
Creates a barcode image for this BasicSubmission.
|
||||
# def make_plate_barcode(self, width:int=100, height:int=25) -> Drawing:
|
||||
# """
|
||||
# Creates a barcode image for this BasicSubmission.
|
||||
|
||||
Args:
|
||||
width (int, optional): Width (pixels) of image. Defaults to 100.
|
||||
height (int, optional): Height (pixels) of image. Defaults to 25.
|
||||
# Args:
|
||||
# width (int, optional): Width (pixels) of image. Defaults to 100.
|
||||
# height (int, optional): Height (pixels) of image. Defaults to 25.
|
||||
|
||||
Returns:
|
||||
Drawing: image object
|
||||
"""
|
||||
return createBarcodeImageInMemory('Code128', value=self.rsl_plate_num, width=width*mm, height=height*mm, humanReadable=True, format="png")
|
||||
# Returns:
|
||||
# Drawing: image object
|
||||
# """
|
||||
# return createBarcodeImageInMemory('Code128', value=self.rsl_plate_num, width=width*mm, height=height*mm, humanReadable=True, format="png")
|
||||
|
||||
@classmethod
|
||||
def submissions_to_df(cls, submission_type:str|None=None, limit:int=0) -> pd.DataFrame:
|
||||
@@ -384,13 +438,19 @@ class BasicSubmission(BaseClass):
|
||||
case item if item in self.jsons():
|
||||
logger.debug(f"Setting JSON attribute.")
|
||||
existing = self.__getattribute__(key)
|
||||
if value == "" or value is None or value == 'null':
|
||||
logger.error(f"No value given, not setting.")
|
||||
return
|
||||
if existing is None:
|
||||
existing = []
|
||||
if value in existing:
|
||||
logger.warning("Value already exists. Preventing duplicate addition.")
|
||||
return
|
||||
else:
|
||||
existing.append(value)
|
||||
if isinstance(value, list):
|
||||
existing += value
|
||||
else:
|
||||
existing.append(value)
|
||||
self.__setattr__(key, existing)
|
||||
flag_modified(self, key)
|
||||
return
|
||||
@@ -634,7 +694,7 @@ class BasicSubmission(BaseClass):
|
||||
from backend.validators import RSLNamer
|
||||
logger.debug(f"instr coming into {cls}: {instr}")
|
||||
logger.debug(f"data coming into {cls}: {data}")
|
||||
defaults = cls.get_default_info()
|
||||
defaults = cls.get_default_info("abbreviation", "submission_type")
|
||||
data['abbreviation'] = defaults['abbreviation']
|
||||
if 'submission_type' not in data.keys() or data['submission_type'] in [None, ""]:
|
||||
data['submission_type'] = defaults['submission_type']
|
||||
@@ -737,9 +797,7 @@ class BasicSubmission(BaseClass):
|
||||
Returns:
|
||||
Tuple(dict, Template): (Updated dictionary, Template to be rendered)
|
||||
"""
|
||||
base_dict['excluded'] = ['excluded', 'reagents', 'samples', 'controls',
|
||||
'extraction_info', 'pcr_info', 'comment',
|
||||
'barcode', 'platemap', 'export_map', 'equipment']
|
||||
base_dict['excluded'] = cls.get_default_info('details_ignore')
|
||||
env = jinja_template_loading()
|
||||
temp_name = f"{cls.__name__.lower()}_details.html"
|
||||
logger.debug(f"Returning template: {temp_name}")
|
||||
@@ -1067,9 +1125,9 @@ class BacterialCulture(BasicSubmission):
|
||||
output['controls'] = [item.to_sub_dict() for item in self.controls]
|
||||
return output
|
||||
|
||||
@classmethod
|
||||
def get_default_info(cls) -> dict:
|
||||
return dict(abbreviation="BC", submission_type="Bacterial Culture")
|
||||
# @classmethod
|
||||
# def get_default_info(cls) -> dict:
|
||||
# return dict(abbreviation="BC", submission_type="Bacterial Culture")
|
||||
|
||||
@classmethod
|
||||
def custom_platemap(cls, xl: pd.ExcelFile, plate_map: pd.DataFrame) -> pd.DataFrame:
|
||||
@@ -1214,13 +1272,19 @@ class Wastewater(BasicSubmission):
|
||||
output['pcr_info'] = self.pcr_info
|
||||
except TypeError as e:
|
||||
pass
|
||||
ext_tech = self.ext_technician or self.technician
|
||||
pcr_tech = self.pcr_technician or self.technician
|
||||
output['Technician'] = f"Enr: {self.technician}, Ext: {ext_tech}, PCR: {pcr_tech}"
|
||||
if self.ext_technician is None or self.ext_technician == "None":
|
||||
output['Ext Technician'] = self.technician
|
||||
else:
|
||||
output["Ext Technician"] = self.ext_technician
|
||||
if self.pcr_technician is None or self.pcr_technician == "None":
|
||||
output["PCR Technician"] = self.technician
|
||||
else:
|
||||
output['PCR Technician'] = self.pcr_technician
|
||||
# output['Technician'] = self.technician}, Ext: {ext_tech}, PCR: {pcr_tech}"
|
||||
return output
|
||||
|
||||
@classmethod
|
||||
def get_default_info(cls) -> dict:
|
||||
# @classmethod
|
||||
# def get_default_info(cls) -> dict:
|
||||
return dict(abbreviation="WW", submission_type="Wastewater")
|
||||
|
||||
@classmethod
|
||||
@@ -1334,36 +1398,6 @@ class Wastewater(BasicSubmission):
|
||||
from frontend.widgets import select_open_file
|
||||
fname = select_open_file(obj=obj, file_extension="xlsx")
|
||||
parser = PCRParser(filepath=fname)
|
||||
# Check if PCR info already exists
|
||||
# if hasattr(self, 'pcr_info') and self.pcr_info != None:
|
||||
# # existing = json.loads(sub.pcr_info)
|
||||
# existing = self.pcr_info
|
||||
# logger.debug(f"Found existing pcr info: {pformat(self.pcr_info)}")
|
||||
# else:
|
||||
# existing = None
|
||||
# if existing != None:
|
||||
# # update pcr_info
|
||||
# try:
|
||||
# logger.debug(f"Updating {type(existing)}:\n {pformat(existing)} with {type(parser.pcr)}:\n {pformat(parser.pcr)}")
|
||||
# # if json.dumps(parser.pcr) not in sub.pcr_info:
|
||||
# if parser.pcr not in self.pcr_info:
|
||||
# logger.debug(f"This is new pcr info, appending to existing")
|
||||
# existing.append(parser.pcr)
|
||||
# else:
|
||||
# logger.debug("This info already exists, skipping.")
|
||||
# # logger.debug(f"Setting {self.rsl_plate_num} PCR to:\n {pformat(existing)}")
|
||||
# # sub.pcr_info = json.dumps(existing)
|
||||
# self.pcr_info = existing
|
||||
# except TypeError:
|
||||
# logger.error(f"Error updating!")
|
||||
# # sub.pcr_info = json.dumps([parser.pcr])
|
||||
# self.pcr_info = [parser.pcr]
|
||||
# logger.debug(f"Final pcr info for {self.rsl_plate_num}:\n {pformat(self.pcr_info)}")
|
||||
# else:
|
||||
# # sub.pcr_info = json.dumps([parser.pcr])
|
||||
# self.pcr_info = [parser.pcr]
|
||||
# # logger.debug(f"Existing {type(self.pcr_info)}: {self.pcr_info}")
|
||||
# # logger.debug(f"Inserting {type(parser.pcr)}: {parser.pcr}")
|
||||
self.set_attribute("pcr_info", parser.pcr)
|
||||
self.save(original=False)
|
||||
logger.debug(f"Got {len(parser.samples)} samples to update!")
|
||||
@@ -1390,7 +1424,6 @@ class WastewaterArtic(BasicSubmission):
|
||||
gel_controls = Column(JSON) #: locations of controls on the gel
|
||||
source_plates = Column(JSON) #: wastewater plates that samples come from
|
||||
|
||||
|
||||
__mapper_args__ = dict(polymorphic_identity="Wastewater Artic",
|
||||
polymorphic_load="inline",
|
||||
inherit_condition=(id == BasicSubmission.id))
|
||||
@@ -1411,9 +1444,9 @@ class WastewaterArtic(BasicSubmission):
|
||||
output['source_plates'] = self.source_plates
|
||||
return output
|
||||
|
||||
@classmethod
|
||||
def get_default_info(cls) -> str:
|
||||
return dict(abbreviation="AR", submission_type="Wastewater Artic")
|
||||
# @classmethod
|
||||
# def get_default_info(cls) -> str:
|
||||
# return dict(abbreviation="AR", submission_type="Wastewater Artic")
|
||||
|
||||
@classmethod
|
||||
def parse_info(cls, input_dict:dict, xl:pd.ExcelFile|None=None) -> dict:
|
||||
@@ -1429,12 +1462,16 @@ class WastewaterArtic(BasicSubmission):
|
||||
"""
|
||||
# from backend.validators import RSLNamer
|
||||
input_dict = super().parse_info(input_dict)
|
||||
ws = load_workbook(xl.io, data_only=True)['Egel results']
|
||||
data = [ws.cell(row=jj,column=ii) for ii in range(15,27) for jj in range(10,18)]
|
||||
workbook = load_workbook(xl.io, data_only=True)
|
||||
ws = workbook['Egel results']
|
||||
data = [ws.cell(row=ii,column=jj) for jj in range(15,27) for ii in range(10,18)]
|
||||
data = [cell for cell in data if cell.value is not None and "NTC" in cell.value]
|
||||
input_dict['gel_controls'] = [dict(sample_id=cell.value, location=f"{row_map[cell.row-9]}{str(cell.column-14).zfill(2)}") for cell in data]
|
||||
# df = xl.parse("Egel results").iloc[7:16, 13:26]
|
||||
# df = df.set_index(df.columns[0])
|
||||
ws = workbook['First Strand List']
|
||||
data = [dict(plate=ws.cell(row=ii, column=3).value, starting_sample=ws.cell(row=ii, column=4).value) for ii in range(8,11)]
|
||||
input_dict['source_plates'] = data
|
||||
return input_dict
|
||||
|
||||
@classmethod
|
||||
@@ -1500,34 +1537,38 @@ class WastewaterArtic(BasicSubmission):
|
||||
Returns:
|
||||
str: output name
|
||||
"""
|
||||
logger.debug(f"input string raw: {input_str}")
|
||||
# Remove letters.
|
||||
processed = re.sub(r"[A-Z]", "", input_str)
|
||||
processed = re.sub(r"[A-QS-Z]+\d*", "", input_str)
|
||||
# Remove trailing '-' if any
|
||||
processed = processed.strip("-")
|
||||
logger.debug(f"Processed after stripping letters: {processed}")
|
||||
try:
|
||||
en_num = re.search(r"\-\d{1}$", processed).group()
|
||||
processed = rreplace(processed, en_num, "")
|
||||
except AttributeError:
|
||||
en_num = "1"
|
||||
en_num = en_num.strip("-")
|
||||
# logger.debug(f"Processed after en-num: {processed}")
|
||||
logger.debug(f"Processed after en-num: {processed}")
|
||||
try:
|
||||
plate_num = re.search(r"\-\d{1}$", processed).group()
|
||||
plate_num = re.search(r"\-\d{1}R?\d?$", processed).group()
|
||||
processed = rreplace(processed, plate_num, "")
|
||||
except AttributeError:
|
||||
plate_num = "1"
|
||||
plate_num = plate_num.strip("-")
|
||||
# logger.debug(f"Processed after plate-num: {processed}")
|
||||
logger.debug(f"Processed after plate-num: {processed}")
|
||||
day = re.search(r"\d{2}$", processed).group()
|
||||
processed = rreplace(processed, day, "")
|
||||
# logger.debug(f"Processed after day: {processed}")
|
||||
logger.debug(f"Processed after day: {processed}")
|
||||
month = re.search(r"\d{2}$", processed).group()
|
||||
processed = rreplace(processed, month, "")
|
||||
processed = processed.replace("--", "")
|
||||
# logger.debug(f"Processed after month: {processed}")
|
||||
logger.debug(f"Processed after month: {processed}")
|
||||
year = re.search(r'^(?:\d{2})?\d{2}', processed).group()
|
||||
year = f"20{year}"
|
||||
return f"EN{year}{month}{day}-{en_num}"
|
||||
final_en_name = f"EN{year}{month}{day}-{en_num}"
|
||||
logger.debug(f"Final EN name: {final_en_name}")
|
||||
return final_en_name
|
||||
|
||||
@classmethod
|
||||
def get_regex(cls) -> str:
|
||||
@@ -1599,7 +1640,23 @@ class WastewaterArtic(BasicSubmission):
|
||||
# for jjj, value in enumerate(plate, start=3):
|
||||
# worksheet.cell(row=iii, column=jjj, value=value)
|
||||
logger.debug(f"Info:\n{pformat(info)}")
|
||||
check = 'gel_info' in info.keys() and info['gel_info']['value'] != None
|
||||
check = 'source_plates' in info.keys() and info['source_plates'] is not None
|
||||
if check:
|
||||
worksheet = input_excel['First Strand List']
|
||||
start_row = 8
|
||||
for iii, plate in enumerate(info['source_plates']['value']):
|
||||
logger.debug(f"Plate: {plate}")
|
||||
row = start_row + iii
|
||||
try:
|
||||
worksheet.cell(row=row, column=3, value=plate['plate'])
|
||||
except TypeError:
|
||||
pass
|
||||
try:
|
||||
worksheet.cell(row=row, column=4, value=plate['starting_sample'])
|
||||
except TypeError:
|
||||
pass
|
||||
# sys.exit(f"Hardcoded stop: backend.models.submissions:1629")
|
||||
check = 'gel_info' in info.keys() and info['gel_info']['value'] is not None
|
||||
if check:
|
||||
# logger.debug(f"Gel info check passed.")
|
||||
if info['gel_info'] != None:
|
||||
@@ -1618,7 +1675,7 @@ class WastewaterArtic(BasicSubmission):
|
||||
column = start_column + 2 + jjj
|
||||
worksheet.cell(row=start_row, column=column, value=kj['name'])
|
||||
worksheet.cell(row=row, column=column, value=kj['value'])
|
||||
check = 'gel_image' in info.keys() and info['gel_image']['value'] != None
|
||||
check = 'gel_image' in info.keys() and info['gel_image']['value'] is not None
|
||||
if check:
|
||||
if info['gel_image'] != None:
|
||||
worksheet = input_excel['Egel results']
|
||||
|
||||
@@ -62,9 +62,11 @@ class SheetParser(object):
|
||||
parser = InfoParser(xl=self.xl, submission_type=self.sub['submission_type']['value'])
|
||||
info = parser.parse_info()
|
||||
self.info_map = parser.map
|
||||
# exclude_from_info = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=self.sub['submission_type']).exclude_from_info_parser()
|
||||
for k,v in info.items():
|
||||
match k:
|
||||
case "sample":
|
||||
# case item if
|
||||
pass
|
||||
case _:
|
||||
self.sub[k] = v
|
||||
@@ -97,9 +99,9 @@ class SheetParser(object):
|
||||
"""
|
||||
Enforce that the parser has an extraction kit
|
||||
"""
|
||||
from frontend.widgets.pop_ups import KitSelector
|
||||
from frontend.widgets.pop_ups import ObjectSelector
|
||||
if not check_not_nan(self.sub['extraction_kit']['value']):
|
||||
dlg = KitSelector(title="Kit Needed", message="At minimum a kit is needed. Please select one.")
|
||||
dlg = ObjectSelector(title="Kit Needed", message="At minimum a kit is needed. Please select one.", obj_type=KitType)
|
||||
if dlg.exec():
|
||||
self.sub['extraction_kit'] = dict(value=dlg.getValues(), missing=True)
|
||||
else:
|
||||
@@ -133,7 +135,11 @@ class SheetParser(object):
|
||||
"""
|
||||
# logger.debug(f"Submission dictionary coming into 'to_pydantic':\n{pformat(self.sub)}")
|
||||
logger.debug(f"Equipment: {self.sub['equipment']}")
|
||||
if len(self.sub['equipment']) == 0:
|
||||
try:
|
||||
check = len(self.sub['equipment']) == 0
|
||||
except TypeError:
|
||||
check = True
|
||||
if check:
|
||||
self.sub['equipment'] = None
|
||||
psm = PydSubmission(filepath=self.filepath, **self.sub)
|
||||
return psm
|
||||
@@ -142,11 +148,12 @@ class InfoParser(object):
|
||||
|
||||
def __init__(self, xl:pd.ExcelFile, submission_type:str):
|
||||
logger.info(f"\n\Hello from InfoParser!\n\n")
|
||||
self.map = self.fetch_submission_info_map(submission_type=submission_type)
|
||||
self.submission_type = submission_type
|
||||
self.map = self.fetch_submission_info_map()
|
||||
self.xl = xl
|
||||
logger.debug(f"Info map for InfoParser: {pformat(self.map)}")
|
||||
|
||||
def fetch_submission_info_map(self, submission_type:str|dict) -> dict:
|
||||
def fetch_submission_info_map(self) -> dict:
|
||||
"""
|
||||
Gets location of basic info from the submission_type object in the database.
|
||||
|
||||
@@ -156,10 +163,10 @@ class InfoParser(object):
|
||||
Returns:
|
||||
dict: Location map of all info for this submission type
|
||||
"""
|
||||
if isinstance(submission_type, str):
|
||||
submission_type = dict(value=submission_type, missing=True)
|
||||
logger.debug(f"Looking up submission type: {submission_type['value']}")
|
||||
submission_type = SubmissionType.query(name=submission_type['value'])
|
||||
if isinstance(self.submission_type, str):
|
||||
self.submission_type = dict(value=self.submission_type, missing=True)
|
||||
logger.debug(f"Looking up submission type: {self.submission_type['value']}")
|
||||
submission_type = SubmissionType.query(name=self.submission_type['value'])
|
||||
info_map = submission_type.info_map
|
||||
# Get the parse_info method from the submission type specified
|
||||
self.custom_parser = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=submission_type.name).parse_info
|
||||
@@ -172,16 +179,25 @@ class InfoParser(object):
|
||||
Returns:
|
||||
dict: key:value of basic info
|
||||
"""
|
||||
if isinstance(self.submission_type, str):
|
||||
self.submission_type = dict(value=self.submission_type, missing=True)
|
||||
dicto = {}
|
||||
exclude_from_generic = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=self.submission_type['value']).get_default_info("parser_ignore")
|
||||
# This loop parses generic info
|
||||
logger.debug(f"Map: {self.map}")
|
||||
# time.sleep(5)
|
||||
for sheet in self.xl.sheet_names:
|
||||
df = self.xl.parse(sheet, header=None)
|
||||
relevant = {}
|
||||
for k, v in self.map.items():
|
||||
# exclude from generic parsing
|
||||
if k in exclude_from_generic:
|
||||
continue
|
||||
# If the value is hardcoded put it in the dictionary directly.
|
||||
if isinstance(v, str):
|
||||
dicto[k] = dict(value=v, missing=False)
|
||||
continue
|
||||
if k in ["samples", "all_sheets"]:
|
||||
continue
|
||||
logger.debug(f"Looking for {k} in self.map")
|
||||
if sheet in self.map[k]['sheets']:
|
||||
relevant[k] = v
|
||||
logger.debug(f"relevant map for {sheet}: {pformat(relevant)}")
|
||||
@@ -252,6 +268,7 @@ class ReagentParser(object):
|
||||
lot = df.iat[relevant[item]['lot']['row']-1, relevant[item]['lot']['column']-1]
|
||||
expiry = df.iat[relevant[item]['expiry']['row']-1, relevant[item]['expiry']['column']-1]
|
||||
if 'comment' in relevant[item].keys():
|
||||
logger.debug(f"looking for {relevant[item]} comment.")
|
||||
comment = df.iat[relevant[item]['comment']['row']-1, relevant[item]['comment']['column']-1]
|
||||
else:
|
||||
comment = ""
|
||||
@@ -294,7 +311,7 @@ class SampleParser(object):
|
||||
sample_info_map = self.fetch_sample_info_map(submission_type=submission_type, sample_map=sample_map)
|
||||
logger.debug(f"sample_info_map: {sample_info_map}")
|
||||
self.plate_map = self.construct_plate_map(plate_map_location=sample_info_map['plate_map'])
|
||||
logger.debug(f"plate_map: {self.plate_map}")
|
||||
# logger.debug(f"plate_map: {self.plate_map}")
|
||||
self.lookup_table = self.construct_lookup_table(lookup_table_location=sample_info_map['lookup_table'])
|
||||
if "plates" in sample_info_map:
|
||||
self.plates = sample_info_map['plates']
|
||||
@@ -439,7 +456,7 @@ class SampleParser(object):
|
||||
"""
|
||||
result = None
|
||||
new_samples = []
|
||||
logger.debug(f"Starting samples: {pformat(self.samples)}")
|
||||
# logger.debug(f"Starting samples: {pformat(self.samples)}")
|
||||
for sample in self.samples:
|
||||
translated_dict = {}
|
||||
for k, v in sample.items():
|
||||
|
||||
@@ -74,8 +74,8 @@ class RSLNamer(object):
|
||||
check = True
|
||||
if check:
|
||||
# logger.debug("Final option, ask the user for submission type")
|
||||
from frontend.widgets import SubmissionTypeSelector
|
||||
dlg = SubmissionTypeSelector(title="Couldn't parse submission type.", message="Please select submission type from list below.")
|
||||
from frontend.widgets import ObjectSelector
|
||||
dlg = ObjectSelector(title="Couldn't parse submission type.", message="Please select submission type from list below.", obj_type=SubmissionType)
|
||||
if dlg.exec():
|
||||
submission_type = dlg.parse_form()
|
||||
submission_type = submission_type.replace("_", " ")
|
||||
|
||||
@@ -8,13 +8,13 @@ from pydantic import BaseModel, field_validator, Field
|
||||
from datetime import date, datetime, timedelta
|
||||
from dateutil.parser import parse
|
||||
from dateutil.parser._parser import ParserError
|
||||
from typing import List, Tuple
|
||||
from typing import List, Tuple, Literal
|
||||
from . import RSLNamer
|
||||
from pathlib import Path
|
||||
from tools import check_not_nan, convert_nans_to_nones, Report, Result, row_map
|
||||
from backend.db.models import *
|
||||
from sqlalchemy.exc import StatementError, IntegrityError
|
||||
from PyQt6.QtWidgets import QComboBox, QWidget
|
||||
from PyQt6.QtWidgets import QWidget
|
||||
from openpyxl import load_workbook, Workbook
|
||||
from io import BytesIO
|
||||
|
||||
@@ -24,7 +24,7 @@ class PydReagent(BaseModel):
|
||||
|
||||
lot: str|None
|
||||
type: str|None
|
||||
expiry: date|None
|
||||
expiry: date|Literal['NA']|None
|
||||
name: str|None
|
||||
missing: bool = Field(default=True)
|
||||
comment: str|None = Field(default="", validate_default=True)
|
||||
@@ -77,6 +77,8 @@ class PydReagent(BaseModel):
|
||||
match value:
|
||||
case int():
|
||||
return datetime.fromordinal(datetime(1900, 1, 1).toordinal() + value - 2).date()
|
||||
case 'NA':
|
||||
return value
|
||||
case str():
|
||||
return parse(value)
|
||||
case date():
|
||||
@@ -87,6 +89,13 @@ class PydReagent(BaseModel):
|
||||
value = date.today()
|
||||
return value
|
||||
|
||||
@field_validator("expiry")
|
||||
@classmethod
|
||||
def date_na(cls, value):
|
||||
if isinstance(value, date) and value.year == 1970:
|
||||
value = "NA"
|
||||
return value
|
||||
|
||||
@field_validator("name", mode="before")
|
||||
@classmethod
|
||||
def enforce_name(cls, value, values):
|
||||
@@ -125,6 +134,10 @@ class PydReagent(BaseModel):
|
||||
reagent.type.append(reagent_type)
|
||||
case "comment":
|
||||
continue
|
||||
case "expiry":
|
||||
if isinstance(value, str):
|
||||
value = date(year=1970, month=1, day=1)
|
||||
reagent.expiry = value
|
||||
case _:
|
||||
try:
|
||||
reagent.__setattr__(key, value)
|
||||
@@ -271,6 +284,7 @@ class PydSubmission(BaseModel, extra='allow'):
|
||||
reagents: List[dict]|List[PydReagent] = []
|
||||
samples: List[PydSample]
|
||||
equipment: List[PydEquipment]|None =[]
|
||||
cost_centre: dict|None = Field(default=dict(value=None, missing=True), validate_default=True)
|
||||
|
||||
@field_validator('equipment', mode='before')
|
||||
@classmethod
|
||||
@@ -332,10 +346,28 @@ class PydSubmission(BaseModel, extra='allow'):
|
||||
@field_validator("submitting_lab", mode="before")
|
||||
@classmethod
|
||||
def rescue_submitting_lab(cls, value):
|
||||
if value == None:
|
||||
if value is None:
|
||||
return dict(value=None, missing=True)
|
||||
return value
|
||||
|
||||
|
||||
@field_validator("submitting_lab")
|
||||
@classmethod
|
||||
def lookup_submitting_lab(cls, value):
|
||||
if isinstance(value['value'], str):
|
||||
try:
|
||||
value['value'] = Organization.query(name=value['value']).name
|
||||
except AttributeError:
|
||||
value['value'] = None
|
||||
if value['value'] is None:
|
||||
value['missing'] = True
|
||||
from frontend.widgets.pop_ups import ObjectSelector
|
||||
dlg = ObjectSelector(title="Missing Submitting Lab", message="We need a submitting lab. Please select from the list.", obj_type=Organization)
|
||||
if dlg.exec():
|
||||
value['value'] = dlg.getValues()
|
||||
else:
|
||||
value['value'] = None
|
||||
return value
|
||||
|
||||
@field_validator("rsl_plate_num", mode='before')
|
||||
@classmethod
|
||||
def rescue_rsl_number(cls, value):
|
||||
@@ -427,6 +459,30 @@ class PydSubmission(BaseModel, extra='allow'):
|
||||
output.append(sample)
|
||||
return output
|
||||
|
||||
@field_validator("cost_centre", mode="before")
|
||||
@classmethod
|
||||
def rescue_cost_centre(cls, value):
|
||||
match value:
|
||||
case dict():
|
||||
return value
|
||||
case _:
|
||||
return dict(value=value, missing=True)
|
||||
|
||||
@field_validator("cost_centre")
|
||||
@classmethod
|
||||
def get_cost_centre(cls, value, values):
|
||||
# logger.debug(f"Value coming in for cost_centre: {value}")
|
||||
match value['value']:
|
||||
case None:
|
||||
from backend.db.models import Organization
|
||||
org = Organization.query(name=values.data['submitting_lab']['value'])
|
||||
try:
|
||||
return dict(value=org.cost_centre, missing=True)
|
||||
except AttributeError:
|
||||
return dict(value="xxx", missing=True)
|
||||
case _:
|
||||
return value
|
||||
|
||||
def set_attribute(self, key, value):
|
||||
self.__setattr__(name=key, value=value)
|
||||
|
||||
@@ -599,6 +655,7 @@ class PydSubmission(BaseModel, extra='allow'):
|
||||
else:
|
||||
info = {k:v for k,v in self.improved_dict().items() if isinstance(v, dict)}
|
||||
reagents = self.reagents
|
||||
|
||||
if len(reagents + list(info.keys())) == 0:
|
||||
# logger.warning("No info to fill in, returning")
|
||||
return None
|
||||
@@ -616,14 +673,14 @@ class PydSubmission(BaseModel, extra='allow'):
|
||||
new_reagent = {}
|
||||
new_reagent['type'] = reagent.type
|
||||
new_reagent['lot'] = excel_map[new_reagent['type']]['lot']
|
||||
new_reagent['lot']['value'] = reagent.lot
|
||||
new_reagent['lot']['value'] = reagent.lot or "NA"
|
||||
new_reagent['expiry'] = excel_map[new_reagent['type']]['expiry']
|
||||
new_reagent['expiry']['value'] = reagent.expiry
|
||||
new_reagent['expiry']['value'] = reagent.expiry or "NA"
|
||||
new_reagent['sheet'] = excel_map[new_reagent['type']]['sheet']
|
||||
# name is only present for Bacterial Culture
|
||||
try:
|
||||
new_reagent['name'] = excel_map[new_reagent['type']]['name']
|
||||
new_reagent['name']['value'] = reagent.name
|
||||
new_reagent['name']['value'] = reagent.name or "Not Applicable"
|
||||
except Exception as e:
|
||||
logger.error(f"Couldn't get name due to {e}")
|
||||
new_reagents.append(new_reagent)
|
||||
@@ -657,6 +714,8 @@ class PydSubmission(BaseModel, extra='allow'):
|
||||
# logger.debug(f"Attempting to write lot {reagent['lot']['value']} in: row {reagent['lot']['row']}, column {reagent['lot']['column']}")
|
||||
worksheet.cell(row=reagent['lot']['row'], column=reagent['lot']['column'], value=reagent['lot']['value'])
|
||||
# logger.debug(f"Attempting to write expiry {reagent['expiry']['value']} in: row {reagent['expiry']['row']}, column {reagent['expiry']['column']}")
|
||||
if reagent['expiry']['value'].year == 1970:
|
||||
reagent['expiry']['value'] = "NA"
|
||||
worksheet.cell(row=reagent['expiry']['row'], column=reagent['expiry']['column'], value=reagent['expiry']['value'])
|
||||
try:
|
||||
# logger.debug(f"Attempting to write name {reagent['name']['value']} in: row {reagent['name']['row']}, column {reagent['name']['column']}")
|
||||
@@ -790,14 +849,13 @@ class PydSubmission(BaseModel, extra='allow'):
|
||||
logger.debug(f"Extraction kit: {extraction_kit}. Is it a string? {isinstance(extraction_kit, str)}")
|
||||
if isinstance(extraction_kit, str):
|
||||
extraction_kit = dict(value=extraction_kit)
|
||||
if extraction_kit is not None:
|
||||
if extraction_kit != self.extraction_kit['value']:
|
||||
if extraction_kit is not None and extraction_kit != self.extraction_kit['value']:
|
||||
self.extraction_kit['value'] = extraction_kit['value']
|
||||
reagenttypes = []
|
||||
else:
|
||||
reagenttypes = [item.type for item in self.reagents]
|
||||
else:
|
||||
reagenttypes = [item.type for item in self.reagents]
|
||||
# reagenttypes = []
|
||||
# else:
|
||||
# reagenttypes = [item.type for item in self.reagents]
|
||||
# else:
|
||||
# reagenttypes = [item.type for item in self.reagents]
|
||||
logger.debug(f"Looking up {self.extraction_kit['value']}")
|
||||
ext_kit = KitType.query(name=self.extraction_kit['value'])
|
||||
ext_kit_rtypes = [item.to_pydantic() for item in ext_kit.get_reagents(required=True, submission_type=self.submission_type['value'])]
|
||||
@@ -808,21 +866,26 @@ class PydSubmission(BaseModel, extra='allow'):
|
||||
# logger.debug(f"Checking if reagents match kit contents: {check}")
|
||||
# # what reagent types are in both lists?
|
||||
# missing = list(set(ext_kit_rtypes).difference(reagenttypes))
|
||||
missing = []
|
||||
output_reagents = self.reagents
|
||||
# output_reagents = ext_kit_rtypes
|
||||
logger.debug(f"Already have these reagent types: {reagenttypes}")
|
||||
for rt in ext_kit_rtypes:
|
||||
if rt.type not in reagenttypes:
|
||||
missing.append(rt)
|
||||
if rt.type not in [item.type for item in output_reagents]:
|
||||
output_reagents.append(rt)
|
||||
logger.debug(f"Missing reagents types: {missing}")
|
||||
# missing = []
|
||||
# Exclude any reagenttype found in this pyd not expected in kit.
|
||||
expected_check = [item.type for item in ext_kit_rtypes]
|
||||
output_reagents = [rt for rt in self.reagents if rt.type in expected_check]
|
||||
logger.debug(f"Already have these reagent types: {output_reagents}")
|
||||
missing_check = [item.type for item in output_reagents]
|
||||
missing_reagents = [rt for rt in ext_kit_rtypes if rt.type not in missing_check]
|
||||
missing_reagents += [rt for rt in output_reagents if rt.missing]
|
||||
# for rt in ext_kit_rtypes:
|
||||
# if rt.type not in [item.type for item in output_reagents]:
|
||||
# missing.append(rt)
|
||||
# if rt.type not in [item.type for item in output_reagents]:
|
||||
# output_reagents.append(rt)
|
||||
output_reagents += [rt for rt in missing_reagents if rt not in output_reagents]
|
||||
logger.debug(f"Missing reagents types: {missing_reagents}")
|
||||
# if lists are equal return no problem
|
||||
if len(missing)==0:
|
||||
if len(missing_reagents)==0:
|
||||
result = None
|
||||
else:
|
||||
result = Result(msg=f"The submission you are importing is missing some reagents expected by the kit.\n\nIt looks like you are missing: {[item.type.upper() for item in missing]}\n\nAlternatively, you may have set the wrong extraction kit.\n\nThe program will populate lists using existing reagents.\n\nPlease make sure you check the lots carefully!", status="Warning")
|
||||
result = Result(msg=f"The excel sheet you are importing is missing some reagents expected by the kit.\n\nIt looks like you are missing: {[item.type.upper() for item in missing_reagents]}\n\nAlternatively, you may have set the wrong extraction kit.\n\nThe program will populate lists using existing reagents.\n\nPlease make sure you check the lots carefully!", status="Warning")
|
||||
report.add_result(result)
|
||||
return output_reagents, report
|
||||
|
||||
|
||||
Reference in New Issue
Block a user