landon/Submissions-App
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Moments before disaster.

Browse Source
This commit is contained in:
lwark
2025-01-16 08:36:15 -06:00
parent 5cded949ed
commit bf711369c6
21 changed files with 541 additions and 368 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/submissions/backend/db/__init__.py
View File

@@ -20,11 +20,11 @@ def set_sqlite_pragma(dbapi_connection, connection_record):
cursor = dbapi_connection.cursor()
if ctx.database_schema == "sqlite":
execution_phrase = "PRAGMA foreign_keys=ON"
print(f"Executing '{execution_phrase}' in sql.")
else:
# print("Nothing to execute, returning")
cursor.close()
return
print(f"Executing '{execution_phrase}' in sql.")
cursor.execute(execution_phrase)
cursor.close()
@@ -33,6 +33,17 @@ from .models import *
def update_log(mapper, connection, target):
"""
Updates log table whenever an object with LogMixin is updated.
Args:
mapper ():
connection ():
target ():
Returns:
None
"""
state = inspect(target)
object_name = state.object.truncated_name
update = dict(user=getuser(), time=datetime.now(), object=object_name, changes=[])
@@ -43,6 +54,7 @@ def update_log(mapper, connection, target):
if attr.key == "custom":
continue
added = [str(item) for item in hist.added]
# NOTE: Attributes left out to save space
if attr.key in ['artic_technician', 'submission_sample_associations', 'submission_reagent_associations',
'submission_equipment_associations', 'submission_tips_associations', 'contact_id', 'gel_info',
'gel_controls', 'source_plates']:

11
src/submissions/backend/db/models/__init__.py
View File

@@ -175,7 +175,7 @@ class BaseClass(Base):
try:
records = [obj.to_sub_dict(**kwargs) for obj in objects]
except AttributeError:
records = [obj.to_omnigui_dict() for obj in objects]
records = [obj.omnigui_dict for obj in objects]
return DataFrame.from_records(records)
@classmethod
@@ -241,7 +241,8 @@ class BaseClass(Base):
report.add_result(Result(msg=e, status="Critical"))
return report
def to_omnigui_dict(self) -> dict:
@property
def omnigui_dict(self) -> dict:
"""
For getting any object in an omni-thing friendly output.
@@ -255,8 +256,8 @@ class BaseClass(Base):
pass
return dicto
@classmethod
def get_pydantic_model(cls) -> BaseModel:
@classproperty
def pydantic_model(cls) -> BaseModel:
"""
Gets the pydantic model corresponding to this object.
@@ -271,7 +272,7 @@ class BaseClass(Base):
return model
@classproperty
def add_edit_tooltips(self) -> dict:
def add_edit_tooltips(cls) -> dict:
"""
Gets tooltips for Omni-add-edit

9
src/submissions/backend/db/models/controls.py
View File

@@ -81,7 +81,8 @@ class ControlType(BaseClass):
subtypes = sorted(list(jsoner[genera].keys()), reverse=True)
return subtypes
def get_instance_class(self) -> Control:
@property
def instance_class(self) -> Control:
"""
Retrieves the Control class associated with this controltype
@@ -314,7 +315,7 @@ class PCRControl(Control):
def to_sub_dict(self) -> dict:
"""
Creates dictionary of fields for this object
Creates dictionary of fields for this object.
Returns:
dict: Output dict of name, ct, subtype, target, reagent_lot and submitted_date
@@ -471,8 +472,8 @@ class IridaControl(Control):
_dict[key] = data[genus][key]
yield _dict
@classmethod
def get_modes(cls) -> List[str]:
@classproperty
def modes(cls) -> List[str]:
"""
Get all control modes from database

154
src/submissions/backend/db/models/kits.py
View File

@@ -9,7 +9,7 @@ from sqlalchemy.orm import relationship, validates, Query
from sqlalchemy.ext.associationproxy import association_proxy
from datetime import date, datetime, timedelta
from tools import check_authorization, setup_lookup, Report, Result, check_regex_match, yaml_regex_creator, timezone
from typing import List, Literal, Generator, Any
from typing import List, Literal, Generator, Any, Tuple
from pandas import ExcelFile
from pathlib import Path
from . import Base, BaseClass, Organization, LogMixin
@@ -157,30 +157,62 @@ class KitType(BaseClass):
else:
return (item.reagent_role for item in relevant_associations)
def construct_xl_map_for_use(self, submission_type: str | SubmissionType) -> Generator[(str, str), None, None]:
def construct_xl_map_for_use(self, submission_type: str | SubmissionType) -> Tuple[dict|None, KitType]:
"""
Creates map of locations in Excel workbook for a SubmissionType
Args:
new_kit ():
submission_type (str | SubmissionType): Submissiontype.name
Returns:
Generator[(str, str), None, None]: Tuple containing information locations.
"""
new_kit = self
# NOTE: Account for submission_type variable type.
match submission_type:
case str():
assocs = [item for item in self.kit_reagentrole_associations if
item.submission_type.name == submission_type]
# assocs = [item for item in self.kit_reagentrole_associations if
# item.submission_type.name == submission_type]
logger.debug(f"Query for {submission_type}")
submission_type = SubmissionType.query(name=submission_type)
case SubmissionType():
assocs = [item for item in self.kit_reagentrole_associations if item.submission_type == submission_type]
pass
case _:
raise ValueError(f"Wrong variable type: {type(submission_type)} used!")
for assoc in assocs:
try:
yield assoc.reagent_role.name, assoc.uses
except TypeError:
continue
logger.debug(f"Submission type: {submission_type}, Kit: {self}")
assocs = [item for item in self.kit_reagentrole_associations if item.submission_type == submission_type]
logger.debug(f"Associations: {assocs}")
# NOTE: rescue with submission type's default kit.
if not assocs:
logger.error(
f"No associations found with {self}. Attempting rescue with default kit: {submission_type.default_kit}")
new_kit = submission_type.default_kit
if not new_kit:
from frontend.widgets.pop_ups import ObjectSelector
dlg = ObjectSelector(
title="Select Kit",
message="Could not find reagents for this submission type/kit type combo.\nSelect new kit.",
obj_type=self.__class__,
values=[kit.name for kit in submission_type.kit_types]
)
if dlg.exec():
dlg_result = dlg.parse_form()
logger.debug(f"Dialog result: {dlg_result}")
new_kit = self.__class__.query(name=dlg_result)
logger.debug(f"Query result: {new_kit}")
# return new_kit.construct_xl_map_for_use(submission_type=submission_type)
else:
return None, new_kit
assocs = [item for item in new_kit.kit_reagentrole_associations if item.submission_type == submission_type]
# for assoc in assocs:
# try:
# yield assoc.reagent_role.name, assoc.uses
# except TypeError:
# continue
output = {assoc.reagent_role.name: assoc.uses for assoc in assocs}
logger.debug(f"Output: {output}")
return output, new_kit
@classmethod
@setup_lookup
@@ -444,7 +476,7 @@ class Reagent(BaseClass, LogMixin):
Concrete reagent instance
"""
searchables = ["lot"]
searchables = [dict(label="Lot", field="lot")]
id = Column(INTEGER, primary_key=True) #: primary key
role = relationship("ReagentRole", back_populates="instances",
@@ -548,7 +580,9 @@ class Reagent(BaseClass, LogMixin):
def query_or_create(cls, **kwargs) -> Reagent:
from backend.validators.pydant import PydReagent
new = False
instance = cls.query(**kwargs)
disallowed = ['expiry']
sanitized_kwargs = {k:v for k,v in kwargs.items() if k not in disallowed}
instance = cls.query(**sanitized_kwargs)
if not instance or isinstance(instance, list):
if "role" not in kwargs:
try:
@@ -557,7 +591,7 @@ class Reagent(BaseClass, LogMixin):
pass
instance = PydReagent(**kwargs)
new = True
instance, _ = instance.toSQL()
instance = instance.to_sql()
logger.info(f"Instance from query or create: {instance}")
return instance, new
@@ -644,38 +678,15 @@ class Reagent(BaseClass, LogMixin):
except AttributeError as e:
logger.error(f"Could not set {key} due to {e}")
@check_authorization
def edit_from_search(self, obj, **kwargs):
from frontend.widgets.omni_add_edit import AddEdit
role = ReagentRole.query(kwargs['role'])
if role:
role_name = role.name
else:
role_name = None
# dlg = AddReagentForm(reagent_lot=self.lot, reagent_role=role_name, expiry=self.expiry, reagent_name=self.name)
dlg = AddEdit(parent=None, instance=self)
if dlg.exec():
pyd = dlg.parse_form()
for field in pyd.model_fields:
self.set_attribute(field, pyd.__getattribute__(field))
# for key, value in vars.items():
# match key:
# case "expiry":
# if isinstance(value, str):
# field_value = datetime.strptime(value, "%Y-%m-%d")
# elif isinstance(value, date):
# field_value = datetime.combine(value, datetime.max.time())
# else:
# field_value = value
# field_value.replace(tzinfo=timezone)
# case "role":
# continue
# case _:
# field_value = value
# self.__setattr__(key, field_value)
self.save()
# print(self.__dict__)
@classproperty
def add_edit_tooltips(self):
@@ -801,8 +812,8 @@ class SubmissionType(BaseClass):
"""
return f"<SubmissionType({self.name})>"
@classmethod
def retrieve_template_file(cls) -> bytes:
@classproperty
def basic_template(cls) -> bytes:
"""
Grabs the default excel template file.
@@ -812,7 +823,8 @@ class SubmissionType(BaseClass):
submission_type = cls.query(name="Bacterial Culture")
return submission_type.template_file
def get_template_file_sheets(self) -> List[str]:
@property
def template_file_sheets(self) -> List[str]:
"""
Gets names of sheet in the stored blank form.
@@ -870,15 +882,6 @@ class SubmissionType(BaseClass):
output['custom'] = self.info_map['custom']
return output
def construct_sample_map(self) -> dict:
"""
Returns sample map
Returns:
dict: sample location map
"""
return self.sample_map
def construct_field_map(self, field: Literal['equipment', 'tip']) -> Generator[(str, dict), None, None]:
"""
Make a map of all locations for tips or equipment.
@@ -895,7 +898,8 @@ class SubmissionType(BaseClass):
fmap = {}
yield getattr(item, f"{field}_role").name, fmap
def get_default_kit(self) -> KitType | None:
@property
def default_kit(self) -> KitType | None:
"""
If only one kits exists for this Submission Type, return it.
@@ -941,7 +945,8 @@ class SubmissionType(BaseClass):
raise TypeError(f"Type {type(equipment_role)} is not allowed")
return list(set([item for items in relevant for item in items if item is not None]))
def get_submission_class(self) -> "BasicSubmission":
@property
def submission_class(self) -> "BasicSubmission":
"""
Gets submission class associated with this submission type.
@@ -993,7 +998,8 @@ class SubmissionType(BaseClass):
base_dict = dict(name=self.name)
base_dict['info'] = self.construct_info_map(mode='export')
base_dict['defaults'] = self.defaults
base_dict['samples'] = self.construct_sample_map()
# base_dict['samples'] = self.construct_sample_map()
base_dict['samples'] = self.sample_map
base_dict['kits'] = [item.to_export_dict() for item in self.submissiontype_kit_associations]
return base_dict
@@ -1413,7 +1419,8 @@ class Equipment(BaseClass, LogMixin):
return {k: v for k, v in self.__dict__.items()}
def get_processes(self, submission_type: str | SubmissionType | None = None,
extraction_kit: str | KitType | None = None) -> List[str]:
extraction_kit: str | KitType | None = None,
equipment_role: str | EquipmentRole | None=None) -> List[str]:
"""
Get all processes associated with this Equipment for a given SubmissionType
@@ -1433,6 +1440,8 @@ class Equipment(BaseClass, LogMixin):
continue
if extraction_kit and extraction_kit not in process.kit_types:
continue
if equipment_role and equipment_role not in process.equipment_roles:
continue
yield process
@classmethod
@@ -1489,12 +1498,12 @@ class Equipment(BaseClass, LogMixin):
PydEquipment: pydantic equipment object
"""
from backend.validators.pydant import PydEquipment
processes = self.get_processes(submission_type=submission_type, extraction_kit=extraction_kit)
processes = self.get_processes(submission_type=submission_type, extraction_kit=extraction_kit, equipment_role=role)
return PydEquipment(processes=processes, role=role,
**self.to_dict(processes=False))
@classmethod
def get_regex(cls) -> re.Pattern:
@classproperty
def manufacturer_regex(cls) -> re.Pattern:
"""
Creates regex to determine tip manufacturer
@@ -1809,6 +1818,9 @@ class Process(BaseClass):
def query(cls,
name: str | None = None,
id: int | None = None,
submission_type: str | SubmissionType | None = None,
extraction_kit : str | KitType | None = None,
equipment_role: str | KitType | None = None,
limit: int = 0) -> Process | List[Process]:
"""
Lookup Processes
@@ -1822,6 +1834,30 @@ class Process(BaseClass):
Process|List[Process]: Process(es) matching criteria
"""
query = cls.__database_session__.query(cls)
match submission_type:
case str():
submission_type = SubmissionType.query(name=submission_type)
query = query.filter(cls.submission_types.contains(submission_type))
case SubmissionType():
query = query.filter(cls.submission_types.contains(submission_type))
case _:
pass
match extraction_kit:
case str():
extraction_kit = KitType.query(name=extraction_kit)
query = query.filter(cls.kit_types.contains(extraction_kit))
case KitType():
query = query.filter(cls.kit_types.contains(extraction_kit))
case _:
pass
match equipment_role:
case str():
equipment_role = EquipmentRole.query(name=equipment_role)
query = query.filter(cls.equipment_roles.contains(equipment_role))
case EquipmentRole():
query = query.filter(cls.equipment_roles.contains(equipment_role))
case _:
pass
match name:
case str():
query = query.filter(cls.name == name)
@@ -1975,6 +2011,14 @@ class SubmissionTipsAssociation(BaseClass):
query = query.filter(cls.role_name == role)
return cls.execute_query(query=query, limit=limit, **kwargs)
@classmethod
def query_or_create(cls, tips, submission, role: str, **kwargs):
instance = cls.query(tip_id=tips.id, role=role, submission_id=submission.id, limit=1, **kwargs)
if instance is None:
instance = SubmissionTipsAssociation(submission=submission, tips=tips, role_name=role)
return instance
def to_pydantic(self):
from backend.validators import PydTips
return PydTips(name=self.tips.name, lot=self.tips.lot, role=self.role_name)

2
src/submissions/backend/db/models/organizations.py
View File

@@ -124,7 +124,7 @@ class Contact(BaseClass):
Base of Contact
"""
searchables =[]
searchables = []
id = Column(INTEGER, primary_key=True) #: primary key
name = Column(String(64)) #: contact name

299
src/submissions/backend/db/models/submissions.py
View File

@@ -2,8 +2,6 @@
Models for the main submission and sample types.
"""
from __future__ import annotations
from collections import OrderedDict
from copy import deepcopy
from getpass import getuser
import logging, uuid, tempfile, re, base64, numpy as np, pandas as pd, types, sys
@@ -12,7 +10,7 @@ from zipfile import ZipFile, BadZipfile
from tempfile import TemporaryDirectory, TemporaryFile
from operator import itemgetter
from pprint import pformat
from . import BaseClass, Reagent, SubmissionType, KitType, Organization, Contact, LogMixin
from . import BaseClass, Reagent, SubmissionType, KitType, Organization, Contact, LogMixin, SubmissionReagentAssociation
from sqlalchemy import Column, String, TIMESTAMP, INTEGER, ForeignKey, JSON, FLOAT, case, func
from sqlalchemy.orm import relationship, validates, Query
from sqlalchemy.orm.attributes import flag_modified
@@ -25,13 +23,15 @@ from openpyxl.drawing.image import Image as OpenpyxlImage
from tools import row_map, setup_lookup, jinja_template_loading, rreplace, row_keys, check_key_or_attr, Result, Report, \
report_result, create_holidays_for_year
from datetime import datetime, date, timedelta
from typing import List, Any, Tuple, Literal, Generator
from typing import List, Any, Tuple, Literal, Generator, Type
from dateutil.parser import parse
from pathlib import Path
from jinja2.exceptions import TemplateNotFound
from jinja2 import Template
from PIL import Image
logger = logging.getLogger(f"submissions.{__name__}")
@@ -126,7 +126,7 @@ class BasicSubmission(BaseClass, LogMixin):
def __repr__(self) -> str:
return f"<Submission({self.rsl_plate_num})>"
@classmethod
@classproperty
def jsons(cls) -> List[str]:
"""
Get list of JSON db columns
@@ -136,10 +136,10 @@ class BasicSubmission(BaseClass, LogMixin):
"""
output = [item.name for item in cls.__table__.columns if isinstance(item.type, JSON)]
if issubclass(cls, BasicSubmission) and not cls.__name__ == "BasicSubmission":
output += BasicSubmission.jsons()
output += BasicSubmission.jsons
return output
@classmethod
@classproperty
def timestamps(cls) -> List[str]:
"""
Get list of TIMESTAMP columns
@@ -149,7 +149,7 @@ class BasicSubmission(BaseClass, LogMixin):
"""
output = [item.name for item in cls.__table__.columns if isinstance(item.type, TIMESTAMP)]
if issubclass(cls, BasicSubmission) and not cls.__name__ == "BasicSubmission":
output += BasicSubmission.timestamps()
output += BasicSubmission.timestamps
return output
@classmethod
@@ -259,7 +259,8 @@ class BasicSubmission(BaseClass, LogMixin):
Returns:
dict: sample location map
"""
return cls.get_submission_type(submission_type).construct_sample_map()
# return cls.get_submission_type(submission_type).construct_sample_map()
return cls.get_submission_type(submission_type).sample_map
def generate_associations(self, name: str, extra: str | None = None):
try:
@@ -277,6 +278,7 @@ class BasicSubmission(BaseClass, LogMixin):
Constructs dictionary used in submissions summary
Args:
report (bool, optional): indicates if to be used for a report. Defaults to False.
full_data (bool, optional): indicates if sample dicts to be constructed. Defaults to False.
backup (bool, optional): passed to adjust_to_dict_samples. Defaults to False.
@@ -323,7 +325,8 @@ class BasicSubmission(BaseClass, LogMixin):
logger.error(f"We got an error retrieving reagents: {e}")
reagents = []
finally:
for k, v in self.extraction_kit.construct_xl_map_for_use(self.submission_type):
dicto, _ = self.extraction_kit.construct_xl_map_for_use(self.submission_type)
for k, v in dicto.items():
if k == 'info':
continue
if not any([item['role'] == k for item in reagents]):
@@ -381,7 +384,8 @@ class BasicSubmission(BaseClass, LogMixin):
output["completed_date"] = self.completed_date
return output
def calculate_column_count(self) -> int:
@property
def column_count(self) -> int:
"""
Calculate the number of columns in this submission
@@ -391,13 +395,14 @@ class BasicSubmission(BaseClass, LogMixin):
columns = set([assoc.column for assoc in self.submission_sample_associations])
return len(columns)
def calculate_base_cost(self):
def calculate_base_cost(self) -> None:
"""
Calculates cost of the plate
"""
# NOTE: Calculate number of columns based on largest column number
try:
cols_count_96 = self.calculate_column_count()
cols_count_96 = self.column_count
except Exception as e:
logger.error(f"Column count error: {e}")
# NOTE: Get kit associated with this submission
@@ -418,14 +423,15 @@ class BasicSubmission(BaseClass, LogMixin):
logger.error(f"Calculation error: {e}")
self.run_cost = round(self.run_cost, 2)
def hitpick_plate(self) -> list:
@property
def hitpicked(self) -> list:
"""
Returns positve sample locations for plate
Returns:
list: list of hitpick dictionaries for each sample
"""
output_list = [assoc.to_hitpick() for assoc in self.submission_sample_associations]
output_list = [assoc.hitpicked for assoc in self.submission_sample_associations]
return output_list
@classmethod
@@ -454,7 +460,8 @@ class BasicSubmission(BaseClass, LogMixin):
html = template.render(samples=output_samples, PLATE_ROWS=plate_rows, PLATE_COLUMNS=plate_columns)
return html + "<br/>"
def get_used_equipment(self) -> List[str]:
@property
def used_equipment(self) -> Generator[str, None, None]:
"""
Gets EquipmentRole names associated with this BasicSubmission
@@ -490,6 +497,7 @@ class BasicSubmission(BaseClass, LogMixin):
'source_plates', 'pcr_technician', 'ext_technician', 'artic_technician', 'cost_centre',
'signed_by', 'artic_date', 'gel_barcode', 'gel_date', 'ngs_date', 'contact_phone', 'contact',
'tips', 'gel_image_path', 'custom']
# NOTE: dataframe equals dataframe of all columns not in exclude
df = df.loc[:, ~df.columns.isin(exclude)]
if chronologic:
try:
@@ -531,7 +539,7 @@ class BasicSubmission(BaseClass, LogMixin):
field_value = value
case "ctx" | "csv" | "filepath" | "equipment" | "controls":
return
case item if item in self.jsons():
case item if item in self.jsons:
match key:
case "custom" | "source_plates":
existing = value
@@ -549,7 +557,7 @@ class BasicSubmission(BaseClass, LogMixin):
if isinstance(value, list):
existing += value
else:
if value is not None:
if value:
existing.append(value)
self.__setattr__(key, existing)
# NOTE: Make sure this gets updated by telling SQLAlchemy it's been modified.
@@ -636,12 +644,6 @@ class BasicSubmission(BaseClass, LogMixin):
field_value = [item.to_pydantic() for item in self.submission_tips_associations]
case "submission_type":
field_value = dict(value=self.__getattribute__(key).name, missing=missing)
# case "contact":
# try:
# field_value = dict(value=self.__getattribute__(key).name, missing=missing)
# except AttributeError:
# contact = self.submitting_lab.contacts[0]
# field_value = dict(value=contact.name, missing=True)
case "plate_number":
key = 'rsl_plate_num'
field_value = dict(value=self.rsl_plate_num, missing=missing)
@@ -677,7 +679,7 @@ class BasicSubmission(BaseClass, LogMixin):
return super().save()
@classmethod
def get_regex(cls, submission_type: SubmissionType | str | None = None) -> str:
def get_regex(cls, submission_type: SubmissionType | str | None = None) -> re.Pattern:
"""
Gets the regex string for identifying a certain class of submission.
@@ -685,18 +687,26 @@ class BasicSubmission(BaseClass, LogMixin):
submission_type (SubmissionType | str | None, optional): submission type of interest. Defaults to None.
Returns:
str: _description_
str: String from which regex will be compiled.
"""
# logger.debug(f"Class for regex: {cls}")
try:
return cls.get_submission_type(submission_type).defaults['regex']
regex = cls.get_submission_type(submission_type).defaults['regex']
except AttributeError as e:
logger.error(f"Couldn't get submission type for {cls.__mapper_args__['polymorphic_identity']}")
return ""
regex = None
try:
regex = re.compile(rf"{regex}", flags=re.IGNORECASE | re.VERBOSE)
except re.error as e:
regex = cls.construct_regex()
# logger.debug(f"Returning regex: {regex}")
return regex
# NOTE: Polymorphic functions
@classmethod
def construct_regex(cls) -> re.Pattern:
@classproperty
def regex(cls) -> re.Pattern:
"""
Constructs catchall regex.
@@ -762,7 +772,9 @@ class BasicSubmission(BaseClass, LogMixin):
"""
input_dict['custom'] = {}
for k, v in custom_fields.items():
logger.debug(f"Custom info parser getting type: {v['type']}")
match v['type']:
# NOTE: 'exempt' type not currently used
case "exempt":
continue
case "cell":
@@ -796,7 +808,7 @@ class BasicSubmission(BaseClass, LogMixin):
@classmethod
def custom_validation(cls, pyd: "PydSubmission") -> "PydSubmission":
"""
Performs any final custom parsing of the excel file.
Performs any final parsing of the pydantic object that only needs to be done for this cls.
Args:
input_dict (dict): Parser product up to this point.
@@ -849,6 +861,14 @@ class BasicSubmission(BaseClass, LogMixin):
@classmethod
def custom_sample_writer(self, sample: dict) -> dict:
"""
Performs any final alterations to sample writing unique to this submission type.
Args:
sample (dict): Dictionary of sample values.
Returns:
dict: Finalized dictionary.
"""
return sample
@classmethod
@@ -884,7 +904,7 @@ class BasicSubmission(BaseClass, LogMixin):
logger.error(f"Error making outstr: {e}, sending to RSLNamer to make new plate name.")
outstr = RSLNamer.construct_new_plate_name(data=data)
try:
# NOTE: Grab plate number
# NOTE: Grab plate number as number after a -|_ not followed by another number
plate_number = re.search(r"(?:(-|_)\d)(?!\d)", outstr).group().strip("_").strip("-")
except AttributeError as e:
plate_number = "1"
@@ -910,7 +930,7 @@ class BasicSubmission(BaseClass, LogMixin):
Args:
xl (pd.DataFrame): pcr info form
rsl_plate_number (str): rsl plate num of interest
rsl_plate_num (str): rsl plate num of interest
Returns:
Generator[dict, None, None]: Updated samples
@@ -943,16 +963,16 @@ class BasicSubmission(BaseClass, LogMixin):
submission = cls.query(rsl_plate_num=rsl_plate_num)
name_column = 1
for item in location_map:
logger.debug(f"Checking {item}")
# logger.debug(f"Checking {item}")
worksheet = xl[item['sheet']]
for iii, row in enumerate(worksheet.iter_rows(max_row=len(worksheet['A']), max_col=name_column), start=1):
logger.debug(f"Checking row {row}, {iii}")
# logger.debug(f"Checking row {row}, {iii}")
for cell in row:
logger.debug(f"Checking cell: {cell}, with value {cell.value} against {item['name']}")
# logger.debug(f"Checking cell: {cell}, with value {cell.value} against {item['name']}")
if cell.value == item['name']:
subtype, _ = item['name'].split("-")
target = item['target']
logger.debug(f"Subtype: {subtype}, target: {target}")
# logger.debug(f"Subtype: {subtype}, target: {target}")
ct = worksheet.cell(row=iii, column=item['ct_column']).value
# NOTE: Kind of a stop gap solution to find control reagents.
if subtype == "PC":
@@ -966,7 +986,7 @@ class BasicSubmission(BaseClass, LogMixin):
assoc.reagent.role])), None)
else:
ctrl = None
logger.debug(f"Control reagent: {ctrl.__dict__}")
# logger.debug(f"Control reagent: {ctrl.__dict__}")
try:
ct = float(ct)
except ValueError:
@@ -982,7 +1002,7 @@ class BasicSubmission(BaseClass, LogMixin):
target=target,
reagent_lot=ctrl
)
logger.debug(f"Control output: {pformat(output)}")
# logger.debug(f"Control output: {pformat(output)}")
yield output
@classmethod
@@ -1010,7 +1030,7 @@ class BasicSubmission(BaseClass, LogMixin):
return samples
@classmethod
def get_details_template(cls, base_dict: dict) -> Template:
def get_details_template(cls, base_dict: dict) -> Tuple[dict, Template]:
"""
Get the details jinja template for the correct class
@@ -1040,8 +1060,8 @@ class BasicSubmission(BaseClass, LogMixin):
submission_type_name: str | None = None,
id: int | str | None = None,
rsl_plate_num: str | None = None,
start_date: date | str | int | None = None,
end_date: date | str | int | None = None,
start_date: date | datetime | str | int | None = None,
end_date: date | datetime | str | int | None = None,
reagent: Reagent | str | None = None,
chronologic: bool = False,
limit: int = 0,
@@ -1065,6 +1085,7 @@ class BasicSubmission(BaseClass, LogMixin):
Returns:
models.BasicSubmission | List[models.BasicSubmission]: Submission(s) of interest
"""
from ... import SubmissionReagentAssociation
# NOTE: if you go back to using 'model' change the appropriate cls to model in the query filters
if submission_type is not None:
model = cls.find_polymorphic_subclass(polymorphic_identity=submission_type)
@@ -1078,41 +1099,48 @@ class BasicSubmission(BaseClass, LogMixin):
logger.warning(f"Start date with no end date, using today.")
end_date = date.today()
if end_date is not None and start_date is None:
logger.warning(f"End date with no start date, using Jan 1, 2023")
# NOTE: this query returns a tuple of (object, datetime), need to get only datetime.
start_date = cls.__database_session__.query(cls, func.min(cls.submitted_date)).first()[1]
logger.warning(f"End date with no start date, using first submission date: {start_date}")
if start_date is not None:
match start_date:
case date() | datetime():
start_date = start_date.strftime("%Y-%m-%d")
case date():
pass
case datetime():
start_date = start_date.date()
case int():
start_date = datetime.fromordinal(
datetime(1900, 1, 1).toordinal() + start_date - 2).date().strftime("%Y-%m-%d")
datetime(1900, 1, 1).toordinal() + start_date - 2).date()
case _:
start_date = parse(start_date).strftime("%Y-%m-%d")
start_date = parse(start_date).date()
# start_date = start_date.strftime("%Y-%m-%d")
match end_date:
case date() | datetime():
end_date = end_date + timedelta(days=1)
end_date = end_date.strftime("%Y-%m-%d")
case date():
pass
case datetime():
end_date = end_date# + timedelta(days=1)
# pass
case int():
end_date = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + end_date - 2).date() \
+ timedelta(days=1)
end_date = end_date.strftime("%Y-%m-%d")
end_date = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + end_date - 2).date()# \
# + timedelta(days=1)
case _:
end_date = parse(end_date) + timedelta(days=1)
end_date = end_date.strftime("%Y-%m-%d")
if start_date == end_date:
start_date = datetime.strptime(start_date, "%Y-%m-%d").strftime("%Y-%m-%d %H:%M:%S.%f")
query = query.filter(model.submitted_date == start_date)
else:
query = query.filter(model.submitted_date.between(start_date, end_date))
end_date = parse(end_date).date()# + timedelta(days=1)
# end_date = end_date.strftime("%Y-%m-%d")
start_date = datetime.combine(start_date, datetime.min.time()).strftime("%Y-%m-%d %H:%M:%S.%f")
end_date = datetime.combine(end_date, datetime.max.time()).strftime("%Y-%m-%d %H:%M:%S.%f")
# if start_date == end_date:
# start_date = start_date.strftime("%Y-%m-%d %H:%M:%S.%f")
# query = query.filter(model.submitted_date == start_date)
# else:
query = query.filter(model.submitted_date.between(start_date, end_date))
# NOTE: by reagent (for some reason)
match reagent:
case str():
query = query.join(model.submission_reagent_associations).filter(
SubmissionSampleAssociation.reagent.lot == reagent)
query = query.join(SubmissionReagentAssociation).join(Reagent).filter(
Reagent.lot == reagent)
case Reagent():
query = query.join(model.submission_reagent_associations).join(
SubmissionSampleAssociation.reagent).filter(Reagent.lot == reagent)
query = query.join(SubmissionReagentAssociation).filter(
SubmissionReagentAssociation.reagent == reagent)
case _:
pass
# NOTE: by rsl number (returns only a single value)
@@ -1217,6 +1245,7 @@ class BasicSubmission(BaseClass, LogMixin):
msg = QuestionAsker(title="Delete?", message=f"Are you sure you want to delete {self.rsl_plate_num}?\n")
if msg.exec():
try:
# NOTE: backs up file as xlsx, same as export.
self.backup(fname=fname, full_backup=True)
except BadZipfile:
logger.error("Couldn't open zipfile for writing.")
@@ -1285,16 +1314,16 @@ class BasicSubmission(BaseClass, LogMixin):
if dlg.exec():
equipment = dlg.parse_form()
for equip in equipment:
_, assoc = equip.toSQL(submission=self)
_, assoc = equip.to_sql(submission=self)
try:
assoc.save()
except AttributeError as e:
logger.error(f"Couldn't save association with {equip} due to {e}")
if equip.tips:
for tips in equip.tips:
logger.debug(f"Attempting to add tips assoc: {tips} (pydantic)")
# logger.debug(f"Attempting to add tips assoc: {tips} (pydantic)")
tassoc = tips.to_sql(submission=self)
logger.debug(f"Attempting to add tips assoc: {tips.__dict__} (sql)")
# logger.debug(f"Attempting to add tips assoc: {tips.__dict__} (sql)")
if tassoc not in self.submission_tips_associations:
tassoc.save()
else:
@@ -1320,7 +1349,8 @@ class BasicSubmission(BaseClass, LogMixin):
writer = pyd.to_writer()
writer.xl.save(filename=fname.with_suffix(".xlsx"))
def get_turnaround_time(self) -> Tuple[int | None, bool | None]:
@property
def turnaround_time(self) -> int:
try:
completed = self.completed_date.date()
except AttributeError:
@@ -1328,25 +1358,24 @@ class BasicSubmission(BaseClass, LogMixin):
return self.calculate_turnaround(start_date=self.submitted_date.date(), end_date=completed)
@classmethod
def calculate_turnaround(cls, start_date: date | None = None, end_date: date | None = None) -> Tuple[
int | None, bool | None]:
if 'pytest' not in sys.modules:
from tools import ctx
else:
from test_settings import ctx
def calculate_turnaround(cls, start_date: date | None = None, end_date: date | None = None) -> int:
"""
Calculates number of business days between data submitted and date completed
Args:
start_date (date, optional): Date submitted. defaults to None.
end_date (date, optional): Date completed. defaults to None.
Returns:
int: Number of business days.
"""
if not end_date:
return None, None
return None
try:
delta = np.busday_count(start_date, end_date, holidays=create_holidays_for_year(start_date.year)) + 1
except ValueError:
return None, None
try:
tat = cls.get_default_info("turnaround_time")
except (AttributeError, KeyError):
tat = None
if not tat:
tat = ctx.TaT_threshold
return delta, delta <= tat
return None
return delta
# NOTE: Below are the custom submission types
@@ -1385,7 +1414,7 @@ class BacterialCulture(BasicSubmission):
return template
@classmethod
def custom_validation(cls, pyd) -> dict:
def custom_validation(cls, pyd) -> "PydSubmission":
"""
Extends parent. Currently finds control sample and adds to reagents.
@@ -1395,7 +1424,7 @@ class BacterialCulture(BasicSubmission):
info_map (dict | None, optional): _description_. Defaults to None.
Returns:
dict: Updated dictionary.
PydSubmission: Updated pydantic.
"""
from . import ControlType
pyd = super().custom_validation(pyd)
@@ -1549,9 +1578,10 @@ class Wastewater(BasicSubmission):
"""
samples = [item for item in super().parse_pcr(xl=xl, rsl_plate_num=rsl_plate_num)]
# NOTE: Due to having to run through samples in for loop we need to convert to list.
# NOTE: Also, you can't change the size of a list while iterating it, so don't even think about it.
output = []
for sample in samples:
logger.debug(sample)
# logger.debug(sample)
# NOTE: remove '-{target}' from controls
sample['sample'] = re.sub('-N\\d*$', '', sample['sample'])
# NOTE: if sample is already in output skip
@@ -1559,7 +1589,7 @@ class Wastewater(BasicSubmission):
logger.warning(f"Already have {sample['sample']}")
continue
# NOTE: Set ct values
logger.debug(f"Sample ct: {sample['ct']}")
# logger.debug(f"Sample ct: {sample['ct']}")
sample[f"ct_{sample['target'].lower()}"] = sample['ct'] if isinstance(sample['ct'], float) else 0.0
# NOTE: Set assessment
logger.debug(f"Sample assessemnt: {sample['assessment']}")
@@ -1578,7 +1608,7 @@ class Wastewater(BasicSubmission):
except KeyError:
pass
output.append(sample)
# NOTE: And then convert back to list ot keep fidelity with parent method.
# NOTE: And then convert back to list to keep fidelity with parent method.
for sample in output:
yield sample
@@ -1644,7 +1674,7 @@ class Wastewater(BasicSubmission):
return events
@report_result
def link_pcr(self, obj):
def link_pcr(self, obj) -> Report:
"""
PYQT6 function to add PCR info to this submission
@@ -1660,7 +1690,8 @@ class Wastewater(BasicSubmission):
report.add_result(Result(msg="No file selected, cancelling.", status="Warning"))
return report
parser = PCRParser(filepath=fname, submission=self)
self.set_attribute("pcr_info", parser.pcr)
self.set_attribute("pcr_info", parser.pcr_info)
# NOTE: These are generators here, need to expand.
pcr_samples = [sample for sample in parser.samples]
pcr_controls = [control for control in parser.controls]
self.save(original=False)
@@ -1674,19 +1705,19 @@ class Wastewater(BasicSubmission):
result = assoc.save()
report.add_result(result)
controltype = ControlType.query(name="PCR Control")
submitted_date = datetime.strptime(" ".join(parser.pcr['run_start_date/time'].split(" ")[:-1]),
submitted_date = datetime.strptime(" ".join(parser.pcr_info['run_start_date/time'].split(" ")[:-1]),
"%Y-%m-%d %I:%M:%S %p")
for control in pcr_controls:
logger.debug(f"Control coming into save: {control}")
# logger.debug(f"Control coming into save: {control}")
new_control = PCRControl(**control)
new_control.submitted_date = submitted_date
new_control.controltype = controltype
new_control.submission = self
logger.debug(f"Control coming into save: {new_control.__dict__}")
# logger.debug(f"Control coming into save: {new_control.__dict__}")
new_control.save()
return report
def update_subsampassoc(self, sample: BasicSample, input_dict: dict):
def update_subsampassoc(self, sample: BasicSample, input_dict: dict) -> SubmissionSampleAssociation:
"""
Updates a joined submission sample association by assigning ct values to n1 or n2 based on alphabetical sorting.
@@ -1722,7 +1753,7 @@ class WastewaterArtic(BasicSubmission):
artic_date = Column(TIMESTAMP) #: Date Artic Performed
ngs_date = Column(TIMESTAMP) #: Date submission received
gel_date = Column(TIMESTAMP) #: Date submission received
gel_barcode = Column(String(16))
gel_barcode = Column(String(16)) #: Identifier for the used gel.
__mapper_args__ = dict(polymorphic_identity="Wastewater Artic",
polymorphic_load="inline",
@@ -1769,6 +1800,16 @@ class WastewaterArtic(BasicSubmission):
from openpyxl_image_loader.sheet_image_loader import SheetImageLoader
def scrape_image(wb: Workbook, info_dict: dict) -> Image or None:
"""
Pulls image from excel workbook
Args:
wb (Workbook): Workbook of interest.
info_dict (dict): Location map.
Returns:
Image or None: Image of interest.
"""
ws = wb[info_dict['sheet']]
img_loader = SheetImageLoader(ws)
for ii in range(info_dict['start_row'], info_dict['end_row'] + 1):
@@ -1805,7 +1846,7 @@ class WastewaterArtic(BasicSubmission):
if datum['plate'] in ["None", None, ""]:
continue
else:
datum['plate'] = RSLNamer(filename=datum['plate'], sub_type="Wastewater").parsed_name
datum['plate'] = RSLNamer(filename=datum['plate'], submission_type="Wastewater").parsed_name
if xl is not None:
try:
input_dict['csv'] = xl["hitpicks_csv_to_export"]
@@ -1864,6 +1905,7 @@ class WastewaterArtic(BasicSubmission):
Returns:
str: Updated name.
"""
logger.debug(f"Incoming String: {instr}")
try:
# NOTE: Deal with PCR file.
instr = re.sub(r"Artic", "", instr, flags=re.IGNORECASE)
@@ -1900,8 +1942,7 @@ class WastewaterArtic(BasicSubmission):
input_dict['source_plate_number'] = int(input_dict['source_plate_number'])
except (ValueError, KeyError):
input_dict['source_plate_number'] = 0
# NOTE: Because generate_sample_object needs the submitter_id and the artic has the "({origin well})"
# at the end, this has to be done here. No moving to sqlalchemy object :(
# NOTE: Because generate_sample_object needs the submitter_id and the artic has the "({origin well})" at the end, this has to be done here. No moving to sqlalchemy object :(
input_dict['submitter_id'] = re.sub(r"\s\(.+\)\s?$", "", str(input_dict['submitter_id'])).strip()
try:
input_dict['ww_processing_num'] = input_dict['sample_name_(lims)']
@@ -1988,7 +2029,11 @@ class WastewaterArtic(BasicSubmission):
except AttributeError:
plate_num = "1"
plate_num = plate_num.strip("-")
repeat_num = re.search(r"R(?P<repeat>\d)?$", "PBS20240426-2R").groups()[0]
# repeat_num = re.search(r"R(?P<repeat>\d)?$", "PBS20240426-2R").groups()[0]
try:
repeat_num = re.search(r"R(?P<repeat>\d)?$", processed).groups()[0]
except:
repeat_num = None
if repeat_num is None and "R" in plate_num:
repeat_num = "1"
plate_num = re.sub(r"R", rf"R{repeat_num}", plate_num)
@@ -2192,7 +2237,7 @@ class BasicSample(BaseClass, LogMixin):
Base of basic sample which polymorphs into BCSample and WWSample
"""
searchables = ['submitter_id']
searchables = [dict(label="Submitter ID", field="submitter_id")]
id = Column(INTEGER, primary_key=True) #: primary key
submitter_id = Column(String(64), nullable=False, unique=True) #: identification from submitter
@@ -2242,7 +2287,7 @@ class BasicSample(BaseClass, LogMixin):
except AttributeError:
return f"<Sample({self.submitter_id})"
@classmethod
@classproperty
def timestamps(cls) -> List[str]:
"""
Constructs a list of all attributes stored as SQL Timestamps
@@ -2252,7 +2297,7 @@ class BasicSample(BaseClass, LogMixin):
"""
output = [item.name for item in cls.__table__.columns if isinstance(item.type, TIMESTAMP)]
if issubclass(cls, BasicSample) and not cls.__name__ == "BasicSample":
output += BasicSample.timestamps()
output += BasicSample.timestamps
return output
def to_sub_dict(self, full_data: bool = False) -> dict:
@@ -2293,7 +2338,7 @@ class BasicSample(BaseClass, LogMixin):
@classmethod
def find_polymorphic_subclass(cls, polymorphic_identity: str | None = None,
attrs: dict | None = None) -> BasicSample:
attrs: dict | None = None) -> Type[BasicSample]:
"""
Retrieves subclasses of BasicSample based on type name.
@@ -2340,8 +2385,8 @@ class BasicSample(BaseClass, LogMixin):
"""
return input_dict
@classmethod
def get_details_template(cls) -> Template:
@classproperty
def details_template(cls) -> Template:
"""
Get the details jinja template for the correct class
@@ -2458,15 +2503,15 @@ class BasicSample(BaseClass, LogMixin):
def delete(self):
raise AttributeError(f"Delete not implemented for {self.__class__}")
@classmethod
def get_searchables(cls) -> List[dict]:
"""
Delivers a list of fields that can be used in fuzzy search.
Returns:
List[str]: List of fields.
"""
return [dict(label="Submitter ID", field="submitter_id")]
# @classmethod
# def get_searchables(cls) -> List[dict]:
# """
# Delivers a list of fields that can be used in fuzzy search.
#
# Returns:
# List[str]: List of fields.
# """
# return [dict(label="Submitter ID", field="submitter_id")]
@classmethod
def samples_to_df(cls, sample_list: List[BasicSample], **kwargs) -> pd.DataFrame:
@@ -2504,6 +2549,16 @@ class BasicSample(BaseClass, LogMixin):
pass
def edit_from_search(self, obj, **kwargs):
"""
Function called form search. "Edit" is dependent on function as this one just shows details.
Args:
obj (__type__): Parent widget.
**kwargs (): Required for all edit from search functions.
Returns:
"""
self.show_details(obj)
@@ -2514,7 +2569,7 @@ class WastewaterSample(BasicSample):
Derivative wastewater sample
"""
searchables = BasicSample.searchables + ['ww_processing_num', 'ww_full_sample_id', 'rsl_number']
# searchables = BasicSample.searchables + ['ww_processing_num', 'ww_full_sample_id', 'rsl_number']
id = Column(INTEGER, ForeignKey('_basicsample.id'), primary_key=True)
ww_processing_num = Column(String(64)) #: wastewater processing number
@@ -2594,15 +2649,15 @@ class WastewaterSample(BasicSample):
# logger.debug(pformat(output_dict, indent=4))
return output_dict
@classmethod
def get_searchables(cls) -> List[str]:
@classproperty
def searchables(cls) -> List[dict]:
"""
Delivers a list of fields that can be used in fuzzy search. Extends parent.
Returns:
List[str]: List of fields.
"""
searchables = super().get_searchables()
searchables = super().searchables
for item in ["ww_processing_num", "ww_full_sample_id", "rsl_number"]:
label = item.strip("ww_").replace("_", " ").replace("rsl", "RSL").title()
searchables.append(dict(label=label, field=item))
@@ -2726,7 +2781,8 @@ class SubmissionSampleAssociation(BaseClass):
from backend.validators import PydSample
return PydSample(**self.to_sub_dict())
def to_hitpick(self) -> dict | None:
@property
def hitpicked(self) -> dict | None:
"""
Outputs a dictionary usable for html plate maps.
@@ -2948,14 +3004,15 @@ class WastewaterAssociation(SubmissionSampleAssociation):
logger.error(f"Couldn't check positives for {self.sample.rsl_number}. Looks like there isn't PCR data.")
return sample
def to_hitpick(self) -> dict | None:
@property
def hitpicked(self) -> dict | None:
"""
Outputs a dictionary usable for html plate maps. Extends parent
Returns:
dict: dictionary of sample id, row and column in elution plate
"""
sample = super().to_hitpick()
sample = super().hitpicked
try:
scaler = max([self.ct_n1, self.ct_n2])
except TypeError:

183
src/submissions/backend/excel/parser.py
View File

@@ -59,25 +59,27 @@ class SheetParser(object):
Pulls basic information from the excel sheet
"""
parser = InfoParser(xl=self.xl, submission_type=self.submission_type, sub_object=self.sub_object)
info = parser.parse_info()
self.info_map = parser.map
# NOTE: in order to accommodate generic submission types we have to check for the type in the excel sheet and
# rerun accordingly
# info = parser.parsed_info
self.info_map = parser.info_map
# NOTE: in order to accommodate generic submission types we have to check for the type in the excel sheet and rerun accordingly
try:
check = info['submission_type']['value'] not in [None, "None", "", " "]
except KeyError:
check = parser.parsed_info['submission_type']['value'] not in [None, "None", "", " "]
except KeyError as e:
logger.error(f"Couldn't check submission type due to KeyError: {e}")
return
logger.info(
f"Checking for updated submission type: {self.submission_type.name} against new: {info['submission_type']['value']}")
if self.submission_type.name != info['submission_type']['value']:
f"Checking for updated submission type: {self.submission_type.name} against new: {parser.parsed_info['submission_type']['value']}")
if self.submission_type.name != parser.parsed_info['submission_type']['value']:
if check:
self.submission_type = SubmissionType.query(name=info['submission_type']['value'])
# NOTE: If initial submission type doesn't match parsed submission type, defer to parsed submission type.
self.submission_type = SubmissionType.query(name=parser.parsed_info['submission_type']['value'])
logger.info(f"Updated self.submission_type to {self.submission_type}. Rerunning parse.")
self.parse_info()
else:
self.submission_type = RSLNamer.retrieve_submission_type(filename=self.filepath)
self.parse_info()
[self.sub.__setitem__(k, v) for k, v in info.items()]
for k, v in parser.parsed_info.items():
self.sub.__setitem__(k, v)
def parse_reagents(self, extraction_kit: str | None = None):
"""
@@ -90,28 +92,28 @@ class SheetParser(object):
extraction_kit = self.sub['extraction_kit']
parser = ReagentParser(xl=self.xl, submission_type=self.submission_type,
extraction_kit=extraction_kit)
self.sub['reagents'] = parser.parse_reagents()
self.sub['reagents'] = parser.parsed_reagents
def parse_samples(self):
"""
Calls sample parser to pull info from the excel sheet
"""
parser = SampleParser(xl=self.xl, submission_type=self.submission_type)
self.sub['samples'] = parser.parse_samples()
self.sub['samples'] = parser.parsed_samples
def parse_equipment(self):
"""
Calls equipment parser to pull info from the excel sheet
"""
parser = EquipmentParser(xl=self.xl, submission_type=self.submission_type)
self.sub['equipment'] = parser.parse_equipment()
self.sub['equipment'] = parser.parsed_equipment
def parse_tips(self):
"""
Calls tips parser to pull info from the excel sheet
"""
parser = TipParser(xl=self.xl, submission_type=self.submission_type)
self.sub['tips'] = parser.parse_tips()
self.sub['tips'] = parser.parsed_tips
def import_kit_validation_check(self):
"""
@@ -156,23 +158,23 @@ class InfoParser(object):
if sub_object is None:
sub_object = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=submission_type.name)
self.submission_type_obj = submission_type
self.submission_type = dict(value=self.submission_type_obj.name, missing=True)
self.sub_object = sub_object
self.map = self.fetch_submission_info_map()
self.xl = xl
def fetch_submission_info_map(self) -> dict:
@property
def info_map(self) -> dict:
"""
Gets location of basic info from the submission_type object in the database.
Returns:
dict: Location map of all info for this submission type
"""
self.submission_type = dict(value=self.submission_type_obj.name, missing=True)
info_map = self.sub_object.construct_info_map(submission_type=self.submission_type_obj, mode="read")
# NOTE: Get the parse_info method from the submission type specified
return info_map
return self.sub_object.construct_info_map(submission_type=self.submission_type_obj, mode="read")
def parse_info(self) -> dict:
@property
def parsed_info(self) -> dict:
"""
Pulls basic info from the excel sheet.
@@ -184,7 +186,7 @@ class InfoParser(object):
for sheet in self.xl.sheetnames:
ws = self.xl[sheet]
relevant = []
for k, v in self.map.items():
for k, v in self.info_map.items():
# NOTE: If the value is hardcoded put it in the dictionary directly. Ex. Artic kit
if k == "custom":
continue
@@ -215,7 +217,7 @@ class InfoParser(object):
case "submitted_date":
value, missing = is_missing(value)
# NOTE: is field a JSON? Includes: Extraction info, PCR info, comment, custom
case thing if thing in self.sub_object.jsons():
case thing if thing in self.sub_object.jsons:
value, missing = is_missing(value)
if missing: continue
value = dict(name=f"Parser_{sheet}", text=value, time=datetime.now())
@@ -232,7 +234,7 @@ class InfoParser(object):
except (KeyError, IndexError):
continue
# NOTE: Return after running the parser components held in submission object.
return self.sub_object.custom_info_parser(input_dict=dicto, xl=self.xl, custom_fields=self.map['custom'])
return self.sub_object.custom_info_parser(input_dict=dicto, xl=self.xl, custom_fields=self.info_map['custom'])
class ReagentParser(object):
@@ -252,16 +254,17 @@ class ReagentParser(object):
if isinstance(submission_type, str):
submission_type = SubmissionType.query(name=submission_type)
self.submission_type_obj = submission_type
if not sub_object:
sub_object = submission_type.submission_class
self.sub_object = sub_object
if isinstance(extraction_kit, dict):
extraction_kit = extraction_kit['value']
self.kit_object = KitType.query(name=extraction_kit)
self.map = self.fetch_kit_info_map(submission_type=submission_type)
logger.debug(f"Setting map: {self.map}")
# self.kit_map = self.kit_map(submission_type=submission_type)
self.xl = xl
# @report_result
def fetch_kit_info_map(self, submission_type: str | SubmissionType) -> Tuple[Report, dict]:
@property
def kit_map(self) -> dict:
"""
Gets location of kit reagents from database
@@ -271,38 +274,41 @@ class ReagentParser(object):
Returns:
dict: locations of reagent info for the kit.
"""
report = Report()
if isinstance(submission_type, dict):
submission_type = submission_type['value']
if isinstance(submission_type, str):
submission_type = SubmissionType.query(name=submission_type)
reagent_map = {k: v for k, v in self.kit_object.construct_xl_map_for_use(submission_type)}
# report = Report()
# if isinstance(submission_type, dict):
# submission_type = submission_type['value']
# if isinstance(submission_type, str):
# submission_type = SubmissionType.query(name=submission_type)
logger.debug("Running kit map")
associations, self.kit_object = self.kit_object.construct_xl_map_for_use(submission_type=self.submission_type_obj)
reagent_map = {k: v for k, v in associations.items() if k != 'info'}
try:
del reagent_map['info']
except KeyError:
pass
# NOTE: If reagent map is empty, maybe the wrong kit was given, check if there's only one kit for that submission type and use it if so.
if not reagent_map:
temp_kit_object = self.submission_type_obj.get_default_kit()
if temp_kit_object:
self.kit_object = temp_kit_object
logger.warning(f"Attempting to salvage with default kit {self.kit_object} and submission_type: {self.submission_type_obj}")
return self.fetch_kit_info_map(submission_type=self.submission_type_obj)
else:
logger.error(f"Still no reagent map, displaying error.")
try:
ext_kit_loc = self.submission_type_obj.info_map['extraction_kit']['read'][0]
location_string = f"Sheet: {ext_kit_loc['sheet']}, Row: {ext_kit_loc['row']}, Column: {ext_kit_loc['column']}?"
except (IndexError, KeyError):
location_string = ""
report.add_result(Result(owner=__name__, code=0,
msg=f"No kit map found for {self.kit_object.name}.\n\n"
f"Are you sure you put the right kit in:\n\n{location_string}?",
status="Critical"))
logger.debug(f"Here is the map coming out: {reagent_map}")
# # NOTE: If reagent map is empty, maybe the wrong kit was given, check if there's only one kit for that submission type and use it if so.
# if not reagent_map:
# temp_kit_object = self.submission_type_obj.default_kit
# if temp_kit_object:
# self.kit_object = temp_kit_object
# logger.warning(f"Attempting to salvage with default kit {self.kit_object} and submission_type: {self.submission_type_obj}")
# return self.fetch_kit_map(submission_type=self.submission_type_obj)
# else:
# logger.error(f"Still no reagent map, displaying error.")
# try:
# ext_kit_loc = self.submission_type_obj.info_map['extraction_kit']['read'][0]
# location_string = f"Sheet: {ext_kit_loc['sheet']}, Row: {ext_kit_loc['row']}, Column: {ext_kit_loc['column']}?"
# except (IndexError, KeyError):
# location_string = ""
# report.add_result(Result(owner=__name__, code=0,
# msg=f"No kit map found for {self.kit_object.name}.\n\n"
# f"Are you sure you put the right kit in:\n\n{location_string}?",
# status="Critical"))
# logger.debug(f"Here is the map coming out: {reagent_map}")
return reagent_map
def parse_reagents(self) -> Generator[dict, None, None]:
@property
def parsed_reagents(self) -> Generator[dict, None, None]:
"""
Extracts reagent information from the Excel form.
@@ -311,7 +317,7 @@ class ReagentParser(object):
"""
for sheet in self.xl.sheetnames:
ws = self.xl[sheet]
relevant = {k.strip(): v for k, v in self.map.items() if sheet in self.map[k]['sheet']}
relevant = {k.strip(): v for k, v in self.kit_map.items() if sheet in self.kit_map[k]['sheet']}
if not relevant:
continue
for item in relevant:
@@ -367,11 +373,14 @@ class SampleParser(object):
f"Sample parser attempting to fetch submission class with polymorphic identity: {self.submission_type}")
sub_object = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=self.submission_type)
self.sub_object = sub_object
self.sample_info_map = self.fetch_sample_info_map(submission_type=submission_type, sample_map=sample_map)
self.plate_map_samples = self.parse_plate_map()
self.lookup_samples = self.parse_lookup_table()
self.sample_type = self.sub_object.get_default_info("sample_type", submission_type=submission_type)
self.samp_object = BasicSample.find_polymorphic_subclass(polymorphic_identity=self.sample_type)
# self.sample_map = self.sample_map(submission_type=submission_type, sample_map=sample_map)
# self.plate_map_samples = self.parse_plate_map()
# self.lookup_samples = self.parse_lookup_table()
def fetch_sample_info_map(self, submission_type: str, sample_map: dict | None = None) -> dict:
@property
def sample_map(self) -> dict:
"""
Gets info locations in excel book for submission type.
@@ -381,15 +390,16 @@ class SampleParser(object):
Returns:
dict: Info locations.
"""
self.sample_type = self.sub_object.get_default_info("sample_type", submission_type=submission_type)
self.samp_object = BasicSample.find_polymorphic_subclass(polymorphic_identity=self.sample_type)
if sample_map is None:
sample_info_map = self.sub_object.construct_sample_map(submission_type=self.submission_type_obj)
else:
sample_info_map = sample_map
return sample_info_map
def parse_plate_map(self) -> List[dict]:
# if sample_map is None:
# sample_info_map = self.sub_object.construct_sample_map(submission_type=self.submission_type_obj)
# else:
# sample_info_map = sample_map
# return sample_info_map
return self.sub_object.construct_sample_map(submission_type=self.submission_type_obj)
@property
def plate_map_samples(self) -> List[dict]:
"""
Parse sample location/name from plate map
@@ -397,7 +407,7 @@ class SampleParser(object):
List[dict]: List of sample ids and locations.
"""
invalids = [0, "0", "EMPTY"]
smap = self.sample_info_map['plate_map']
smap = self.sample_map['plate_map']
ws = self.xl[smap['sheet']]
plate_map_samples = []
for ii, row in enumerate(range(smap['start_row'], smap['end_row'] + 1), start=1):
@@ -414,7 +424,8 @@ class SampleParser(object):
pass
return plate_map_samples
def parse_lookup_table(self) -> List[dict]:
@property
def lookup_samples(self) -> List[dict]:
"""
Parse misc info from lookup table.
@@ -422,7 +433,7 @@ class SampleParser(object):
List[dict]: List of basic sample info.
"""
lmap = self.sample_info_map['lookup_table']
lmap = self.sample_map['lookup_table']
ws = self.xl[lmap['sheet']]
lookup_samples = []
for ii, row in enumerate(range(lmap['start_row'], lmap['end_row'] + 1), start=1):
@@ -441,7 +452,8 @@ class SampleParser(object):
lookup_samples.append(self.samp_object.parse_sample(row_dict))
return lookup_samples
def parse_samples(self) -> Generator[dict, None, None]:
@property
def parsed_samples(self) -> Generator[dict, None, None]:
"""
Merges sample info from lookup table and plate map.
@@ -461,7 +473,7 @@ class SampleParser(object):
pass
yield new
else:
merge_on_id = self.sample_info_map['lookup_table']['merge_on_id']
merge_on_id = self.sample_map['lookup_table']['merge_on_id']
logger.info(f"Merging sample info using {merge_on_id}")
plate_map_samples = sorted(copy(self.plate_map_samples), key=itemgetter('id'))
lookup_samples = sorted(copy(self.lookup_samples), key=itemgetter(merge_on_id))
@@ -507,9 +519,10 @@ class EquipmentParser(object):
submission_type = SubmissionType.query(name=submission_type)
self.submission_type = submission_type
self.xl = xl
self.map = self.fetch_equipment_map()
# self.equipment_map = self.fetch_equipment_map()
def fetch_equipment_map(self) -> dict:
@property
def equipment_map(self) -> dict:
"""
Gets the map of equipment locations in the submission type's spreadsheet
@@ -528,14 +541,15 @@ class EquipmentParser(object):
Returns:
str: asset number
"""
regex = Equipment.get_regex()
regex = Equipment.manufacturer_regex
try:
return regex.search(input).group().strip("-")
except AttributeError as e:
logger.error(f"Error getting asset number for {input}: {e}")
return input
def parse_equipment(self) -> Generator[dict, None, None]:
@property
def parsed_equipment(self) -> Generator[dict, None, None]:
"""
Scrapes equipment from xl sheet
@@ -545,7 +559,7 @@ class EquipmentParser(object):
for sheet in self.xl.sheetnames:
ws = self.xl[sheet]
try:
relevant = {k: v for k, v in self.map.items() if v['sheet'] == sheet}
relevant = {k: v for k, v in self.equipment_map.items() if v['sheet'] == sheet}
except (TypeError, KeyError) as e:
logger.error(f"Error creating relevant equipment list: {e}")
continue
@@ -566,7 +580,7 @@ class EquipmentParser(object):
nickname=eq.nickname)
except AttributeError:
logger.error(f"Unable to add {eq} to list.")
continue
class TipParser(object):
"""
@@ -583,9 +597,10 @@ class TipParser(object):
submission_type = SubmissionType.query(name=submission_type)
self.submission_type = submission_type
self.xl = xl
self.map = self.fetch_tip_map()
# self.map = self.fetch_tip_map()
def fetch_tip_map(self) -> dict:
@property
def tip_map(self) -> dict:
"""
Gets the map of equipment locations in the submission type's spreadsheet
@@ -594,7 +609,8 @@ class TipParser(object):
"""
return {k: v for k, v in self.submission_type.construct_field_map("tip")}
def parse_tips(self) -> List[dict]:
@property
def parsed_tips(self) -> Generator[dict, None, None]:
"""
Scrapes equipment from xl sheet
@@ -604,7 +620,7 @@ class TipParser(object):
for sheet in self.xl.sheetnames:
ws = self.xl[sheet]
try:
relevant = {k: v for k, v in self.map.items() if v['sheet'] == sheet}
relevant = {k: v for k, v in self.tip_map.items() if v['sheet'] == sheet}
except (TypeError, KeyError) as e:
logger.error(f"Error creating relevant equipment list: {e}")
continue
@@ -653,11 +669,12 @@ class PCRParser(object):
else:
self.submission_obj = submission
rsl_plate_num = self.submission_obj.rsl_plate_num
self.pcr = self.parse_general()
# self.pcr = self.parse_general()
self.samples = self.submission_obj.parse_pcr(xl=self.xl, rsl_plate_num=rsl_plate_num)
self.controls = self.submission_obj.parse_pcr_controls(xl=self.xl, rsl_plate_num=rsl_plate_num)
def parse_general(self):
@property
def pcr_info(self) -> dict:
"""
Parse general info rows for all types of PCR results

24
src/submissions/backend/excel/reports.py
View File

@@ -1,6 +1,7 @@
'''
Contains functions for generating summary reports
'''
import sys
from pprint import pformat
from pandas import DataFrame, ExcelWriter
import logging
@@ -8,7 +9,7 @@ from pathlib import Path
from datetime import date
from typing import Tuple
from backend.db.models import BasicSubmission
from tools import jinja_template_loading, get_first_blank_df_row, row_map
from tools import jinja_template_loading, get_first_blank_df_row, row_map, ctx
from PyQt6.QtWidgets import QWidget
from openpyxl.worksheet.worksheet import Worksheet
@@ -18,6 +19,9 @@ env = jinja_template_loading()
class ReportArchetype(object):
"""
Made for children to inherit 'write_report", etc.
"""
def write_report(self, filename: Path | str, obj: QWidget | None = None):
"""
@@ -168,7 +172,21 @@ class TurnaroundMaker(ReportArchetype):
Returns:
"""
days, tat_ok = sub.get_turnaround_time()
if 'pytest' not in sys.modules:
from tools import ctx
else:
from test_settings import ctx
days = sub.turnaround_time
try:
tat = sub.get_default_info("turnaround_time")
except (AttributeError, KeyError):
tat = None
if not tat:
tat = ctx.TaT_threshold
try:
tat_ok = days <= tat
except TypeError:
return {}
return dict(name=str(sub.rsl_plate_num), days=days, submitted_date=sub.submitted_date,
completed_date=sub.completed_date, acceptable=tat_ok)
@@ -179,5 +197,3 @@ class ChartReportMaker(ReportArchetype):
self.df = df
self.sheet_name = sheet_name

22
src/submissions/backend/excel/writer.py
View File

@@ -45,7 +45,7 @@ class SheetWriter(object):
template = self.submission_type.template_file
if not template:
logger.error(f"No template file found, falling back to Bacterial Culture")
template = SubmissionType.retrieve_template_file()
template = SubmissionType.basic_template
workbook = load_workbook(BytesIO(template))
self.xl = workbook
self.write_info()
@@ -155,8 +155,11 @@ class InfoWriter(object):
"""
final_info = {}
for k, v in self.info:
if k == "custom":
continue
match k:
case "custom":
continue
# case "comment":
# NOTE: merge all comments to fit in single cell.
if k == "comment" and isinstance(v['value'], list):
json_join = [item['text'] for item in v['value'] if 'text' in item.keys()]
@@ -170,6 +173,7 @@ class InfoWriter(object):
for loc in locations:
sheet = self.xl[loc['sheet']]
try:
logger.debug(f"Writing {v['value']} to row {loc['row']} and column {loc['column']}")
sheet.cell(row=loc['row'], column=loc['column'], value=v['value'])
except AttributeError as e:
logger.error(f"Can't write {k} to that cell due to AttributeError: {e}")
@@ -196,9 +200,13 @@ class ReagentWriter(object):
self.xl = xl
if isinstance(submission_type, str):
submission_type = SubmissionType.query(name=submission_type)
self.submission_type_obj = submission_type
if isinstance(extraction_kit, str):
extraction_kit = KitType.query(name=extraction_kit)
reagent_map = {k: v for k, v in extraction_kit.construct_xl_map_for_use(submission_type)}
self.kit_object = extraction_kit
associations, self.kit_object = self.kit_object.construct_xl_map_for_use(
submission_type=self.submission_type_obj)
reagent_map = {k: v for k, v in associations.items()}
self.reagents = self.reconcile_map(reagent_list=reagent_list, reagent_map=reagent_map)
def reconcile_map(self, reagent_list: List[dict], reagent_map: dict) -> Generator[dict, None, None]:
@@ -264,7 +272,7 @@ class SampleWriter(object):
submission_type = SubmissionType.query(name=submission_type)
self.submission_type = submission_type
self.xl = xl
self.sample_map = submission_type.construct_sample_map()['lookup_table']
self.sample_map = submission_type.sample_map['lookup_table']
# NOTE: exclude any samples without a submission rank.
samples = [item for item in self.reconcile_map(sample_list) if item['submission_rank'] > 0]
self.samples = sorted(samples, key=itemgetter('submission_rank'))
@@ -282,7 +290,7 @@ class SampleWriter(object):
"""
multiples = ['row', 'column', 'assoc_id', 'submission_rank']
for sample in sample_list:
sample = self.submission_type.get_submission_class().custom_sample_writer(sample)
sample = self.submission_type.submission_class.custom_sample_writer(sample)
for assoc in zip(sample['row'], sample['column'], sample['submission_rank']):
new = dict(row=assoc[0], column=assoc[1], submission_rank=assoc[2])
for k, v in sample.items():
@@ -354,7 +362,7 @@ class EquipmentWriter(object):
equipment_map (dict): Dictionary of equipment locations
Returns:
List[dict]: List of merged dictionaries
List[dict]: List of merged dictionaries
"""
if equipment_list is None:
return

33
src/submissions/backend/validators/__init__.py
View File

@@ -19,21 +19,22 @@ class RSLNamer(object):
Object that will enforce proper formatting on RSL plate names.
"""
def __init__(self, filename: str, sub_type: str | None = None, data: dict | None = None):
def __init__(self, filename: str, submission_type: str | None = None, data: dict | None = None):
# NOTE: Preferred method is path retrieval, but might also need validation for just string.
filename = Path(filename) if Path(filename).exists() else filename
self.submission_type = sub_type
self.submission_type = submission_type
if not self.submission_type:
self.submission_type = self.retrieve_submission_type(filename=filename)
logger.info(f"got submission type: {self.submission_type}")
if self.submission_type:
self.sub_object = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=self.submission_type)
self.parsed_name = self.retrieve_rsl_number(filename=filename, regex=self.sub_object.get_regex(submission_type=sub_type))
self.parsed_name = self.retrieve_rsl_number(filename=filename, regex=self.sub_object.get_regex(submission_type=submission_type))
if not data:
data = dict(submission_type=self.submission_type)
if "submission_type" not in data.keys():
data['submission_type'] = self.submission_type
self.parsed_name = self.sub_object.enforce_name(instr=self.parsed_name, data=data)
logger.info(f"Parsed name: {self.parsed_name}")
@classmethod
def retrieve_submission_type(cls, filename: str | Path) -> str:
@@ -57,7 +58,7 @@ class RSLNamer(object):
categories = wb.properties.category.split(";")
submission_type = next(item.strip().title() for item in categories)
except (StopIteration, AttributeError):
sts = {item.name: item.get_template_file_sheets() for item in SubmissionType.query() if item.template_file}
sts = {item.name: item.template_file_sheets for item in SubmissionType.query() if item.template_file}
try:
submission_type = next(k.title() for k,v in sts.items() if wb.sheetnames==v)
except StopIteration:
@@ -69,7 +70,7 @@ class RSLNamer(object):
def st_from_str(filename:str) -> str:
if filename.startswith("tmp"):
return "Bacterial Culture"
regex = BasicSubmission.construct_regex()
regex = BasicSubmission.regex
m = regex.search(filename)
try:
submission_type = m.lastgroup
@@ -94,14 +95,15 @@ class RSLNamer(object):
raise ValueError("Submission Type came back as None.")
from frontend.widgets import ObjectSelector
dlg = ObjectSelector(title="Couldn't parse submission type.",
message="Please select submission type from list below.", obj_type=SubmissionType)
message="Please select submission type from list below.",
obj_type=SubmissionType)
if dlg.exec():
submission_type = dlg.parse_form()
submission_type = submission_type.replace("_", " ")
return submission_type
@classmethod
def retrieve_rsl_number(cls, filename: str | Path, regex: str | None = None):
def retrieve_rsl_number(cls, filename: str | Path, regex: re.Pattern | None = None):
"""
Uses regex to retrieve the plate number and submission type from an input string
@@ -110,12 +112,7 @@ class RSLNamer(object):
filename (str): string to be parsed
"""
if regex is None:
regex = BasicSubmission.construct_regex()
else:
try:
regex = re.compile(rf'{regex}', re.IGNORECASE | re.VERBOSE)
except re.error as e:
regex = BasicSubmission.construct_regex()
regex = BasicSubmission.regex
match filename:
case Path():
m = regex.search(filename.stem)
@@ -135,7 +132,7 @@ class RSLNamer(object):
@classmethod
def construct_new_plate_name(cls, data: dict) -> str:
"""
Make a brand new plate name from submission data.
Make a brand-new plate name from submission data.
Args:
data (dict): incoming submission data
@@ -179,7 +176,13 @@ class RSLNamer(object):
template = environment.from_string(template)
return template.render(**kwargs)
def calculate_repeat(self):
def calculate_repeat(self) -> str:
"""
Determines what repeat number this plate is.
Returns:
str: Repeat number.
"""
regex = re.compile(r"-\d(?P<repeat>R\d)")
m = regex.search(self.parsed_name)
if m is not None:

80
src/submissions/backend/validators/pydant.py
View File

@@ -73,7 +73,7 @@ class PydReagent(BaseModel):
if value is not None:
match value:
case int():
return datetime.fromordinal(datetime(1900, 1, 1).toordinal() + value - 2).date()
return datetime.fromordinal(datetime(1900, 1, 1).toordinal() + value - 2)
case 'NA':
return value
case str():
@@ -117,7 +117,8 @@ class PydReagent(BaseModel):
fields = list(self.model_fields.keys()) + extras
return {k: getattr(self, k) for k in fields}
def toSQL(self, submission: BasicSubmission | str = None) -> Tuple[Reagent, Report]:
@report_result
def to_sql(self, submission: BasicSubmission | str = None) -> Tuple[Reagent, Report]:
"""
Converts this instance into a backend.db.models.kit.Reagent instance
@@ -128,6 +129,7 @@ class PydReagent(BaseModel):
if self.model_extra is not None:
self.__dict__.update(self.model_extra)
reagent = Reagent.query(lot=self.lot, name=self.name)
logger.debug(f"Reagent: {reagent}")
if reagent is None:
reagent = Reagent()
for key, value in self.__dict__.items():
@@ -140,7 +142,6 @@ class PydReagent(BaseModel):
assoc.comments = self.comment
else:
assoc = None
report.add_result(Result(owner=__name__, code=0, msg="New reagent created.", status="Information"))
else:
if submission is not None and reagent not in submission.reagents:
submission.update_reagentassoc(reagent=reagent, role=self.role)
@@ -160,7 +161,7 @@ class PydSample(BaseModel, extra='allow'):
def validate_model(cls, data):
model = BasicSample.find_polymorphic_subclass(polymorphic_identity=data.sample_type)
for k, v in data.model_extra.items():
if k in model.timestamps():
if k in model.timestamps:
if isinstance(v, str):
v = datetime.strptime(v, "%Y-%m-%d")
data.__setattr__(k, v)
@@ -202,7 +203,7 @@ class PydSample(BaseModel, extra='allow'):
fields = list(self.model_fields.keys()) + list(self.model_extra.keys())
return {k: getattr(self, k) for k in fields}
def toSQL(self, submission: BasicSubmission | str = None) -> Tuple[
def to_sql(self, submission: BasicSubmission | str = None) -> Tuple[
BasicSample, List[SubmissionSampleAssociation], Result | None]:
"""
Converts this instance into a backend.db.models.submissions.Sample object
@@ -271,7 +272,7 @@ class PydTips(BaseModel):
def to_sql(self, submission: BasicSubmission) -> SubmissionTipsAssociation:
"""
Con
Convert this object to the SQL version for database storage.
Args:
submission (BasicSubmission): A submission object to associate tips represented here.
@@ -280,10 +281,10 @@ class PydTips(BaseModel):
SubmissionTipsAssociation: Association between queried tips and submission
"""
tips = Tips.query(name=self.name, limit=1)
logger.debug(f"Tips query has yielded: {tips}")
assoc = SubmissionTipsAssociation.query(tip_id=tips.id, submission_id=submission.id, role=self.role, limit=1)
if assoc is None:
assoc = SubmissionTipsAssociation(submission=submission, tips=tips, role_name=self.role)
# logger.debug(f"Tips query has yielded: {tips}")
assoc = SubmissionTipsAssociation.query_or_create(tips=tips, submission=submission, role=self.role, limit=1)
# if assoc is None:
# assoc = SubmissionTipsAssociation(submission=submission, tips=tips, role_name=self.role)
return assoc
@@ -316,7 +317,7 @@ class PydEquipment(BaseModel, extra='ignore'):
pass
return value
def toSQL(self, submission: BasicSubmission | str = None) -> Tuple[Equipment, SubmissionEquipmentAssociation]:
def to_sql(self, submission: BasicSubmission | str = None, extraction_kit: KitType | str = None) -> Tuple[Equipment, SubmissionEquipmentAssociation]:
"""
Creates Equipment and SubmssionEquipmentAssociations for this PydEquipment
@@ -328,6 +329,8 @@ class PydEquipment(BaseModel, extra='ignore'):
"""
if isinstance(submission, str):
submission = BasicSubmission.query(rsl_plate_num=submission)
if isinstance(extraction_kit, str):
extraction_kit = KitType.query(name=extraction_kit)
equipment = Equipment.query(asset_number=self.asset_number)
if equipment is None:
logger.error("No equipment found. Returning None.")
@@ -343,7 +346,12 @@ class PydEquipment(BaseModel, extra='ignore'):
if assoc is None:
assoc = SubmissionEquipmentAssociation(submission=submission, equipment=equipment)
# TODO: This seems precarious. What if there is more than one process?
process = Process.query(name=self.processes[0])
# NOTE: It looks like the way fetching the processes is done in the SQL model, this shouldn't be a problem, but I'll include a failsafe.
# NOTE: I need to find a way to filter this by the kit involved.
if len(self.processes) > 1:
process = Process.query(submission_type=submission.get_submission_type(), extraction_kit=extraction_kit, equipment_role=self.role)
else:
process = Process.query(name=self.processes[0])
if process is None:
logger.error(f"Found unknown process: {process}.")
assoc.process = process
@@ -405,10 +413,12 @@ class PydSubmission(BaseModel, extra='allow'):
@field_validator('equipment', mode='before')
@classmethod
def convert_equipment_dict(cls, value):
if isinstance(value, Generator):
return [PydEquipment(**equipment) for equipment in value]
if isinstance(value, dict):
return value['value']
if isinstance(value, Generator):
return [PydEquipment(**equipment) for equipment in value]
if not value:
return []
return value
@field_validator('comment', mode='before')
@@ -443,12 +453,11 @@ class PydSubmission(BaseModel, extra='allow'):
def strip_datetime_string(cls, value):
match value['value']:
case date():
return value
output = datetime.combine(value['value'], datetime.min.time())
case datetime():
return value.date()
pass
case int():
return dict(value=datetime.fromordinal(datetime(1900, 1, 1).toordinal() + value['value'] - 2).date(),
missing=True)
output = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + value['value'] - 2)
case str():
string = re.sub(r"(_|-)\d(R\d)?$", "", value['value'])
try:
@@ -456,12 +465,15 @@ class PydSubmission(BaseModel, extra='allow'):
except ParserError as e:
logger.error(f"Problem parsing date: {e}")
try:
output = dict(value=parse(string.replace("-", "")).date(), missing=True)
output = parse(string.replace("-", "")).date()
except Exception as e:
logger.error(f"Problem with parse fallback: {e}")
return output
return value
case _:
raise ValueError(f"Could not get datetime from {value['value']}")
value['value'] = output.replace(tzinfo=timezone)
return value
@field_validator("submitting_lab", mode="before")
@classmethod
@@ -511,7 +523,7 @@ class PydSubmission(BaseModel, extra='allow'):
if "pytest" in sys.modules and sub_type.replace(" ", "") == "BasicSubmission":
output = "RSL-BS-Test001"
else:
output = RSLNamer(filename=values.data['filepath'].__str__(), sub_type=sub_type,
output = RSLNamer(filename=values.data['filepath'].__str__(), submission_type=sub_type,
data=values.data).parsed_name
return dict(value=output, missing=True)
@@ -653,9 +665,9 @@ class PydSubmission(BaseModel, extra='allow'):
return value
if isinstance(contact, tuple):
contact = contact[0]
value = dict(value=f"Defaulted to: {contact}", missing=True)
value = dict(value=f"Defaulted to: {contact}", missing=False)
logger.debug(f"Value after query: {value}")
return
return value
else:
logger.debug(f"Value after bypass check: {value}")
return value
@@ -665,7 +677,7 @@ class PydSubmission(BaseModel, extra='allow'):
# NOTE: this could also be done with default_factory
self.submission_object = BasicSubmission.find_polymorphic_subclass(
polymorphic_identity=self.submission_type['value'])
self.namer = RSLNamer(self.rsl_plate_num['value'], sub_type=self.submission_type['value'])
self.namer = RSLNamer(self.rsl_plate_num['value'], submission_type=self.submission_type['value'])
if run_custom:
self.submission_object.custom_validation(pyd=self)
@@ -777,10 +789,10 @@ class PydSubmission(BaseModel, extra='allow'):
match key:
case "reagents":
for reagent in self.reagents:
reagent, _ = reagent.toSQL(submission=instance)
reagent = reagent.to_sql(submission=instance)
case "samples":
for sample in self.samples:
sample, associations, _ = sample.toSQL(submission=instance)
sample, associations, _ = sample.to_sql(submission=instance)
for assoc in associations:
if assoc is not None:
if assoc not in instance.submission_sample_associations:
@@ -791,7 +803,7 @@ class PydSubmission(BaseModel, extra='allow'):
for equip in self.equipment:
if equip is None:
continue
equip, association = equip.toSQL(submission=instance)
equip, association = equip.to_sql(submission=instance, extraction_kit=self.extraction_kit)
if association is not None:
instance.submission_equipment_associations.append(association)
case "tips":
@@ -807,7 +819,7 @@ class PydSubmission(BaseModel, extra='allow'):
instance.submission_tips_associations.append(association)
else:
logger.warning(f"Tips association {association} is already present in {instance}")
case item if item in instance.timestamps():
case item if item in instance.timestamps:
logger.warning(f"Incoming timestamp key: {item}, with value: {value}")
if isinstance(value, date):
value = datetime.combine(value, datetime.now().time())
@@ -818,7 +830,7 @@ class PydSubmission(BaseModel, extra='allow'):
else:
value = value
instance.set_attribute(key=key, value=value)
case item if item in instance.jsons():
case item if item in instance.jsons:
try:
ii = value.items()
except AttributeError:
@@ -989,7 +1001,7 @@ class PydContact(BaseModel):
logger.debug(f"Output phone: {value}")
return value
def toSQL(self) -> Tuple[Contact, Report]:
def to_sql(self) -> Tuple[Contact, Report]:
"""
Converts this instance into a backend.db.models.organization. Contact instance.
Does not query for existing contacts.
@@ -1024,7 +1036,7 @@ class PydOrganization(BaseModel):
cost_centre: str
contacts: List[PydContact] | None
def toSQL(self) -> Organization:
def to_sql(self) -> Organization:
"""
Converts this instance into a backend.db.models.organization.Organization instance.
@@ -1055,7 +1067,7 @@ class PydReagentRole(BaseModel):
return timedelta(days=value)
return value
def toSQL(self, kit: KitType) -> ReagentRole:
def to_sql(self, kit: KitType) -> ReagentRole:
"""
Converts this instance into a backend.db.models.ReagentType instance
@@ -1082,7 +1094,7 @@ class PydKit(BaseModel):
name: str
reagent_roles: List[PydReagentRole] = []
def toSQL(self) -> Tuple[KitType, Report]:
def to_sql(self) -> Tuple[KitType, Report]:
"""
Converts this instance into a backend.db.models.kits.KitType instance
@@ -1093,7 +1105,7 @@ class PydKit(BaseModel):
instance = KitType.query(name=self.name)
if instance is None:
instance = KitType(name=self.name)
[item.toSQL(instance) for item in self.reagent_roles]
[item.to_sql(instance) for item in self.reagent_roles]
return instance, report