landon/Submissions-App
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Post code clean-up, before attempt to upgrade controls to FigureWidgets

Browse Source
This commit is contained in:
lwark
2024-10-21 08:17:43 -05:00
parent 495d1a5a7f
commit 1f83b61c81
29 changed files with 441 additions and 1089 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/submissions/backend/db/models/__init__.py
View File

@@ -96,7 +96,7 @@ class BaseClass(Base):
output = {}
for k, v in dicto.items():
if len(args) > 0 and k not in args:
# logger.debug(f"Don't want {k}")
# logger.debug(f"{k} not selected as being of interest.")
continue
else:
output[k] = v

104
src/submissions/backend/db/models/controls.py
View File

@@ -3,14 +3,12 @@ All control related models.
"""
from __future__ import annotations
from pprint import pformat
from PyQt6.QtWidgets import QWidget, QCheckBox, QLabel
from pandas import DataFrame
from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey, case, FLOAT
from sqlalchemy.orm import relationship, Query, validates
import logging, re
from operator import itemgetter
from . import BaseClass
from tools import setup_lookup, report_result, Result, Report, Settings, get_unique_values_in_df_column, super_splitter
from datetime import date, datetime, timedelta
@@ -73,7 +71,6 @@ class ControlType(BaseClass):
if not self.instances:
return
jsoner = getattr(self.instances[0], mode)
# logger.debug(f"JSON retrieved: {jsoner.keys()}")
try:
# NOTE: Pick genera (all should have same subtypes)
genera = list(jsoner.keys())[0]
@@ -81,10 +78,15 @@ class ControlType(BaseClass):
return []
# NOTE: remove items that don't have relevant data
subtypes = [item for item in jsoner[genera] if "_hashes" not in item and "_ratio" not in item]
# logger.debug(f"subtypes out: {pformat(subtypes)}")
return subtypes
def get_instance_class(self):
def get_instance_class(self) -> Control:
"""
Retrieves the Control class associated with this controltype
Returns:
Control: Associated Control class
"""
return Control.find_polymorphic_subclass(polymorphic_identity=self.name)
@classmethod
@@ -93,7 +95,7 @@ class ControlType(BaseClass):
Gets list of Control types if they have targets
Returns:
List[ControlType]: Control types that have targets
Generator[str, None, None]: Control types that have targets
"""
ct = cls.query(name=control_type).targets
return (item for item in ct.keys() if ct[item])
@@ -106,7 +108,6 @@ class ControlType(BaseClass):
Returns:
Pattern: Constructed pattern
"""
# strings = list(set([item.name.split("-")[0] for item in cls.get_positive_control_types()]))
strings = list(set([super_splitter(item, "-", 0) for item in cls.get_positive_control_types(control_type)]))
return re.compile(rf"(^{'|^'.join(strings)})-.*", flags=re.IGNORECASE)
@@ -144,7 +145,7 @@ class Control(BaseClass):
@classmethod
def find_polymorphic_subclass(cls, polymorphic_identity: str | ControlType | None = None,
attrs: dict | None = None):
attrs: dict | None = None) -> Control:
"""
Find subclass based on polymorphic identity or relevant attributes.
@@ -153,7 +154,7 @@ class Control(BaseClass):
attrs (str | SubmissionType | None, optional): Attributes of the relevant class. Defaults to None.
Returns:
_type_: Subclass of interest.
Control: Subclass of interest.
"""
if isinstance(polymorphic_identity, dict):
# logger.debug(f"Controlling for dict value")
@@ -184,7 +185,8 @@ class Control(BaseClass):
@classmethod
def make_parent_buttons(cls, parent: QWidget) -> None:
"""
Super that will make buttons in a CustomFigure. Made to be overrided.
Args:
parent (QWidget): chart holding widget to add buttons to.
@@ -196,13 +198,11 @@ class Control(BaseClass):
@classmethod
def make_chart(cls, parent, chart_settings: dict, ctx):
"""
Dummy operation to be overridden by child classes.
Args:
chart_settings (dict): settings passed down from chart widget
ctx (Settings): settings passed down from gui
Returns:
"""
return None
@@ -220,7 +220,13 @@ class PCRControl(Control):
polymorphic_load="inline",
inherit_condition=(id == Control.id))
def to_sub_dict(self):
def to_sub_dict(self) -> dict:
"""
Creates dictionary of fields for this object
Returns:
dict: Output dict of name, ct, subtype, target, reagent_lot and submitted_date
"""
return dict(name=self.name, ct=self.ct, subtype=self.subtype, target=self.target, reagent_lot=self.reagent_lot,
submitted_date=self.submitted_date.date())
@@ -237,15 +243,16 @@ class PCRControl(Control):
Lookup control objects in the database based on a number of parameters.
Args:
sub_type (models.ControlType | str | None, optional): Control archetype. Defaults to None.
sub_type (str | None, optional): Control archetype. Defaults to None.
start_date (date | str | int | None, optional): Beginning date to search by. Defaults to 2023-01-01 if end_date not None.
end_date (date | str | int | None, optional): End date to search by. Defaults to today if start_date not None.
control_name (str | None, optional): Name of control. Defaults to None.
limit (int, optional): Maximum number of results to return (0 = all). Defaults to 0.
Returns:
models.Control|List[models.Control]: Control object of interest.
PCRControl|List[PCRControl]: Control object of interest.
"""
from backend.db import SubmissionType
query: Query = cls.__database_session__.query(cls)
# NOTE: by date range
if start_date is not None and end_date is None:
@@ -282,7 +289,12 @@ class PCRControl(Control):
match sub_type:
case str():
from backend import BasicSubmission, SubmissionType
# logger.debug(f"Lookup controls by SubmissionType str: {sub_type}")
query = query.join(BasicSubmission).join(SubmissionType).filter(SubmissionType.name == sub_type)
case SubmissionType():
from backend import BasicSubmission
# logger.debug(f"Lookup controls by SubmissionType: {sub_type}")
query = query.join(BasicSubmission).filter(BasicSubmission.submission_type_name==sub_type.name)
case _:
pass
match control_name:
@@ -295,7 +307,18 @@ class PCRControl(Control):
return cls.execute_query(query=query, limit=limit)
@classmethod
def make_chart(cls, parent, chart_settings: dict, ctx):
def make_chart(cls, parent, chart_settings: dict, ctx: Settings) -> Tuple[Report, "PCRFigure"]:
"""
Creates a PCRFigure. Overrides parent
Args:
parent (__type__): Widget to contain the chart.
chart_settings (dict): settings passed down from chart widget
ctx (Settings): settings passed down from gui. Not used here.
Returns:
Tuple[Report, "PCRFigure"]: Report of status and resulting figure.
"""
from frontend.visualizations.pcr_charts import PCRFigure
parent.mode_typer.clear()
parent.mode_typer.setEnabled(False)
@@ -308,7 +331,7 @@ class PCRControl(Control):
df = df[df.ct > 0.0]
except AttributeError:
df = df
fig = PCRFigure(df=df, modes=None)
fig = PCRFigure(df=df, modes=[])
return report, fig
@@ -324,16 +347,26 @@ class IridaControl(Control):
sample = relationship("BacterialCultureSample", back_populates="control") #: This control's submission sample
sample_id = Column(INTEGER,
ForeignKey("_basicsample.id", ondelete="SET NULL", name="cont_BCS_id")) #: sample id key
# submission_id = Column(INTEGER, ForeignKey("_basicsubmission.id")) #: parent submission id
# submission = relationship("BacterialCulture", back_populates="controls",
# foreign_keys=[submission_id]) #: parent submission
__mapper_args__ = dict(polymorphic_identity="Irida Control",
polymorphic_load="inline",
inherit_condition=(id == Control.id))
@validates("sub_type")
def enforce_subtype_literals(self, key: str, value: str):
def enforce_subtype_literals(self, key: str, value: str) -> str:
"""
Validates sub_type field with acceptable values
Args:
key (str): Field name
value (str): Field Value
Raises:
KeyError: Raised if value is not in the acceptable list.
Returns:
str: Validated string.
"""
acceptables = ['ATCC49226', 'ATCC49619', 'EN-NOS', "EN-SSTI", "MCS-NOS", "MCS-SSTI", "SN-NOS", "SN-SSTI"]
if value.upper() not in acceptables:
raise KeyError(f"Sub-type must be in {acceptables}")
@@ -346,7 +379,6 @@ class IridaControl(Control):
Returns:
dict: output dictionary containing: Name, Type, Targets, Top Kraken results
"""
# logger.debug("loading json string into dict")
try:
kraken = self.kraken
except TypeError:
@@ -405,8 +437,6 @@ class IridaControl(Control):
k.strip("*") not in self.controltype.targets[control_sub_type])
on_tar['Off-target'] = {f"{mode}_ratio": off_tar}
data = on_tar
# logger.debug(pformat(data))
# logger.debug(f"Length of data: {len(data)}")
# logger.debug("dict keys are genera of bacteria, e.g. 'Streptococcus'")
for genus in data:
_dict = dict(
@@ -416,7 +446,6 @@ class IridaControl(Control):
target='Target' if genus.strip("*") in self.controltype.targets[control_sub_type] else "Off-target"
)
# logger.debug("get Target or Off-target of genus")
# logger.debug("set 'contains_hashes', etc for genus")
for key in data[genus]:
_dict[key] = data[genus][key]
yield _dict
@@ -462,12 +491,7 @@ class IridaControl(Control):
query: Query = cls.__database_session__.query(cls)
# NOTE: by control type
match sub_type:
# case ControlType():
# # logger.debug(f"Looking up control by control type: {sub_type}")
# query = query.filter(cls.controltype == sub_type)
case str():
# logger.debug(f"Looking up control by control type: {sub_type}")
# query = query.join(ControlType).filter(ControlType.name == sub_type)
query = query.filter(cls.sub_type == sub_type)
case _:
pass
@@ -519,8 +543,6 @@ class IridaControl(Control):
Args:
parent (QWidget): chart holding widget to add buttons to.
Returns:
"""
super().make_parent_buttons(parent=parent)
rows = parent.layout.rowCount()
@@ -536,6 +558,17 @@ class IridaControl(Control):
@classmethod
@report_result
def make_chart(cls, chart_settings: dict, parent, ctx) -> Tuple[Report, "IridaFigure" | None]:
"""
Creates a IridaFigure. Overrides parent
Args:
parent (__type__): Widget to contain the chart.
chart_settings (dict): settings passed down from chart widget
ctx (Settings): settings passed down from gui.
Returns:
Tuple[Report, "IridaFigure"]: Report of status and resulting figure.
"""
from frontend.visualizations import IridaFigure
try:
checker = parent.findChild(QCheckBox, name="irida_check")
@@ -574,8 +607,6 @@ class IridaControl(Control):
# NOTE: send dataframe to chart maker
df, modes = cls.prep_df(ctx=ctx, df=df)
# logger.debug(f"prepped df: \n {df}")
# assert modes
# logger.debug(f"modes: {modes}")
fig = IridaFigure(df=df, ytitle=title, modes=modes, parent=parent,
months=chart_settings['months'])
return report, fig
@@ -604,7 +635,6 @@ class IridaControl(Control):
else:
safe.append(column)
if "percent" in column:
# count_col = [item for item in df.columns if "count" in item][0]
try:
count_col = next(item for item in df.columns if "count" in item)
except StopIteration:

216
src/submissions/backend/db/models/kits.py
View File

@@ -2,7 +2,6 @@
All kit and reagent related models
"""
from __future__ import annotations
import datetime
import json
from pprint import pformat
@@ -152,7 +151,7 @@ class KitType(BaseClass):
submission_type (str | Submissiontype | None, optional): Submission type to narrow results. Defaults to None.
Returns:
List[ReagentRole]: List of reagents linked to this kit.
Generator[ReagentRole, None, None]: List of reagents linked to this kit.
"""
match submission_type:
case SubmissionType():
@@ -173,7 +172,7 @@ class KitType(BaseClass):
return (item.reagent_role for item in relevant_associations)
# TODO: Move to BasicSubmission?
def construct_xl_map_for_use(self, submission_type: str | SubmissionType) -> Generator[(str, str)]:
def construct_xl_map_for_use(self, submission_type: str | SubmissionType) -> Generator[(str, str), None, None]:
"""
Creates map of locations in Excel workbook for a SubmissionType
@@ -181,9 +180,8 @@ class KitType(BaseClass):
submission_type (str | SubmissionType): Submissiontype.name
Returns:
dict: Dictionary containing information locations.
Generator[(str, str), None, None]: Tuple containing information locations.
"""
# info_map = {}
# NOTE: Account for submission_type variable type.
match submission_type:
case str():
@@ -221,7 +219,7 @@ class KitType(BaseClass):
limit (int, optional): Maximum number of results to return (0 = all). Defaults to 0.
Returns:
models.KitType|List[models.KitType]: KitType(s) of interest.
KitType|List[KitType]: KitType(s) of interest.
"""
query: Query = cls.__database_session__.query(cls)
match used_for:
@@ -257,7 +255,16 @@ class KitType(BaseClass):
def save(self):
super().save()
def to_export_dict(self, submission_type: SubmissionType):
def to_export_dict(self, submission_type: SubmissionType) -> dict:
"""
Creates dictionary for exporting to yml used in new SubmissionType Construction
Args:
submission_type (SubmissionType): SubmissionType of interest.
Returns:
dict: Dictionary containing relevant info for SubmissionType construction
"""
base_dict = dict(name=self.name)
base_dict['reagent roles'] = []
base_dict['equipment roles'] = []
@@ -382,7 +389,13 @@ class ReagentRole(BaseClass):
from backend.validators.pydant import PydReagent
return PydReagent(lot=None, role=self.name, name=self.name, expiry=date.today())
def to_export_dict(self):
def to_export_dict(self) -> dict:
"""
Creates dictionary for exporting to yml used in new SubmissionType Construction
Returns:
dict: Dictionary containing relevant info for SubmissionType construction
"""
return dict(role=self.name, extension_of_life=self.eol_ext.days)
@check_authorization
@@ -664,7 +677,7 @@ class SubmissionType(BaseClass):
"SubmissionTypeTipRoleAssociation",
back_populates="submission_type",
cascade="all, delete-orphan"
)
) #: Association of tiproles
def __repr__(self) -> str:
"""
@@ -738,12 +751,12 @@ class SubmissionType(BaseClass):
"""
return self.sample_map
def construct_equipment_map(self) -> Generator[str, dict]:
def construct_equipment_map(self) -> Generator[(str, dict), None, None]:
"""
Constructs map of equipment to excel cells.
Returns:
dict: Map equipment locations in excel sheet
Generator[(str, dict), None, None]: Map equipment locations in excel sheet
"""
# logger.debug("Iterating through equipment roles")
for item in self.submissiontype_equipmentrole_associations:
@@ -752,12 +765,12 @@ class SubmissionType(BaseClass):
emap = {}
yield item.equipment_role.name, emap
def construct_tips_map(self) -> Generator[str, dict]:
def construct_tips_map(self) -> Generator[(str, dict), None, None]:
"""
Constructs map of tips to excel cells.
Returns:
dict: Tip locations in the excel sheet.
Generator[(str, dict), None, None]: Tip locations in the excel sheet.
"""
for item in self.submissiontype_tiprole_associations:
tmap = item.uses
@@ -770,7 +783,7 @@ class SubmissionType(BaseClass):
Returns PydEquipmentRole of all equipment associated with this SubmissionType
Returns:
List[PydEquipmentRole]: List of equipment roles
Generator['PydEquipmentRole', None, None]: List of equipment roles
"""
return (item.to_pydantic(submission_type=self, extraction_kit=extraction_kit) for item in self.equipment)
@@ -846,6 +859,12 @@ class SubmissionType(BaseClass):
return cls.execute_query(query=query, limit=limit)
def to_export_dict(self):
"""
Creates dictionary for exporting to yml used in new SubmissionType Construction
Returns:
dict: Dictionary containing relevant info for SubmissionType construction
"""
base_dict = dict(name=self.name)
base_dict['info'] = self.construct_info_map(mode='export')
base_dict['defaults'] = self.defaults
@@ -862,7 +881,20 @@ class SubmissionType(BaseClass):
@classmethod
@check_authorization
def import_from_json(cls, filepath: Path | str):
def import_from_json(cls, filepath: Path | str) -> SubmissionType:
"""
Creates a new SubmissionType from a yml file
Args:
filepath (Path | str): Input yml file.
Raises:
Exception: Raised if filetype is not a yml or json
Returns:
SubmissionType: Created SubmissionType
"""
full = True
yaml.add_constructor("!regex", yaml_regex_creator)
if isinstance(filepath, str):
filepath = Path(filepath)
@@ -874,70 +906,76 @@ class SubmissionType(BaseClass):
else:
raise Exception(f"Filetype {filepath.suffix} not supported.")
logger.debug(pformat(import_dict))
submission_type = cls.query(name=import_dict['name'])
if submission_type:
return submission_type
submission_type = cls()
submission_type.name = import_dict['name']
submission_type.info_map = import_dict['info']
submission_type.sample_map = import_dict['samples']
submission_type.defaults = import_dict['defaults']
for kit in import_dict['kits']:
new_kit = KitType.query(name=kit['kit_type']['name'])
if not new_kit:
new_kit = KitType(name=kit['kit_type']['name'])
for role in kit['kit_type']['reagent roles']:
new_role = ReagentRole.query(name=role['role'])
if new_role:
check = input(f"Found existing role: {new_role.name}. Use this? [Y/n]: ")
if check.lower() == "n":
new_role = None
else:
pass
if not new_role:
eol = datetime.timedelta(role['extension_of_life'])
new_role = ReagentRole(name=role['role'], eol_ext=eol)
uses = dict(expiry=role['expiry'], lot=role['lot'], name=role['name'], sheet=role['sheet'])
ktrr_assoc = KitTypeReagentRoleAssociation(kit_type=new_kit, reagent_role=new_role, uses=uses)
ktrr_assoc.submission_type = submission_type
ktrr_assoc.required = role['required']
ktst_assoc = SubmissionTypeKitTypeAssociation(
kit_type=new_kit,
submission_type=submission_type,
mutable_cost_sample=kit['mutable_cost_sample'],
mutable_cost_column=kit['mutable_cost_column'],
constant_cost=kit['constant_cost']
)
for role in kit['kit_type']['equipment roles']:
new_role = EquipmentRole.query(name=role['role'])
if new_role:
check = input(f"Found existing role: {new_role.name}. Use this? [Y/n]: ")
if check.lower() == "n":
new_role = None
else:
pass
if not new_role:
new_role = EquipmentRole(name=role['role'])
for equipment in Equipment.assign_equipment(equipment_role=new_role):
new_role.instances.append(equipment)
ster_assoc = SubmissionTypeEquipmentRoleAssociation(submission_type=submission_type,
equipment_role=new_role)
try:
uses = dict(name=role['name'], process=role['process'], sheet=role['sheet'], static=role['static'])
except KeyError:
uses = None
ster_assoc.uses = uses
for process in role['processes']:
new_process = Process.query(name=process)
if not new_process:
new_process = Process(name=process)
new_process.submission_types.append(submission_type)
new_process.kit_types.append(new_kit)
new_process.equipment_roles.append(new_role)
if 'orgs' in import_dict.keys():
logger.info("Found Organizations to be imported.")
Organization.import_from_yml(filepath=filepath)
return submission_type
try:
submission_type = cls.query(name=import_dict['name'])
except KeyError:
logger.error(f"Submission type has no name")
submission_type = None
full = False
if full:
if submission_type:
return submission_type
submission_type = cls()
submission_type.name = import_dict['name']
submission_type.info_map = import_dict['info']
submission_type.sample_map = import_dict['samples']
submission_type.defaults = import_dict['defaults']
for kit in import_dict['kits']:
new_kit = KitType.query(name=kit['kit_type']['name'])
if not new_kit:
new_kit = KitType(name=kit['kit_type']['name'])
for role in kit['kit_type']['reagent roles']:
new_role = ReagentRole.query(name=role['role'])
if new_role:
check = input(f"Found existing role: {new_role.name}. Use this? [Y/n]: ")
if check.lower() == "n":
new_role = None
else:
pass
if not new_role:
eol = datetime.timedelta(role['extension_of_life'])
new_role = ReagentRole(name=role['role'], eol_ext=eol)
uses = dict(expiry=role['expiry'], lot=role['lot'], name=role['name'], sheet=role['sheet'])
ktrr_assoc = KitTypeReagentRoleAssociation(kit_type=new_kit, reagent_role=new_role, uses=uses)
ktrr_assoc.submission_type = submission_type
ktrr_assoc.required = role['required']
ktst_assoc = SubmissionTypeKitTypeAssociation(
kit_type=new_kit,
submission_type=submission_type,
mutable_cost_sample=kit['mutable_cost_sample'],
mutable_cost_column=kit['mutable_cost_column'],
constant_cost=kit['constant_cost']
)
for role in kit['kit_type']['equipment roles']:
new_role = EquipmentRole.query(name=role['role'])
if new_role:
check = input(f"Found existing role: {new_role.name}. Use this? [Y/n]: ")
if check.lower() == "n":
new_role = None
else:
pass
if not new_role:
new_role = EquipmentRole(name=role['role'])
for equipment in Equipment.assign_equipment(equipment_role=new_role):
new_role.instances.append(equipment)
ster_assoc = SubmissionTypeEquipmentRoleAssociation(submission_type=submission_type,
equipment_role=new_role)
try:
uses = dict(name=role['name'], process=role['process'], sheet=role['sheet'], static=role['static'])
except KeyError:
uses = None
ster_assoc.uses = uses
for process in role['processes']:
new_process = Process.query(name=process)
if not new_process:
new_process = Process(name=process)
new_process.submission_types.append(submission_type)
new_process.kit_types.append(new_kit)
new_process.equipment_roles.append(new_role)
if 'orgs' in import_dict.keys():
logger.info("Found Organizations to be imported.")
Organization.import_from_yml(filepath=filepath)
return submission_type
class SubmissionTypeKitTypeAssociation(BaseClass):
@@ -1574,10 +1612,7 @@ class EquipmentRole(BaseClass):
"""
return dict(role=self.name,
processes=self.get_processes(submission_type=submission_type, extraction_kit=kit_type))
# base_dict['role'] = self.name
# base_dict['processes'] = self.get_processes(submission_type=submission_type, extraction_kit=kit_type)
# return base_dict
class SubmissionEquipmentAssociation(BaseClass):
"""
@@ -1598,7 +1633,7 @@ class SubmissionEquipmentAssociation(BaseClass):
equipment = relationship(Equipment, back_populates="equipment_submission_associations") #: associated equipment
def __repr__(self):
def __repr__(self) -> str:
return f"<SubmissionEquipmentAssociation({self.submission.rsl_plate_num} & {self.equipment.name})>"
def __init__(self, submission, equipment, role: str = "None"):
@@ -1699,9 +1734,18 @@ class SubmissionTypeEquipmentRoleAssociation(BaseClass):
def save(self):
super().save()
def to_export_dict(self, kit_type: KitType):
def to_export_dict(self, extraction_kit: KitType) -> dict:
"""
Creates dictionary for exporting to yml used in new SubmissionType Construction
Args:
kit_type (KitType): KitType of interest.
Returns:
dict: Dictionary containing relevant info for SubmissionType construction
"""
base_dict = {k: v for k, v in self.equipment_role.to_export_dict(submission_type=self.submission_type,
kit_type=kit_type).items()}
kit_type=extraction_kit).items()}
base_dict['static'] = self.static
return base_dict

7
src/submissions/backend/db/models/organizations.py
View File

@@ -2,7 +2,6 @@
All client organization related models.
'''
from __future__ import annotations
import json, yaml, logging
from pathlib import Path
from pprint import pformat
@@ -118,8 +117,6 @@ class Organization(BaseClass):
cont.__setattr__(k, v)
organ.contacts.append(cont)
organ.save()
# logger.debug(pformat(organ.__dict__))
class Contact(BaseClass):
@@ -136,10 +133,6 @@ class Contact(BaseClass):
submissions = relationship("BasicSubmission", back_populates="contact") #: submissions this contact has submitted
def __repr__(self) -> str:
"""
Returns:
str: Representation of this Contact
"""
return f"<Contact({self.name})>"
@classmethod

299
src/submissions/backend/db/models/submissions.py
View File

@@ -2,7 +2,6 @@
Models for the main submission and sample types.
"""
from __future__ import annotations
import sys
import types
from copy import deepcopy
@@ -125,10 +124,6 @@ class BasicSubmission(BaseClass):
}
def __repr__(self) -> str:
"""
Returns:
str: Representation of this BasicSubmission
"""
submission_type = self.submission_type or "Basic"
return f"<{submission_type}Submission({self.rsl_plate_num})>"
@@ -184,10 +179,8 @@ class BasicSubmission(BaseClass):
# NOTE: Fields not placed in ui form to be moved to pydantic
form_recover=recover
)
# logger.debug(dicto['singles'])
# NOTE: Singles tells the query which fields to set limit to 1
dicto['singles'] = parent_defs['singles']
# logger.debug(dicto['singles'])
# NOTE: Grab mode_sub_type specific info.
output = {}
for k, v in dicto.items():
@@ -233,7 +226,7 @@ class BasicSubmission(BaseClass):
sub_type (str | SubmissionType, Optional): Identity of the submission type to retrieve. Defaults to None.
Returns:
SubmissionType: SubmissionType with name equal to this polymorphic identity
SubmissionType: SubmissionType with name equal sub_type or this polymorphic identity if sub_type is None.
"""
# logger.debug(f"Running search for {sub_type}")
if isinstance(sub_type, dict):
@@ -274,20 +267,6 @@ class BasicSubmission(BaseClass):
"""
return cls.get_submission_type(submission_type).construct_sample_map()
@classmethod
def finalize_details(cls, input_dict: dict) -> dict:
"""
Make final adjustments to the details dictionary before display.
Args:
input_dict (dict): Incoming dictionary.
Returns:
dict: Final details dictionary.
"""
del input_dict['id']
return input_dict
def generate_associations(self, name: str, extra: str | None = None):
try:
field = self.__getattribute__(name)
@@ -362,25 +341,10 @@ class BasicSubmission(BaseClass):
dict(role=k, name="Not Applicable", lot="NA", expiry=expiry,
missing=True))
# logger.debug(f"Running samples.")
# samples = self.adjust_to_dict_samples(backup=backup)
samples = self.generate_associations(name="submission_sample_associations")
# logger.debug("Running equipment")
equipment = self.generate_associations(name="submission_equipment_associations")
# try:
# equipment = [item.to_sub_dict() for item in self.submission_equipment_associations]
# if not equipment:
# equipment = None
# except Exception as e:
# logger.error(f"Error setting equipment: {e}")
# equipment = None
tips = self.generate_associations(name="submission_tips_associations")
# try:
# tips = [item.to_sub_dict() for item in self.submission_tips_associations]
# if not tips:
# tips = None
# except Exception as e:
# logger.error(f"Error setting tips: {e}")
# tips = None
cost_centre = self.cost_centre
custom = self.custom
else:
@@ -428,7 +392,6 @@ class BasicSubmission(BaseClass):
Returns:
int: Number of unique columns.
"""
# logger.debug(f"Here's the samples: {self.samples}")
columns = set([assoc.column for assoc in self.submission_sample_associations])
# logger.debug(f"Here are the columns for {self.rsl_plate_num}: {columns}")
return len(columns)
@@ -513,6 +476,7 @@ class BasicSubmission(BaseClass):
Convert all submissions to dataframe
Args:
page_size (int, optional): Number of items to include in query result. Defaults to 250.
page (int, optional): Limits the number of submissions to a page size. Defaults to 1.
chronologic (bool, optional): Sort submissions in chronologic order. Defaults to True.
submission_type (str | None, optional): Filter by SubmissionType. Defaults to None.
@@ -537,11 +501,6 @@ class BasicSubmission(BaseClass):
'signed_by', 'artic_date', 'gel_barcode', 'gel_date', 'ngs_date', 'contact_phone', 'contact',
'tips', 'gel_image_path', 'custom']
df = df.loc[:, ~df.columns.isin(exclude)]
# for item in excluded:
# try:
# df = df.drop(item, axis=1)
# except:
# logger.warning(f"Couldn't drop '{item}' column from submissionsheet df.")
if chronologic:
try:
df.sort_values(by="id", axis=0, inplace=True, ascending=False)
@@ -611,6 +570,7 @@ class BasicSubmission(BaseClass):
if value is not None:
existing.append(value)
self.__setattr__(key, existing)
# NOTE: Make sure this gets updated by telling SQLAlchemy it's been modified.
flag_modified(self, key)
return
case _:
@@ -645,7 +605,7 @@ class BasicSubmission(BaseClass):
for k, v in input_dict.items():
try:
setattr(assoc, k, v)
# NOTE: for some reason I don't think assoc.__setattr__(k, v) doesn't work here.
# NOTE: for some reason I don't think assoc.__setattr__(k, v) works here.
except AttributeError:
logger.error(f"Can't set {k} to {v}")
result = assoc.save()
@@ -703,7 +663,16 @@ class BasicSubmission(BaseClass):
return super().save()
@classmethod
def get_regex(cls, submission_type: SubmissionType | str | None = None):
def get_regex(cls, submission_type: SubmissionType | str | None = None) -> str:
"""
Gets the regex string for identifying a certain class of submission.
Args:
submission_type (SubmissionType | str | None, optional): submission type of interest. Defaults to None.
Returns:
str: _description_
"""
# logger.debug(f"Attempting to get regex for {cls.__mapper_args__['polymorphic_identity']}")
logger.debug(f"Attempting to get regex for {submission_type}")
try:
@@ -755,11 +724,8 @@ class BasicSubmission(BaseClass):
f"Could not get polymorph {polymorphic_identity} of {cls} due to {e}, falling back to BasicSubmission")
case _:
pass
# if attrs is None or len(attrs) == 0:
# logger.info(f"Recruiting: {cls}")
# return model
if attrs and any([not hasattr(cls, attr) for attr in attrs.keys()]):
# looks for first model that has all included kwargs
# NOTE: looks for first model that has all included kwargs
try:
model = next(subclass for subclass in cls.__subclasses__() if
all([hasattr(subclass, attr) for attr in attrs.keys()]))
@@ -797,7 +763,6 @@ class BasicSubmission(BaseClass):
input_dict['custom'][k] = ws.cell(row=v['read']['row'], column=v['read']['column']).value
case "range":
ws = xl[v['sheet']]
# input_dict['custom'][k] = []
if v['start_row'] != v['end_row']:
v['end_row'] = v['end_row'] + 1
rows = range(v['start_row'], v['end_row'])
@@ -806,10 +771,6 @@ class BasicSubmission(BaseClass):
columns = range(v['start_column'], v['end_column'])
input_dict['custom'][k] = [dict(value=ws.cell(row=row, column=column).value, row=row, column=column)
for row in rows for column in columns]
# for ii in range(v['start_row'], v['end_row']):
# for jj in range(v['start_column'], v['end_column'] + 1):
# input_dict['custom'][k].append(
# dict(value=ws.cell(row=ii, column=jj).value, row=ii, column=jj))
return input_dict
@classmethod
@@ -952,7 +913,7 @@ class BasicSubmission(BaseClass):
rsl_plate_number (str): rsl plate num of interest
Returns:
list: _description_
Generator[dict, None, None]: Updated samples
"""
# logger.debug(f"Hello from {cls.__mapper_args__['polymorphic_identity']} PCR parser!")
pcr_sample_map = cls.get_submission_type().sample_map['pcr_samples']
@@ -968,7 +929,17 @@ class BasicSubmission(BaseClass):
yield sample
@classmethod
def parse_pcr_controls(cls, xl: Workbook, rsl_plate_num: str) -> list:
def parse_pcr_controls(cls, xl: Workbook, rsl_plate_num: str) -> Generator[dict, None, None]:
"""
Custom parsing of pcr controls from Design & Analysis Software export.
Args:
xl (Workbook): D&A export file
rsl_plate_num (str): Plate number of the submission to be joined.
Yields:
Generator[dict, None, None]: Dictionaries of row values.
"""
location_map = cls.get_submission_type().sample_map['pcr_controls']
submission = cls.query(rsl_plate_num=rsl_plate_num)
name_column = 1
@@ -981,12 +952,16 @@ class BasicSubmission(BaseClass):
logger.debug(f"Pulling from row {iii}, column {item['ct_column']}")
subtype, target = item['name'].split("-")
ct = worksheet.cell(row=iii, column=item['ct_column']).value
# NOTE: Kind of a stop gap solution to find control reagents.
if subtype == "PC":
ctrl = next((assoc.reagent for assoc in submission.submission_reagent_associations
if any(["positive control" in item.name.lower() for item in assoc.reagent.role])), None)
if
any(["positive control" in item.name.lower() for item in assoc.reagent.role])),
None)
elif subtype == "NC":
ctrl = next((assoc.reagent for assoc in submission.submission_reagent_associations
if any(["molecular grade water" in item.name.lower() for item in assoc.reagent.role])), None)
if any(["molecular grade water" in item.name.lower() for item in
assoc.reagent.role])), None)
try:
ct = float(ct)
except ValueError:
@@ -1124,8 +1099,6 @@ class BasicSubmission(BaseClass):
case _:
# logger.debug(f"Lookup BasicSubmission by parsed str end_date {end_date}")
end_date = parse(end_date).strftime("%Y-%m-%d")
# logger.debug(f"Looking up BasicSubmissions from start date: {start_date} and end date: {end_date}")
# logger.debug(f"Start date {start_date} == End date {end_date}: {start_date == end_date}")
# logger.debug(f"Compensating for same date by using time")
if start_date == end_date:
start_date = datetime.strptime(start_date, "%Y-%m-%d").strftime("%Y-%m-%d %H:%M:%S.%f")
@@ -1336,7 +1309,7 @@ class BasicSubmission(BaseClass):
def backup(self, obj=None, fname: Path | None = None, full_backup: bool = False):
"""
Exports xlsx and yml info files for this instance.
Exports xlsx info files for this instance.
Args:
obj (_type_, optional): _description_. Defaults to None.
@@ -1352,13 +1325,6 @@ class BasicSubmission(BaseClass):
if fname.name == "":
# logger.debug(f"export cancelled.")
return
# if full_backup:
# backup = self.to_dict(full_data=True)
# try:
# with open(self.__backup_path__.joinpath(fname.with_suffix(".yml")), "w") as f:
# yaml.dump(backup, f)
# except KeyError as e:
# logger.error(f"Problem saving yml backup file: {e}")
writer = pyd.to_writer()
writer.xl.save(filename=fname.with_suffix(".xlsx"))
@@ -1436,6 +1402,17 @@ class BacterialCulture(BasicSubmission):
@classmethod
def custom_info_parser(cls, input_dict: dict, xl: Workbook | None = None, custom_fields: dict = {}) -> dict:
"""
Performs class specific info parsing before info parsing is finalized.
Args:
input_dict (dict): Generic input info
xl (Workbook | None, optional): Original xl workbook. Defaults to None.
custom_fields (dict, optional): Map of custom fields to be parsed. Defaults to {}.
Returns:
dict: Updated info dictionary.
"""
input_dict = super().custom_info_parser(input_dict=input_dict, xl=xl, custom_fields=custom_fields)
# logger.debug(f"\n\nInfo dictionary:\n\n{pformat(input_dict)}\n\n")
return input_dict
@@ -1476,12 +1453,30 @@ class Wastewater(BasicSubmission):
output["pcr_technician"] = self.technician
else:
output['pcr_technician'] = self.pcr_technician
############### Updated from finalize_details - testing 2024-1017 ################
if full_data:
output['samples'] = [sample for sample in output['samples']]
dummy_samples = []
for item in output['samples']:
# logger.debug(f"Sample dict: {item}")
thing = deepcopy(item)
try:
thing['row'] = thing['source_row']
thing['column'] = thing['source_column']
except KeyError:
logger.error(f"No row or column for sample: {item['submitter_id']}")
continue
thing['tooltip'] = f"Sample Name: {thing['name']}\nWell: {thing['sample_location']}"
dummy_samples.append(thing)
output['origin_plate'] = self.__class__.make_plate_map(sample_list=dummy_samples, plate_rows=4,
plate_columns=6)
###############################
return output
@classmethod
def custom_info_parser(cls, input_dict: dict, xl: Workbook | None = None, custom_fields: dict = {}) -> dict:
"""
Update submission dictionary with type specific information. Extends parent
Update submission dictionary with class specific information. Extends parent
Args:
input_dict (dict): Input sample dictionary
@@ -1489,7 +1484,7 @@ class Wastewater(BasicSubmission):
custom_fields: Dictionary of locations, ranges, etc to be used by this function
Returns:
dict: Updated sample dictionary
dict: Updated info dictionary
"""
input_dict = super().custom_info_parser(input_dict)
# logger.debug(f"Input dict: {pformat(input_dict)}")
@@ -1513,10 +1508,18 @@ class Wastewater(BasicSubmission):
@classmethod
def parse_pcr(cls, xl: Workbook, rsl_plate_num: str) -> Generator[dict, None, None]:
"""
Parse specific to wastewater samples.
Perform parsing of pcr info. Since most of our PC outputs are the same format, this should work for most.
Args:
xl (pd.DataFrame): pcr info form
rsl_plate_number (str): rsl plate num of interest
Returns:
Generator[dict, None, None]: Updated samples
"""
samples = [item for item in super().parse_pcr(xl=xl, rsl_plate_num=rsl_plate_num)]
# logger.debug(f'Samples from parent pcr parser: {pformat(samples)}')
# NOTE: Due to having to run through samples in for loop we need to convert to list.
output = []
for sample in samples:
# NOTE: remove '-{target}' from controls
@@ -1542,20 +1545,23 @@ class Wastewater(BasicSubmission):
del sample['assessment']
except KeyError:
pass
# yield sample
output.append(sample)
# NOTE: And then convert back to list ot keep fidelity with parent method.
for sample in output:
yield sample
# @classmethod
# def parse_pcr_controls(cls, xl: Workbook, location_map: list) -> list:
@classmethod
def enforce_name(cls, instr: str, data: dict | None = {}) -> str:
"""
Extends parent
"""
Custom naming method for this class. Extends parent.
Args:
instr (str): Initial name.
data (dict | None, optional): Additional parameters for name. Defaults to None.
Returns:
str: Updated name.
"""
try:
# NOTE: Deal with PCR file.
instr = re.sub(r"PCR(-|_)", "", instr)
@@ -1567,27 +1573,18 @@ class Wastewater(BasicSubmission):
@classmethod
def adjust_autofill_samples(cls, samples: List[Any]) -> List[Any]:
"""
Extends parent
Makes adjustments to samples before writing to excel. Extends parent.
Args:
samples (List[Any]): List of Samples
Returns:
List[Any]: Updated list of samples
"""
samples = super().adjust_autofill_samples(samples)
samples = [item for item in samples if not item.submitter_id.startswith("EN")]
return samples
# @classmethod
# def custom_sample_autofill_row(cls, sample, worksheet: Worksheet) -> int:
# """
# Extends parent
# """
# # logger.debug(f"Checking {sample.well}")
# # logger.debug(f"here's the worksheet: {worksheet}")
# row = super().custom_sample_autofill_row(sample, worksheet)
# df = pd.DataFrame(list(worksheet.values))
# # logger.debug(f"Here's the dataframe: {df}")
# idx = df[df[1] == sample.sample_location]
# # logger.debug(f"Here is the row: {idx}")
# row = idx.index.to_list()[0]
# return row + 1
@classmethod
def get_details_template(cls, base_dict: dict) -> Tuple[dict, Template]:
"""
@@ -1603,35 +1600,6 @@ class Wastewater(BasicSubmission):
base_dict['excluded'] += ['origin_plate']
return base_dict, template
@classmethod
def finalize_details(cls, input_dict: dict) -> dict:
"""
Makes changes to information before display
Args:
input_dict (dict): Input information
Returns:
dict: Updated information
"""
input_dict = super().finalize_details(input_dict)
# NOTE: Currently this is preserving the generator items, can we come up with a better way?
input_dict['samples'] = [sample for sample in input_dict['samples']]
dummy_samples = []
for item in input_dict['samples']:
# logger.debug(f"Sample dict: {item}")
thing = deepcopy(item)
try:
thing['row'] = thing['source_row']
thing['column'] = thing['source_column']
except KeyError:
logger.error(f"No row or column for sample: {item['submitter_id']}")
continue
thing['tooltip'] = f"Sample Name: {thing['name']}\nWell: {thing['sample_location']}"
dummy_samples.append(thing)
input_dict['origin_plate'] = cls.make_plate_map(sample_list=dummy_samples, plate_rows=4, plate_columns=6)
return input_dict
def custom_context_events(self) -> dict:
"""
Sets context events for main widget
@@ -1646,7 +1614,7 @@ class Wastewater(BasicSubmission):
@report_result
def link_pcr(self, obj):
"""
Adds PCR info to this submission
PYQT6 function to add PCR info to this submission
Args:
obj (_type_): Parent widget
@@ -1733,7 +1701,7 @@ class WastewaterArtic(BasicSubmission):
@classmethod
def custom_info_parser(cls, input_dict: dict, xl: Workbook | None = None, custom_fields: dict = {}) -> dict:
"""
Update submission dictionary with type specific information
Update submission dictionary with class specific information
Args:
input_dict (dict): Input sample dictionary
@@ -1763,10 +1731,8 @@ class WastewaterArtic(BasicSubmission):
return None
input_dict = super().custom_info_parser(input_dict)
input_dict['submission_type'] = dict(value="Wastewater Artic", missing=False)
logger.debug(f"Custom fields: {custom_fields}")
# logger.debug(f"Custom fields: {custom_fields}")
egel_section = custom_fields['egel_controls']
ws = xl[egel_section['sheet']]
# NOTE: Here we should be scraping the control results.
@@ -1788,7 +1754,7 @@ class WastewaterArtic(BasicSubmission):
ii in
range(source_plates_section['start_row'], source_plates_section['end_row'] + 1)]
for datum in data:
logger.debug(f"Datum: {datum}")
# logger.debug(f"Datum: {datum}")
if datum['plate'] in ["None", None, ""]:
continue
else:
@@ -1843,7 +1809,14 @@ class WastewaterArtic(BasicSubmission):
@classmethod
def enforce_name(cls, instr: str, data: dict = {}) -> str:
"""
Extends parent
Custom naming method for this class. Extends parent.
Args:
instr (str): Initial name.
data (dict | None, optional): Additional parameters for name. Defaults to None.
Returns:
str: Updated name.
"""
try:
# NOTE: Deal with PCR file.
@@ -1873,7 +1846,7 @@ class WastewaterArtic(BasicSubmission):
dict: Updated sample dictionary
"""
input_dict = super().parse_samples(input_dict)
logger.debug(f"WWA input dict: {pformat(input_dict)}")
# logger.debug(f"WWA input dict: {pformat(input_dict)}")
input_dict['sample_type'] = "Wastewater Sample"
# NOTE: Stop gap solution because WW is sloppy with their naming schemes
try:
@@ -1952,14 +1925,14 @@ class WastewaterArtic(BasicSubmission):
@classmethod
def pbs_adapter(cls, input_str):
"""
Stopgap solution because WW names their controls different
Stopgap solution because WW names their controls different
Args:
input_str (str): input name
Args:
input_str (str): input name
Returns:
str: output name
"""
Returns:
str: output name
"""
# logger.debug(f"input string raw: {input_str}")
# NOTE: Remove letters.
processed = input_str.replace("RSL", "")
@@ -2155,7 +2128,7 @@ class WastewaterArtic(BasicSubmission):
def gel_box(self, obj):
"""
Creates widget to perform gel viewing operations
Creates PYQT6 widget to perform gel viewing operations
Args:
obj (_type_): parent widget
@@ -2221,7 +2194,7 @@ class BasicSample(BaseClass):
submissions = association_proxy("sample_submission_associations", "submission") #: proxy of associated submissions
@validates('submitter_id')
def create_id(self, key: str, value: str):
def create_id(self, key: str, value: str) -> str:
"""
Creates a random string as a submitter id.
@@ -2330,7 +2303,7 @@ class BasicSample(BaseClass):
@classmethod
def parse_sample(cls, input_dict: dict) -> dict:
f"""
"""
Custom sample parser
Args:
@@ -2413,7 +2386,7 @@ class BasicSample(BaseClass):
ValueError: Raised if unallowed key is given.
Returns:
_type_: _description_
BasicSample: Instance of BasicSample
"""
disallowed = ["id"]
if kwargs == {}:
@@ -2434,7 +2407,7 @@ class BasicSample(BaseClass):
**kwargs
) -> List[BasicSample]:
"""
Allows for fuzzy search of samples. (Experimental)
Allows for fuzzy search of samples.
Args:
sample_type (str | BasicSample | None, optional): Type of sample. Defaults to None.
@@ -2764,9 +2737,13 @@ class SubmissionSampleAssociation(BaseClass):
Returns:
int: incremented id
"""
if cls.__name__ == "SubmissionSampleAssociation":
model = cls
else:
model = next((base for base in cls.__bases__ if base.__name__ == "SubmissionSampleAssociation"),
SubmissionSampleAssociation)
try:
return max([item.id for item in cls.query()]) + 1
return max([item.id for item in model.query()]) + 1
except ValueError as e:
logger.error(f"Problem incrementing id: {e}")
return 1
@@ -2980,26 +2957,6 @@ class WastewaterAssociation(SubmissionSampleAssociation):
logger.error(f"Couldn't set tooltip for {self.sample.rsl_number}. Looks like there isn't PCR data.")
return sample
@classmethod
def autoincrement_id_local(cls) -> int:
"""
Increments the association id automatically. Overrides parent
Returns:
int: incremented id
"""
try:
parent = next((base for base in cls.__bases__ if base.__name__ == "SubmissionSampleAssociation"),
SubmissionSampleAssociation)
return max([item.id for item in parent.query()]) + 1
except StopIteration as e:
logger.error(f"Problem incrementing id: {e}")
return 1
@classmethod
def autoincrement_id(cls) -> int:
return super().autoincrement_id()
class WastewaterArticAssociation(SubmissionSampleAssociation):
"""
@@ -3030,19 +2987,3 @@ class WastewaterArticAssociation(SubmissionSampleAssociation):
sample['source_plate_number'] = self.source_plate_number
sample['source_well'] = self.source_well
return sample
@classmethod
def autoincrement_id(cls) -> int:
"""
Increments the association id automatically. Overrides parent
Returns:
int: incremented id
"""
try:
parent = next((base for base in cls.__bases__ if base.__name__ == "SubmissionSampleAssociation"),
SubmissionSampleAssociation)
return max([item.id for item in parent.query()]) + 1
except StopIteration as e:
logger.error(f"Problem incrementing id: {e}")
return 1

121
src/submissions/backend/excel/parser.py
View File

@@ -9,10 +9,10 @@ from typing import List
from openpyxl import load_workbook, Workbook
from pathlib import Path
from backend.db.models import *
from backend.validators import PydSubmission, PydReagent, RSLNamer, PydSample, PydEquipment, PydTips
from backend.validators import PydSubmission, RSLNamer
import logging, re
from collections import OrderedDict
from tools import check_not_nan, convert_nans_to_nones, is_missing, check_key_or_attr
from tools import check_not_nan, is_missing, check_key_or_attr
logger = logging.getLogger(f"submissions.{__name__}")
@@ -483,17 +483,21 @@ class SampleParser(object):
lookup_samples[ii] = {}
else:
logger.warning(f"Match for {psample['id']} not direct, running search.")
for jj, lsample in enumerate(lookup_samples):
try:
check = lsample[merge_on_id] == psample['id']
except KeyError:
check = False
if check:
new = lsample | psample
lookup_samples[jj] = {}
break
else:
new = psample
# for jj, lsample in enumerate(lookup_samples):
# try:
# check = lsample[merge_on_id] == psample['id']
# except KeyError:
# check = False
# if check:
# new = lsample | psample
# lookup_samples[jj] = {}
# break
# else:
# new = psample
jj, new = next(((jj, lsample) for jj, lsample in enumerate(lookup_samples) if lsample[merge_on_id] == psample['id']), (-1, psample))
logger.debug(f"Assigning from index {jj} - {new}")
if jj >= 0:
lookup_samples[jj] = {}
if not check_key_or_attr(key='submitter_id', interest=new, check_none=True):
new['submitter_id'] = psample['id']
new = self.sub_object.parse_samples(new)
@@ -546,7 +550,7 @@ class EquipmentParser(object):
logger.error(f"Error getting asset number for {input}: {e}")
return input
def parse_equipment(self) -> List[dict]:
def parse_equipment(self) -> Generator[dict, None, None]:
"""
Scrapes equipment from xl sheet
@@ -554,7 +558,6 @@ class EquipmentParser(object):
List[dict]: list of equipment
"""
# logger.debug(f"Equipment parser going into parsing: {pformat(self.__dict__)}")
output = []
# logger.debug(f"Sheets: {sheets}")
for sheet in self.xl.sheetnames:
ws = self.xl[sheet]
@@ -579,14 +582,11 @@ class EquipmentParser(object):
eq = Equipment.query(name=asset)
process = ws.cell(row=v['process']['row'], column=v['process']['column']).value
try:
# output.append(
yield dict(name=eq.name, processes=[process], role=k, asset_number=eq.asset_number,
nickname=eq.nickname)
except AttributeError:
logger.error(f"Unable to add {eq} to list.")
# logger.debug(f"Here is the output so far: {pformat(output)}")
# return output
class TipParser(object):
"""
@@ -623,7 +623,6 @@ class TipParser(object):
List[dict]: list of equipment
"""
# logger.debug(f"Equipment parser going into parsing: {pformat(self.__dict__)}")
output = []
# logger.debug(f"Sheets: {sheets}")
for sheet in self.xl.sheetnames:
ws = self.xl[sheet]
@@ -647,22 +646,20 @@ class TipParser(object):
# logger.debug(f"asset: {asset}")
eq = Tips.query(lot=lot, name=asset, limit=1)
try:
# output.append(
yield dict(name=eq.name, role=k, lot=lot)
except AttributeError:
logger.error(f"Unable to add {eq} to PydTips list.")
# logger.debug(f"Here is the output so far: {pformat(output)}")
# return output
class PCRParser(object):
"""Object to pull data from Design and Analysis PCR export file."""
def __init__(self, filepath: Path | None = None, submission: BasicSubmission | None = None) -> None:
"""
Args:
filepath (Path | None, optional): file to parse. Defaults to None.
"""
Args:
filepath (Path | None, optional): file to parse. Defaults to None.
submission (BasicSubmission | None, optional): Submission parsed data to be added to.
"""
# logger.debug(f'Parsing {filepath.__str__()}')
if filepath is None:
logger.error('No filepath given.')
@@ -709,73 +706,3 @@ class PCRParser(object):
# logger.debug(f"PCR: {pformat(pcr)}")
return pcr
class EDSParser(object):
expand_device = {"QS7PRO": "QuantStudio tm 7 Pro System"}
expand_block = {"BLOCK_96W_01ML": "96-Well 0.1-mL Block"}
def __init__(self, filepath: str | Path | None = None):
logger.info(f"\n\nParsing {filepath.__str__()}\n\n")
match filepath:
case Path():
self.filepath = filepath
case str():
self.filepath = Path(filepath)
case _:
logger.error(f"No filepath given.")
raise ValueError("No filepath given.")
self.eds = ZipFile(self.filepath)
self.analysis_settings = json.loads(self.eds.read("primary/analysis_setting.json").decode("utf-8"))
self.analysis_results = json.loads(self.eds.read("primary/analysis_setting.json").decode("utf-8"))
self.presence_absence_results = json.loads(
self.eds.read("extensions/am.pa/presence_absence_result.json").decode("utf-8"))
self.presence_absence_settings = json.loads(
self.eds.read("extensions/am.pa/presence_absence_setting.json").decode("utf-8"))
self.run_summary = json.loads(self.eds.read("run/run_summary.json").decode("utf-8"))
self.run_method = json.loads(self.eds.read("setup/run_method.json").decode("utf-8"))
self.plate_setup = json.loads(self.eds.read("setup/plate_setup.json").decode("utf-8"))
self.eds_summary = json.loads(self.eds.read("summary.json").decode("utf-8"))
def parse_DA_date_format(self, value: int) -> datetime:
value = value / 1000
return datetime.utcfromtimestamp(value)
def get_run_time(self, start: datetime, end: datetime) -> Tuple[str, str, str]:
delta = end - start
minutes, seconds = divmod(delta.seconds, 60)
duration = f"{minutes} minutes {seconds} seconds"
start_time = start.strftime("%Y-%m-%d %I:%M:%S %p %Z")
end_time = end.strftime("%Y-%m-%d %I:%M:%S %p %Z")
return start_time, end_time, duration
def parse_summary(self):
summary = dict()
summary['file_name'] = self.filepath.absolute().__str__()
summary['comment'] = self.eds_summary['description']
summary['operator'] = self.run_summary['operator']
summary['barcode'] = self.plate_setup['plateBarcode']
try:
summary['instrument_type'] = self.__class__.expand_device[self.eds_summary['instrumentType']]
except KeyError:
summary['instrument_type'] = self.eds_summary['instrumentType']
try:
summary['block_type'] = self.__class__.expand_block[self.plate_setup['blockType']]
except KeyError:
summary['block_type'] = self.plate_setup['blockType']
summary['instrument_name'] = self.run_summary['instrumentName']
summary['instrument_serial_number'] = self.run_summary['instrumentSerialNumber']
summary['heated_cover_serial_number'] = self.run_summary['coverSerialNumber']
summary['block_serial_number'] = self.run_summary['blockSerialNumber']
run_start = self.parse_DA_date_format(self.run_summary['startTime'])
run_end = self.parse_DA_date_format(self.run_summary['endTime'])
summary['run_start_date/time'], summary['run_end_date/time'], summary['run_duration'] = \
self.get_run_time(run_start, run_end)
summary['sample_volume'] = self.run_method['sampleVolume']
summary['cover_temperature'] = self.run_method['coverTemperature']
summary['passive_reference'] = self.plate_setup['passiveReference']
summary['pcr_stage/step_number'] = f"Stage {self.analysis_settings['cqAnalysisStageNumber']} Step {self.analysis_settings['cqAnalysisStepNumber']}"
summary['quantification_cycle_method'] = self.analysis_results['cqAlgorithmType']
summary['analysis_date/time'] = self.parse_DA_date_format(self.eds_summary['analysis']['primary']['analysisTime'])
summary['software_name_and_version'] = "Design & Analysis Software v2.8.0"
summary['plugin_name_and_version'] = "Primary Analysis v1.8.1, Presence Absence v2.4.0"
return summary

7
src/submissions/backend/excel/reports.py
View File

@@ -7,8 +7,7 @@ from pathlib import Path
from datetime import date
from typing import Tuple
from backend.db.models import BasicSubmission
from tools import jinja_template_loading, get_first_blank_df_row, \
row_map
from tools import jinja_template_loading, get_first_blank_df_row, row_map
from PyQt6.QtWidgets import QWidget
from openpyxl.worksheet.worksheet import Worksheet
@@ -65,7 +64,7 @@ class ReportMaker(object):
old_lab = ""
output = []
# logger.debug(f"Report DataFrame: {df}")
for ii, row in enumerate(df.iterrows()):
for row in df.iterrows():
# logger.debug(f"Row {ii}: {row}")
lab = row[0][0]
# logger.debug(type(row))
@@ -111,7 +110,7 @@ class ReportMaker(object):
def fix_up_xl(self):
"""
Handles formatting of xl file.
Handles formatting of xl file, mediocrely.
"""
# logger.debug(f"Updating worksheet")
worksheet: Worksheet = self.writer.sheets['Report']

33
src/submissions/backend/excel/writer.py
View File

@@ -4,7 +4,7 @@ contains writer objects for pushing values to submission sheet templates.
import logging
from copy import copy
from pprint import pformat
from typing import List, Generator
from typing import List, Generator, Tuple
from openpyxl import load_workbook, Workbook
from backend.db.models import SubmissionType, KitType, BasicSubmission
from backend.validators.pydant import PydSubmission
@@ -19,13 +19,13 @@ class SheetWriter(object):
object to manage data placement into excel file
"""
def __init__(self, submission: PydSubmission, missing_only: bool = False):
def __init__(self, submission: PydSubmission):
"""
Args:
submission (PydSubmission): Object containing submission information.
missing_only (bool, optional): Whether to only fill in missing values. Defaults to False.
"""
self.sub = OrderedDict(submission.improved_dict())
# NOTE: Set values from pydantic object.
for k, v in self.sub.items():
match k:
case 'filepath':
@@ -41,18 +41,16 @@ class SheetWriter(object):
else:
self.sub[k] = v
# logger.debug(f"\n\nWriting to {submission.filepath.__str__()}\n\n")
if self.filepath.stem.startswith("tmp"):
template = self.submission_type.template_file
workbook = load_workbook(BytesIO(template))
missing_only = False
else:
try:
workbook = load_workbook(self.filepath)
except Exception as e:
logger.error(f"Couldn't open workbook due to {e}")
template = self.submission_type.template_file
workbook = load_workbook(BytesIO(template))
missing_only = False
# if self.filepath.stem.startswith("tmp"):
# template = self.submission_type.template_file
# workbook = load_workbook(BytesIO(template))
# else:
# try:
# workbook = load_workbook(self.filepath)
# except Exception as e:
# logger.error(f"Couldn't open workbook due to {e}")
template = self.submission_type.template_file
workbook = load_workbook(BytesIO(template))
# self.workbook = workbook
self.xl = workbook
self.write_info()
@@ -130,7 +128,7 @@ class InfoWriter(object):
self.info = self.reconcile_map(info_dict, self.info_map)
# logger.debug(pformat(self.info))
def reconcile_map(self, info_dict: dict, info_map: dict) -> dict:
def reconcile_map(self, info_dict: dict, info_map: dict) -> Generator[(Tuple[str, dict]), None, None]:
"""
Merge info with its locations
@@ -246,7 +244,7 @@ class ReagentWriter(object):
"""
for reagent in self.reagents:
sheet = self.xl[reagent['sheet']]
for k, v in reagent.items():
for v in reagent.values():
if not isinstance(v, dict):
continue
# logger.debug(
@@ -440,7 +438,6 @@ class TipWriter(object):
if tips_list is None:
return
for ii, tips in enumerate(tips_list, start=1):
# mp_info = tips_map[tips['role']]
mp_info = tips_map[tips.role]
# logger.debug(f"{tips['role']} map: {mp_info}")
placeholder = {}

10
src/submissions/backend/validators/__init__.py
View File

@@ -23,15 +23,15 @@ class RSLNamer(object):
# NOTE: Preferred method is path retrieval, but might also need validation for just string.
filename = Path(filename) if Path(filename).exists() else filename
self.submission_type = sub_type
if self.submission_type is None:
if not self.submission_type:
# logger.debug("Creating submission type because none exists")
self.submission_type = self.retrieve_submission_type(filename=filename)
logger.info(f"got submission type: {self.submission_type}")
if self.submission_type is not None:
if self.submission_type:
# logger.debug("Retrieving BasicSubmission subclass")
self.sub_object = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=self.submission_type)
self.parsed_name = self.retrieve_rsl_number(filename=filename, regex=self.sub_object.get_regex(submission_type=sub_type))
if data is None:
if not data:
data = dict(submission_type=self.submission_type)
if "submission_type" not in data.keys():
data['submission_type'] = self.submission_type
@@ -45,6 +45,9 @@ class RSLNamer(object):
Args:
filename (str | Path): filename
Raises:
TypeError: Raised if unsupported variable type for filename given.
Returns:
str: parsed submission type
"""
@@ -84,6 +87,7 @@ class RSLNamer(object):
case str():
submission_type = st_from_str(filename=filename)
case _:
raise TypeError(f"Unsupported filename type: {type(filename)}.")
submission_type = None
try:
check = submission_type is None

12
src/submissions/backend/validators/pydant.py
View File

@@ -157,7 +157,6 @@ class PydReagent(BaseModel):
if submission is not None and reagent not in submission.reagents:
assoc = SubmissionReagentAssociation(reagent=reagent, submission=submission)
assoc.comments = self.comment
# reagent.reagent_submission_associations.append(assoc)
else:
assoc = None
report.add_result(Result(owner=__name__, code=0, msg="New reagent created.", status="Information"))
@@ -165,7 +164,6 @@ class PydReagent(BaseModel):
if submission is not None and reagent not in submission.reagents:
assoc = SubmissionReagentAssociation(reagent=reagent, submission=submission)
assoc.comments = self.comment
# reagent.reagent_submission_associations.append(assoc)
else:
assoc = None
# add end-of-life extension from reagent type to expiry date
@@ -187,12 +185,10 @@ class PydSample(BaseModel, extra='allow'):
# logger.debug(f"Data for pydsample: {data}")
model = BasicSample.find_polymorphic_subclass(polymorphic_identity=data.sample_type)
for k, v in data.model_extra.items():
# print(k, v)
if k in model.timestamps():
if isinstance(v, str):
v = datetime.strptime(v, "%Y-%m-%d")
data.__setattr__(k, v)
# print(dir(data))
# logger.debug(f"Data coming out of validation: {pformat(data)}")
return data
@@ -329,8 +325,6 @@ class PydEquipment(BaseModel, extra='ignore'):
value = convert_nans_to_nones(value)
if not value:
value = ['']
# if len(value) == 0:
# value = ['']
try:
value = [item.strip() for item in value]
except AttributeError:
@@ -416,7 +410,6 @@ class PydSubmission(BaseModel, extra='allow'):
@field_validator("tips", mode="before")
@classmethod
def expand_tips(cls, value):
# print(f"\n{type(value)}\n")
if isinstance(value, dict):
value = value['value']
if isinstance(value, Generator):
@@ -594,7 +587,6 @@ class PydSubmission(BaseModel, extra='allow'):
value = value['value'].title()
return dict(value=value, missing=False)
else:
# return dict(value=RSLNamer(instr=values.data['filepath'].__str__()).submission_type.title(), missing=True)
return dict(value=RSLNamer.retrieve_submission_type(filename=values.data['filepath']).title(), missing=True)
@field_validator("submission_category", mode="before")
@@ -688,7 +680,6 @@ class PydSubmission(BaseModel, extra='allow'):
def __init__(self, run_custom: bool = False, **data):
super().__init__(**data)
# NOTE: this could also be done with default_factory
logger.debug(data)
self.submission_object = BasicSubmission.find_polymorphic_subclass(
polymorphic_identity=self.submission_type['value'])
self.namer = RSLNamer(self.rsl_plate_num['value'], sub_type=self.submission_type['value'])
@@ -729,7 +720,6 @@ class PydSubmission(BaseModel, extra='allow'):
output.append(dummy)
self.samples = output
# TODO: Return samples, reagents, etc to dictionaries as well.
def improved_dict(self, dictionaries: bool = True) -> dict:
"""
Adds model_extra to fields.
@@ -786,7 +776,6 @@ class PydSubmission(BaseModel, extra='allow'):
Returns:
Tuple[BasicSubmission, Result]: BasicSubmission instance, result object
"""
# self.__dict__.update(self.model_extra)
report = Report()
dicto = self.improved_dict()
instance, result = BasicSubmission.query_or_create(submission_type=self.submission_type['value'],
@@ -817,7 +806,6 @@ class PydSubmission(BaseModel, extra='allow'):
# logger.debug(f"Association: {assoc}")
if assoc is not None: # and assoc not in instance.submission_reagent_associations:
instance.submission_reagent_associations.append(assoc)
# instance.reagents.append(reagent)
case "samples":
for sample in self.samples:
sample, associations, _ = sample.toSQL(submission=instance)