diff --git a/TODO.md b/TODO.md index 399f242..af25dc9 100644 --- a/TODO.md +++ b/TODO.md @@ -1,3 +1,4 @@ +- [ ] Can my "to_dict", "to_sub_dict", "to_pydantic" methods be rewritten as properties? - [ ] Stop displacing date on Irida controls and just do what Turnaround time does. - [ ] Get Manager window working for KitType, maybe SubmissionType - [x] Find a way to merge AddEdit with ReagentAdder diff --git a/src/submissions/backend/db/__init__.py b/src/submissions/backend/db/__init__.py index 923f36d..27f068e 100644 --- a/src/submissions/backend/db/__init__.py +++ b/src/submissions/backend/db/__init__.py @@ -20,11 +20,11 @@ def set_sqlite_pragma(dbapi_connection, connection_record): cursor = dbapi_connection.cursor() if ctx.database_schema == "sqlite": execution_phrase = "PRAGMA foreign_keys=ON" + print(f"Executing '{execution_phrase}' in sql.") else: # print("Nothing to execute, returning") cursor.close() return - print(f"Executing '{execution_phrase}' in sql.") cursor.execute(execution_phrase) cursor.close() @@ -33,6 +33,17 @@ from .models import * def update_log(mapper, connection, target): + """ + Updates log table whenever an object with LogMixin is updated. + + Args: + mapper (): + connection (): + target (): + + Returns: + None + """ state = inspect(target) object_name = state.object.truncated_name update = dict(user=getuser(), time=datetime.now(), object=object_name, changes=[]) @@ -43,6 +54,7 @@ def update_log(mapper, connection, target): if attr.key == "custom": continue added = [str(item) for item in hist.added] + # NOTE: Attributes left out to save space if attr.key in ['artic_technician', 'submission_sample_associations', 'submission_reagent_associations', 'submission_equipment_associations', 'submission_tips_associations', 'contact_id', 'gel_info', 'gel_controls', 'source_plates']: diff --git a/src/submissions/backend/db/models/__init__.py b/src/submissions/backend/db/models/__init__.py index 68bd4be..0d47641 100644 --- a/src/submissions/backend/db/models/__init__.py +++ b/src/submissions/backend/db/models/__init__.py @@ -175,7 +175,7 @@ class BaseClass(Base): try: records = [obj.to_sub_dict(**kwargs) for obj in objects] except AttributeError: - records = [obj.to_omnigui_dict() for obj in objects] + records = [obj.omnigui_dict for obj in objects] return DataFrame.from_records(records) @classmethod @@ -241,7 +241,8 @@ class BaseClass(Base): report.add_result(Result(msg=e, status="Critical")) return report - def to_omnigui_dict(self) -> dict: + @property + def omnigui_dict(self) -> dict: """ For getting any object in an omni-thing friendly output. @@ -255,8 +256,8 @@ class BaseClass(Base): pass return dicto - @classmethod - def get_pydantic_model(cls) -> BaseModel: + @classproperty + def pydantic_model(cls) -> BaseModel: """ Gets the pydantic model corresponding to this object. @@ -271,7 +272,7 @@ class BaseClass(Base): return model @classproperty - def add_edit_tooltips(self) -> dict: + def add_edit_tooltips(cls) -> dict: """ Gets tooltips for Omni-add-edit diff --git a/src/submissions/backend/db/models/controls.py b/src/submissions/backend/db/models/controls.py index 3c5fa42..5ef6807 100644 --- a/src/submissions/backend/db/models/controls.py +++ b/src/submissions/backend/db/models/controls.py @@ -81,7 +81,8 @@ class ControlType(BaseClass): subtypes = sorted(list(jsoner[genera].keys()), reverse=True) return subtypes - def get_instance_class(self) -> Control: + @property + def instance_class(self) -> Control: """ Retrieves the Control class associated with this controltype @@ -314,7 +315,7 @@ class PCRControl(Control): def to_sub_dict(self) -> dict: """ - Creates dictionary of fields for this object + Creates dictionary of fields for this object. Returns: dict: Output dict of name, ct, subtype, target, reagent_lot and submitted_date @@ -471,8 +472,8 @@ class IridaControl(Control): _dict[key] = data[genus][key] yield _dict - @classmethod - def get_modes(cls) -> List[str]: + @classproperty + def modes(cls) -> List[str]: """ Get all control modes from database diff --git a/src/submissions/backend/db/models/kits.py b/src/submissions/backend/db/models/kits.py index f9f2582..a276ce3 100644 --- a/src/submissions/backend/db/models/kits.py +++ b/src/submissions/backend/db/models/kits.py @@ -9,7 +9,7 @@ from sqlalchemy.orm import relationship, validates, Query from sqlalchemy.ext.associationproxy import association_proxy from datetime import date, datetime, timedelta from tools import check_authorization, setup_lookup, Report, Result, check_regex_match, yaml_regex_creator, timezone -from typing import List, Literal, Generator, Any +from typing import List, Literal, Generator, Any, Tuple from pandas import ExcelFile from pathlib import Path from . import Base, BaseClass, Organization, LogMixin @@ -157,30 +157,62 @@ class KitType(BaseClass): else: return (item.reagent_role for item in relevant_associations) - def construct_xl_map_for_use(self, submission_type: str | SubmissionType) -> Generator[(str, str), None, None]: + def construct_xl_map_for_use(self, submission_type: str | SubmissionType) -> Tuple[dict|None, KitType]: """ Creates map of locations in Excel workbook for a SubmissionType Args: + new_kit (): submission_type (str | SubmissionType): Submissiontype.name Returns: Generator[(str, str), None, None]: Tuple containing information locations. """ + new_kit = self # NOTE: Account for submission_type variable type. match submission_type: case str(): - assocs = [item for item in self.kit_reagentrole_associations if - item.submission_type.name == submission_type] + # assocs = [item for item in self.kit_reagentrole_associations if + # item.submission_type.name == submission_type] + logger.debug(f"Query for {submission_type}") + submission_type = SubmissionType.query(name=submission_type) case SubmissionType(): - assocs = [item for item in self.kit_reagentrole_associations if item.submission_type == submission_type] + pass case _: raise ValueError(f"Wrong variable type: {type(submission_type)} used!") - for assoc in assocs: - try: - yield assoc.reagent_role.name, assoc.uses - except TypeError: - continue + logger.debug(f"Submission type: {submission_type}, Kit: {self}") + assocs = [item for item in self.kit_reagentrole_associations if item.submission_type == submission_type] + logger.debug(f"Associations: {assocs}") + # NOTE: rescue with submission type's default kit. + if not assocs: + logger.error( + f"No associations found with {self}. Attempting rescue with default kit: {submission_type.default_kit}") + new_kit = submission_type.default_kit + if not new_kit: + from frontend.widgets.pop_ups import ObjectSelector + dlg = ObjectSelector( + title="Select Kit", + message="Could not find reagents for this submission type/kit type combo.\nSelect new kit.", + obj_type=self.__class__, + values=[kit.name for kit in submission_type.kit_types] + ) + if dlg.exec(): + dlg_result = dlg.parse_form() + logger.debug(f"Dialog result: {dlg_result}") + new_kit = self.__class__.query(name=dlg_result) + logger.debug(f"Query result: {new_kit}") + # return new_kit.construct_xl_map_for_use(submission_type=submission_type) + else: + return None, new_kit + assocs = [item for item in new_kit.kit_reagentrole_associations if item.submission_type == submission_type] + # for assoc in assocs: + # try: + # yield assoc.reagent_role.name, assoc.uses + # except TypeError: + # continue + output = {assoc.reagent_role.name: assoc.uses for assoc in assocs} + logger.debug(f"Output: {output}") + return output, new_kit @classmethod @setup_lookup @@ -444,7 +476,7 @@ class Reagent(BaseClass, LogMixin): Concrete reagent instance """ - searchables = ["lot"] + searchables = [dict(label="Lot", field="lot")] id = Column(INTEGER, primary_key=True) #: primary key role = relationship("ReagentRole", back_populates="instances", @@ -548,7 +580,9 @@ class Reagent(BaseClass, LogMixin): def query_or_create(cls, **kwargs) -> Reagent: from backend.validators.pydant import PydReagent new = False - instance = cls.query(**kwargs) + disallowed = ['expiry'] + sanitized_kwargs = {k:v for k,v in kwargs.items() if k not in disallowed} + instance = cls.query(**sanitized_kwargs) if not instance or isinstance(instance, list): if "role" not in kwargs: try: @@ -557,7 +591,7 @@ class Reagent(BaseClass, LogMixin): pass instance = PydReagent(**kwargs) new = True - instance, _ = instance.toSQL() + instance = instance.to_sql() logger.info(f"Instance from query or create: {instance}") return instance, new @@ -644,38 +678,15 @@ class Reagent(BaseClass, LogMixin): except AttributeError as e: logger.error(f"Could not set {key} due to {e}") - @check_authorization def edit_from_search(self, obj, **kwargs): from frontend.widgets.omni_add_edit import AddEdit - role = ReagentRole.query(kwargs['role']) - if role: - role_name = role.name - else: - role_name = None - # dlg = AddReagentForm(reagent_lot=self.lot, reagent_role=role_name, expiry=self.expiry, reagent_name=self.name) dlg = AddEdit(parent=None, instance=self) if dlg.exec(): pyd = dlg.parse_form() for field in pyd.model_fields: self.set_attribute(field, pyd.__getattribute__(field)) - # for key, value in vars.items(): - # match key: - # case "expiry": - # if isinstance(value, str): - # field_value = datetime.strptime(value, "%Y-%m-%d") - # elif isinstance(value, date): - # field_value = datetime.combine(value, datetime.max.time()) - # else: - # field_value = value - # field_value.replace(tzinfo=timezone) - # case "role": - # continue - # case _: - # field_value = value - # self.__setattr__(key, field_value) self.save() - # print(self.__dict__) @classproperty def add_edit_tooltips(self): @@ -801,8 +812,8 @@ class SubmissionType(BaseClass): """ return f"" - @classmethod - def retrieve_template_file(cls) -> bytes: + @classproperty + def basic_template(cls) -> bytes: """ Grabs the default excel template file. @@ -812,7 +823,8 @@ class SubmissionType(BaseClass): submission_type = cls.query(name="Bacterial Culture") return submission_type.template_file - def get_template_file_sheets(self) -> List[str]: + @property + def template_file_sheets(self) -> List[str]: """ Gets names of sheet in the stored blank form. @@ -870,15 +882,6 @@ class SubmissionType(BaseClass): output['custom'] = self.info_map['custom'] return output - def construct_sample_map(self) -> dict: - """ - Returns sample map - - Returns: - dict: sample location map - """ - return self.sample_map - def construct_field_map(self, field: Literal['equipment', 'tip']) -> Generator[(str, dict), None, None]: """ Make a map of all locations for tips or equipment. @@ -895,7 +898,8 @@ class SubmissionType(BaseClass): fmap = {} yield getattr(item, f"{field}_role").name, fmap - def get_default_kit(self) -> KitType | None: + @property + def default_kit(self) -> KitType | None: """ If only one kits exists for this Submission Type, return it. @@ -941,7 +945,8 @@ class SubmissionType(BaseClass): raise TypeError(f"Type {type(equipment_role)} is not allowed") return list(set([item for items in relevant for item in items if item is not None])) - def get_submission_class(self) -> "BasicSubmission": + @property + def submission_class(self) -> "BasicSubmission": """ Gets submission class associated with this submission type. @@ -993,7 +998,8 @@ class SubmissionType(BaseClass): base_dict = dict(name=self.name) base_dict['info'] = self.construct_info_map(mode='export') base_dict['defaults'] = self.defaults - base_dict['samples'] = self.construct_sample_map() + # base_dict['samples'] = self.construct_sample_map() + base_dict['samples'] = self.sample_map base_dict['kits'] = [item.to_export_dict() for item in self.submissiontype_kit_associations] return base_dict @@ -1413,7 +1419,8 @@ class Equipment(BaseClass, LogMixin): return {k: v for k, v in self.__dict__.items()} def get_processes(self, submission_type: str | SubmissionType | None = None, - extraction_kit: str | KitType | None = None) -> List[str]: + extraction_kit: str | KitType | None = None, + equipment_role: str | EquipmentRole | None=None) -> List[str]: """ Get all processes associated with this Equipment for a given SubmissionType @@ -1433,6 +1440,8 @@ class Equipment(BaseClass, LogMixin): continue if extraction_kit and extraction_kit not in process.kit_types: continue + if equipment_role and equipment_role not in process.equipment_roles: + continue yield process @classmethod @@ -1489,12 +1498,12 @@ class Equipment(BaseClass, LogMixin): PydEquipment: pydantic equipment object """ from backend.validators.pydant import PydEquipment - processes = self.get_processes(submission_type=submission_type, extraction_kit=extraction_kit) + processes = self.get_processes(submission_type=submission_type, extraction_kit=extraction_kit, equipment_role=role) return PydEquipment(processes=processes, role=role, **self.to_dict(processes=False)) - @classmethod - def get_regex(cls) -> re.Pattern: + @classproperty + def manufacturer_regex(cls) -> re.Pattern: """ Creates regex to determine tip manufacturer @@ -1809,6 +1818,9 @@ class Process(BaseClass): def query(cls, name: str | None = None, id: int | None = None, + submission_type: str | SubmissionType | None = None, + extraction_kit : str | KitType | None = None, + equipment_role: str | KitType | None = None, limit: int = 0) -> Process | List[Process]: """ Lookup Processes @@ -1822,6 +1834,30 @@ class Process(BaseClass): Process|List[Process]: Process(es) matching criteria """ query = cls.__database_session__.query(cls) + match submission_type: + case str(): + submission_type = SubmissionType.query(name=submission_type) + query = query.filter(cls.submission_types.contains(submission_type)) + case SubmissionType(): + query = query.filter(cls.submission_types.contains(submission_type)) + case _: + pass + match extraction_kit: + case str(): + extraction_kit = KitType.query(name=extraction_kit) + query = query.filter(cls.kit_types.contains(extraction_kit)) + case KitType(): + query = query.filter(cls.kit_types.contains(extraction_kit)) + case _: + pass + match equipment_role: + case str(): + equipment_role = EquipmentRole.query(name=equipment_role) + query = query.filter(cls.equipment_roles.contains(equipment_role)) + case EquipmentRole(): + query = query.filter(cls.equipment_roles.contains(equipment_role)) + case _: + pass match name: case str(): query = query.filter(cls.name == name) @@ -1975,6 +2011,14 @@ class SubmissionTipsAssociation(BaseClass): query = query.filter(cls.role_name == role) return cls.execute_query(query=query, limit=limit, **kwargs) + @classmethod + def query_or_create(cls, tips, submission, role: str, **kwargs): + instance = cls.query(tip_id=tips.id, role=role, submission_id=submission.id, limit=1, **kwargs) + if instance is None: + instance = SubmissionTipsAssociation(submission=submission, tips=tips, role_name=role) + return instance + + def to_pydantic(self): from backend.validators import PydTips return PydTips(name=self.tips.name, lot=self.tips.lot, role=self.role_name) diff --git a/src/submissions/backend/db/models/organizations.py b/src/submissions/backend/db/models/organizations.py index 6e97e60..c081341 100644 --- a/src/submissions/backend/db/models/organizations.py +++ b/src/submissions/backend/db/models/organizations.py @@ -124,7 +124,7 @@ class Contact(BaseClass): Base of Contact """ - searchables =[] + searchables = [] id = Column(INTEGER, primary_key=True) #: primary key name = Column(String(64)) #: contact name diff --git a/src/submissions/backend/db/models/submissions.py b/src/submissions/backend/db/models/submissions.py index 9551bfd..fe2353e 100644 --- a/src/submissions/backend/db/models/submissions.py +++ b/src/submissions/backend/db/models/submissions.py @@ -2,8 +2,6 @@ Models for the main submission and sample types. """ from __future__ import annotations - -from collections import OrderedDict from copy import deepcopy from getpass import getuser import logging, uuid, tempfile, re, base64, numpy as np, pandas as pd, types, sys @@ -12,7 +10,7 @@ from zipfile import ZipFile, BadZipfile from tempfile import TemporaryDirectory, TemporaryFile from operator import itemgetter from pprint import pformat -from . import BaseClass, Reagent, SubmissionType, KitType, Organization, Contact, LogMixin +from . import BaseClass, Reagent, SubmissionType, KitType, Organization, Contact, LogMixin, SubmissionReagentAssociation from sqlalchemy import Column, String, TIMESTAMP, INTEGER, ForeignKey, JSON, FLOAT, case, func from sqlalchemy.orm import relationship, validates, Query from sqlalchemy.orm.attributes import flag_modified @@ -25,13 +23,15 @@ from openpyxl.drawing.image import Image as OpenpyxlImage from tools import row_map, setup_lookup, jinja_template_loading, rreplace, row_keys, check_key_or_attr, Result, Report, \ report_result, create_holidays_for_year from datetime import datetime, date, timedelta -from typing import List, Any, Tuple, Literal, Generator +from typing import List, Any, Tuple, Literal, Generator, Type from dateutil.parser import parse from pathlib import Path from jinja2.exceptions import TemplateNotFound from jinja2 import Template from PIL import Image + + logger = logging.getLogger(f"submissions.{__name__}") @@ -126,7 +126,7 @@ class BasicSubmission(BaseClass, LogMixin): def __repr__(self) -> str: return f"" - @classmethod + @classproperty def jsons(cls) -> List[str]: """ Get list of JSON db columns @@ -136,10 +136,10 @@ class BasicSubmission(BaseClass, LogMixin): """ output = [item.name for item in cls.__table__.columns if isinstance(item.type, JSON)] if issubclass(cls, BasicSubmission) and not cls.__name__ == "BasicSubmission": - output += BasicSubmission.jsons() + output += BasicSubmission.jsons return output - @classmethod + @classproperty def timestamps(cls) -> List[str]: """ Get list of TIMESTAMP columns @@ -149,7 +149,7 @@ class BasicSubmission(BaseClass, LogMixin): """ output = [item.name for item in cls.__table__.columns if isinstance(item.type, TIMESTAMP)] if issubclass(cls, BasicSubmission) and not cls.__name__ == "BasicSubmission": - output += BasicSubmission.timestamps() + output += BasicSubmission.timestamps return output @classmethod @@ -259,7 +259,8 @@ class BasicSubmission(BaseClass, LogMixin): Returns: dict: sample location map """ - return cls.get_submission_type(submission_type).construct_sample_map() + # return cls.get_submission_type(submission_type).construct_sample_map() + return cls.get_submission_type(submission_type).sample_map def generate_associations(self, name: str, extra: str | None = None): try: @@ -277,6 +278,7 @@ class BasicSubmission(BaseClass, LogMixin): Constructs dictionary used in submissions summary Args: + report (bool, optional): indicates if to be used for a report. Defaults to False. full_data (bool, optional): indicates if sample dicts to be constructed. Defaults to False. backup (bool, optional): passed to adjust_to_dict_samples. Defaults to False. @@ -323,7 +325,8 @@ class BasicSubmission(BaseClass, LogMixin): logger.error(f"We got an error retrieving reagents: {e}") reagents = [] finally: - for k, v in self.extraction_kit.construct_xl_map_for_use(self.submission_type): + dicto, _ = self.extraction_kit.construct_xl_map_for_use(self.submission_type) + for k, v in dicto.items(): if k == 'info': continue if not any([item['role'] == k for item in reagents]): @@ -381,7 +384,8 @@ class BasicSubmission(BaseClass, LogMixin): output["completed_date"] = self.completed_date return output - def calculate_column_count(self) -> int: + @property + def column_count(self) -> int: """ Calculate the number of columns in this submission @@ -391,13 +395,14 @@ class BasicSubmission(BaseClass, LogMixin): columns = set([assoc.column for assoc in self.submission_sample_associations]) return len(columns) - def calculate_base_cost(self): + + def calculate_base_cost(self) -> None: """ Calculates cost of the plate """ # NOTE: Calculate number of columns based on largest column number try: - cols_count_96 = self.calculate_column_count() + cols_count_96 = self.column_count except Exception as e: logger.error(f"Column count error: {e}") # NOTE: Get kit associated with this submission @@ -418,14 +423,15 @@ class BasicSubmission(BaseClass, LogMixin): logger.error(f"Calculation error: {e}") self.run_cost = round(self.run_cost, 2) - def hitpick_plate(self) -> list: + @property + def hitpicked(self) -> list: """ Returns positve sample locations for plate Returns: list: list of hitpick dictionaries for each sample """ - output_list = [assoc.to_hitpick() for assoc in self.submission_sample_associations] + output_list = [assoc.hitpicked for assoc in self.submission_sample_associations] return output_list @classmethod @@ -454,7 +460,8 @@ class BasicSubmission(BaseClass, LogMixin): html = template.render(samples=output_samples, PLATE_ROWS=plate_rows, PLATE_COLUMNS=plate_columns) return html + "
" - def get_used_equipment(self) -> List[str]: + @property + def used_equipment(self) -> Generator[str, None, None]: """ Gets EquipmentRole names associated with this BasicSubmission @@ -490,6 +497,7 @@ class BasicSubmission(BaseClass, LogMixin): 'source_plates', 'pcr_technician', 'ext_technician', 'artic_technician', 'cost_centre', 'signed_by', 'artic_date', 'gel_barcode', 'gel_date', 'ngs_date', 'contact_phone', 'contact', 'tips', 'gel_image_path', 'custom'] + # NOTE: dataframe equals dataframe of all columns not in exclude df = df.loc[:, ~df.columns.isin(exclude)] if chronologic: try: @@ -531,7 +539,7 @@ class BasicSubmission(BaseClass, LogMixin): field_value = value case "ctx" | "csv" | "filepath" | "equipment" | "controls": return - case item if item in self.jsons(): + case item if item in self.jsons: match key: case "custom" | "source_plates": existing = value @@ -549,7 +557,7 @@ class BasicSubmission(BaseClass, LogMixin): if isinstance(value, list): existing += value else: - if value is not None: + if value: existing.append(value) self.__setattr__(key, existing) # NOTE: Make sure this gets updated by telling SQLAlchemy it's been modified. @@ -636,12 +644,6 @@ class BasicSubmission(BaseClass, LogMixin): field_value = [item.to_pydantic() for item in self.submission_tips_associations] case "submission_type": field_value = dict(value=self.__getattribute__(key).name, missing=missing) - # case "contact": - # try: - # field_value = dict(value=self.__getattribute__(key).name, missing=missing) - # except AttributeError: - # contact = self.submitting_lab.contacts[0] - # field_value = dict(value=contact.name, missing=True) case "plate_number": key = 'rsl_plate_num' field_value = dict(value=self.rsl_plate_num, missing=missing) @@ -677,7 +679,7 @@ class BasicSubmission(BaseClass, LogMixin): return super().save() @classmethod - def get_regex(cls, submission_type: SubmissionType | str | None = None) -> str: + def get_regex(cls, submission_type: SubmissionType | str | None = None) -> re.Pattern: """ Gets the regex string for identifying a certain class of submission. @@ -685,18 +687,26 @@ class BasicSubmission(BaseClass, LogMixin): submission_type (SubmissionType | str | None, optional): submission type of interest. Defaults to None. Returns: - str: _description_ + str: String from which regex will be compiled. """ + # logger.debug(f"Class for regex: {cls}") try: - return cls.get_submission_type(submission_type).defaults['regex'] + regex = cls.get_submission_type(submission_type).defaults['regex'] except AttributeError as e: logger.error(f"Couldn't get submission type for {cls.__mapper_args__['polymorphic_identity']}") - return "" + regex = None + try: + regex = re.compile(rf"{regex}", flags=re.IGNORECASE | re.VERBOSE) + except re.error as e: + regex = cls.construct_regex() + # logger.debug(f"Returning regex: {regex}") + return regex + # NOTE: Polymorphic functions - @classmethod - def construct_regex(cls) -> re.Pattern: + @classproperty + def regex(cls) -> re.Pattern: """ Constructs catchall regex. @@ -762,7 +772,9 @@ class BasicSubmission(BaseClass, LogMixin): """ input_dict['custom'] = {} for k, v in custom_fields.items(): + logger.debug(f"Custom info parser getting type: {v['type']}") match v['type']: + # NOTE: 'exempt' type not currently used case "exempt": continue case "cell": @@ -796,7 +808,7 @@ class BasicSubmission(BaseClass, LogMixin): @classmethod def custom_validation(cls, pyd: "PydSubmission") -> "PydSubmission": """ - Performs any final custom parsing of the excel file. + Performs any final parsing of the pydantic object that only needs to be done for this cls. Args: input_dict (dict): Parser product up to this point. @@ -849,6 +861,14 @@ class BasicSubmission(BaseClass, LogMixin): @classmethod def custom_sample_writer(self, sample: dict) -> dict: + """ + Performs any final alterations to sample writing unique to this submission type. + Args: + sample (dict): Dictionary of sample values. + + Returns: + dict: Finalized dictionary. + """ return sample @classmethod @@ -884,7 +904,7 @@ class BasicSubmission(BaseClass, LogMixin): logger.error(f"Error making outstr: {e}, sending to RSLNamer to make new plate name.") outstr = RSLNamer.construct_new_plate_name(data=data) try: - # NOTE: Grab plate number + # NOTE: Grab plate number as number after a -|_ not followed by another number plate_number = re.search(r"(?:(-|_)\d)(?!\d)", outstr).group().strip("_").strip("-") except AttributeError as e: plate_number = "1" @@ -910,7 +930,7 @@ class BasicSubmission(BaseClass, LogMixin): Args: xl (pd.DataFrame): pcr info form - rsl_plate_number (str): rsl plate num of interest + rsl_plate_num (str): rsl plate num of interest Returns: Generator[dict, None, None]: Updated samples @@ -943,16 +963,16 @@ class BasicSubmission(BaseClass, LogMixin): submission = cls.query(rsl_plate_num=rsl_plate_num) name_column = 1 for item in location_map: - logger.debug(f"Checking {item}") + # logger.debug(f"Checking {item}") worksheet = xl[item['sheet']] for iii, row in enumerate(worksheet.iter_rows(max_row=len(worksheet['A']), max_col=name_column), start=1): - logger.debug(f"Checking row {row}, {iii}") + # logger.debug(f"Checking row {row}, {iii}") for cell in row: - logger.debug(f"Checking cell: {cell}, with value {cell.value} against {item['name']}") + # logger.debug(f"Checking cell: {cell}, with value {cell.value} against {item['name']}") if cell.value == item['name']: subtype, _ = item['name'].split("-") target = item['target'] - logger.debug(f"Subtype: {subtype}, target: {target}") + # logger.debug(f"Subtype: {subtype}, target: {target}") ct = worksheet.cell(row=iii, column=item['ct_column']).value # NOTE: Kind of a stop gap solution to find control reagents. if subtype == "PC": @@ -966,7 +986,7 @@ class BasicSubmission(BaseClass, LogMixin): assoc.reagent.role])), None) else: ctrl = None - logger.debug(f"Control reagent: {ctrl.__dict__}") + # logger.debug(f"Control reagent: {ctrl.__dict__}") try: ct = float(ct) except ValueError: @@ -982,7 +1002,7 @@ class BasicSubmission(BaseClass, LogMixin): target=target, reagent_lot=ctrl ) - logger.debug(f"Control output: {pformat(output)}") + # logger.debug(f"Control output: {pformat(output)}") yield output @classmethod @@ -1010,7 +1030,7 @@ class BasicSubmission(BaseClass, LogMixin): return samples @classmethod - def get_details_template(cls, base_dict: dict) -> Template: + def get_details_template(cls, base_dict: dict) -> Tuple[dict, Template]: """ Get the details jinja template for the correct class @@ -1040,8 +1060,8 @@ class BasicSubmission(BaseClass, LogMixin): submission_type_name: str | None = None, id: int | str | None = None, rsl_plate_num: str | None = None, - start_date: date | str | int | None = None, - end_date: date | str | int | None = None, + start_date: date | datetime | str | int | None = None, + end_date: date | datetime | str | int | None = None, reagent: Reagent | str | None = None, chronologic: bool = False, limit: int = 0, @@ -1065,6 +1085,7 @@ class BasicSubmission(BaseClass, LogMixin): Returns: models.BasicSubmission | List[models.BasicSubmission]: Submission(s) of interest """ + from ... import SubmissionReagentAssociation # NOTE: if you go back to using 'model' change the appropriate cls to model in the query filters if submission_type is not None: model = cls.find_polymorphic_subclass(polymorphic_identity=submission_type) @@ -1078,41 +1099,48 @@ class BasicSubmission(BaseClass, LogMixin): logger.warning(f"Start date with no end date, using today.") end_date = date.today() if end_date is not None and start_date is None: - logger.warning(f"End date with no start date, using Jan 1, 2023") + # NOTE: this query returns a tuple of (object, datetime), need to get only datetime. start_date = cls.__database_session__.query(cls, func.min(cls.submitted_date)).first()[1] + logger.warning(f"End date with no start date, using first submission date: {start_date}") if start_date is not None: match start_date: - case date() | datetime(): - start_date = start_date.strftime("%Y-%m-%d") + case date(): + pass + case datetime(): + start_date = start_date.date() case int(): start_date = datetime.fromordinal( - datetime(1900, 1, 1).toordinal() + start_date - 2).date().strftime("%Y-%m-%d") + datetime(1900, 1, 1).toordinal() + start_date - 2).date() case _: - start_date = parse(start_date).strftime("%Y-%m-%d") + start_date = parse(start_date).date() + # start_date = start_date.strftime("%Y-%m-%d") match end_date: - case date() | datetime(): - end_date = end_date + timedelta(days=1) - end_date = end_date.strftime("%Y-%m-%d") + case date(): + pass + case datetime(): + end_date = end_date# + timedelta(days=1) + # pass case int(): - end_date = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + end_date - 2).date() \ - + timedelta(days=1) - end_date = end_date.strftime("%Y-%m-%d") + end_date = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + end_date - 2).date()# \ + # + timedelta(days=1) case _: - end_date = parse(end_date) + timedelta(days=1) - end_date = end_date.strftime("%Y-%m-%d") - if start_date == end_date: - start_date = datetime.strptime(start_date, "%Y-%m-%d").strftime("%Y-%m-%d %H:%M:%S.%f") - query = query.filter(model.submitted_date == start_date) - else: - query = query.filter(model.submitted_date.between(start_date, end_date)) + end_date = parse(end_date).date()# + timedelta(days=1) + # end_date = end_date.strftime("%Y-%m-%d") + start_date = datetime.combine(start_date, datetime.min.time()).strftime("%Y-%m-%d %H:%M:%S.%f") + end_date = datetime.combine(end_date, datetime.max.time()).strftime("%Y-%m-%d %H:%M:%S.%f") + # if start_date == end_date: + # start_date = start_date.strftime("%Y-%m-%d %H:%M:%S.%f") + # query = query.filter(model.submitted_date == start_date) + # else: + query = query.filter(model.submitted_date.between(start_date, end_date)) # NOTE: by reagent (for some reason) match reagent: case str(): - query = query.join(model.submission_reagent_associations).filter( - SubmissionSampleAssociation.reagent.lot == reagent) + query = query.join(SubmissionReagentAssociation).join(Reagent).filter( + Reagent.lot == reagent) case Reagent(): - query = query.join(model.submission_reagent_associations).join( - SubmissionSampleAssociation.reagent).filter(Reagent.lot == reagent) + query = query.join(SubmissionReagentAssociation).filter( + SubmissionReagentAssociation.reagent == reagent) case _: pass # NOTE: by rsl number (returns only a single value) @@ -1217,6 +1245,7 @@ class BasicSubmission(BaseClass, LogMixin): msg = QuestionAsker(title="Delete?", message=f"Are you sure you want to delete {self.rsl_plate_num}?\n") if msg.exec(): try: + # NOTE: backs up file as xlsx, same as export. self.backup(fname=fname, full_backup=True) except BadZipfile: logger.error("Couldn't open zipfile for writing.") @@ -1285,16 +1314,16 @@ class BasicSubmission(BaseClass, LogMixin): if dlg.exec(): equipment = dlg.parse_form() for equip in equipment: - _, assoc = equip.toSQL(submission=self) + _, assoc = equip.to_sql(submission=self) try: assoc.save() except AttributeError as e: logger.error(f"Couldn't save association with {equip} due to {e}") if equip.tips: for tips in equip.tips: - logger.debug(f"Attempting to add tips assoc: {tips} (pydantic)") + # logger.debug(f"Attempting to add tips assoc: {tips} (pydantic)") tassoc = tips.to_sql(submission=self) - logger.debug(f"Attempting to add tips assoc: {tips.__dict__} (sql)") + # logger.debug(f"Attempting to add tips assoc: {tips.__dict__} (sql)") if tassoc not in self.submission_tips_associations: tassoc.save() else: @@ -1320,7 +1349,8 @@ class BasicSubmission(BaseClass, LogMixin): writer = pyd.to_writer() writer.xl.save(filename=fname.with_suffix(".xlsx")) - def get_turnaround_time(self) -> Tuple[int | None, bool | None]: + @property + def turnaround_time(self) -> int: try: completed = self.completed_date.date() except AttributeError: @@ -1328,25 +1358,24 @@ class BasicSubmission(BaseClass, LogMixin): return self.calculate_turnaround(start_date=self.submitted_date.date(), end_date=completed) @classmethod - def calculate_turnaround(cls, start_date: date | None = None, end_date: date | None = None) -> Tuple[ - int | None, bool | None]: - if 'pytest' not in sys.modules: - from tools import ctx - else: - from test_settings import ctx + def calculate_turnaround(cls, start_date: date | None = None, end_date: date | None = None) -> int: + """ + Calculates number of business days between data submitted and date completed + + Args: + start_date (date, optional): Date submitted. defaults to None. + end_date (date, optional): Date completed. defaults to None. + + Returns: + int: Number of business days. + """ if not end_date: - return None, None + return None try: delta = np.busday_count(start_date, end_date, holidays=create_holidays_for_year(start_date.year)) + 1 except ValueError: - return None, None - try: - tat = cls.get_default_info("turnaround_time") - except (AttributeError, KeyError): - tat = None - if not tat: - tat = ctx.TaT_threshold - return delta, delta <= tat + return None + return delta # NOTE: Below are the custom submission types @@ -1385,7 +1414,7 @@ class BacterialCulture(BasicSubmission): return template @classmethod - def custom_validation(cls, pyd) -> dict: + def custom_validation(cls, pyd) -> "PydSubmission": """ Extends parent. Currently finds control sample and adds to reagents. @@ -1395,7 +1424,7 @@ class BacterialCulture(BasicSubmission): info_map (dict | None, optional): _description_. Defaults to None. Returns: - dict: Updated dictionary. + PydSubmission: Updated pydantic. """ from . import ControlType pyd = super().custom_validation(pyd) @@ -1549,9 +1578,10 @@ class Wastewater(BasicSubmission): """ samples = [item for item in super().parse_pcr(xl=xl, rsl_plate_num=rsl_plate_num)] # NOTE: Due to having to run through samples in for loop we need to convert to list. + # NOTE: Also, you can't change the size of a list while iterating it, so don't even think about it. output = [] for sample in samples: - logger.debug(sample) + # logger.debug(sample) # NOTE: remove '-{target}' from controls sample['sample'] = re.sub('-N\\d*$', '', sample['sample']) # NOTE: if sample is already in output skip @@ -1559,7 +1589,7 @@ class Wastewater(BasicSubmission): logger.warning(f"Already have {sample['sample']}") continue # NOTE: Set ct values - logger.debug(f"Sample ct: {sample['ct']}") + # logger.debug(f"Sample ct: {sample['ct']}") sample[f"ct_{sample['target'].lower()}"] = sample['ct'] if isinstance(sample['ct'], float) else 0.0 # NOTE: Set assessment logger.debug(f"Sample assessemnt: {sample['assessment']}") @@ -1578,7 +1608,7 @@ class Wastewater(BasicSubmission): except KeyError: pass output.append(sample) - # NOTE: And then convert back to list ot keep fidelity with parent method. + # NOTE: And then convert back to list to keep fidelity with parent method. for sample in output: yield sample @@ -1644,7 +1674,7 @@ class Wastewater(BasicSubmission): return events @report_result - def link_pcr(self, obj): + def link_pcr(self, obj) -> Report: """ PYQT6 function to add PCR info to this submission @@ -1660,7 +1690,8 @@ class Wastewater(BasicSubmission): report.add_result(Result(msg="No file selected, cancelling.", status="Warning")) return report parser = PCRParser(filepath=fname, submission=self) - self.set_attribute("pcr_info", parser.pcr) + self.set_attribute("pcr_info", parser.pcr_info) + # NOTE: These are generators here, need to expand. pcr_samples = [sample for sample in parser.samples] pcr_controls = [control for control in parser.controls] self.save(original=False) @@ -1674,19 +1705,19 @@ class Wastewater(BasicSubmission): result = assoc.save() report.add_result(result) controltype = ControlType.query(name="PCR Control") - submitted_date = datetime.strptime(" ".join(parser.pcr['run_start_date/time'].split(" ")[:-1]), + submitted_date = datetime.strptime(" ".join(parser.pcr_info['run_start_date/time'].split(" ")[:-1]), "%Y-%m-%d %I:%M:%S %p") for control in pcr_controls: - logger.debug(f"Control coming into save: {control}") + # logger.debug(f"Control coming into save: {control}") new_control = PCRControl(**control) new_control.submitted_date = submitted_date new_control.controltype = controltype new_control.submission = self - logger.debug(f"Control coming into save: {new_control.__dict__}") + # logger.debug(f"Control coming into save: {new_control.__dict__}") new_control.save() return report - def update_subsampassoc(self, sample: BasicSample, input_dict: dict): + def update_subsampassoc(self, sample: BasicSample, input_dict: dict) -> SubmissionSampleAssociation: """ Updates a joined submission sample association by assigning ct values to n1 or n2 based on alphabetical sorting. @@ -1722,7 +1753,7 @@ class WastewaterArtic(BasicSubmission): artic_date = Column(TIMESTAMP) #: Date Artic Performed ngs_date = Column(TIMESTAMP) #: Date submission received gel_date = Column(TIMESTAMP) #: Date submission received - gel_barcode = Column(String(16)) + gel_barcode = Column(String(16)) #: Identifier for the used gel. __mapper_args__ = dict(polymorphic_identity="Wastewater Artic", polymorphic_load="inline", @@ -1769,6 +1800,16 @@ class WastewaterArtic(BasicSubmission): from openpyxl_image_loader.sheet_image_loader import SheetImageLoader def scrape_image(wb: Workbook, info_dict: dict) -> Image or None: + """ + Pulls image from excel workbook + + Args: + wb (Workbook): Workbook of interest. + info_dict (dict): Location map. + + Returns: + Image or None: Image of interest. + """ ws = wb[info_dict['sheet']] img_loader = SheetImageLoader(ws) for ii in range(info_dict['start_row'], info_dict['end_row'] + 1): @@ -1805,7 +1846,7 @@ class WastewaterArtic(BasicSubmission): if datum['plate'] in ["None", None, ""]: continue else: - datum['plate'] = RSLNamer(filename=datum['plate'], sub_type="Wastewater").parsed_name + datum['plate'] = RSLNamer(filename=datum['plate'], submission_type="Wastewater").parsed_name if xl is not None: try: input_dict['csv'] = xl["hitpicks_csv_to_export"] @@ -1864,6 +1905,7 @@ class WastewaterArtic(BasicSubmission): Returns: str: Updated name. """ + logger.debug(f"Incoming String: {instr}") try: # NOTE: Deal with PCR file. instr = re.sub(r"Artic", "", instr, flags=re.IGNORECASE) @@ -1900,8 +1942,7 @@ class WastewaterArtic(BasicSubmission): input_dict['source_plate_number'] = int(input_dict['source_plate_number']) except (ValueError, KeyError): input_dict['source_plate_number'] = 0 - # NOTE: Because generate_sample_object needs the submitter_id and the artic has the "({origin well})" - # at the end, this has to be done here. No moving to sqlalchemy object :( + # NOTE: Because generate_sample_object needs the submitter_id and the artic has the "({origin well})" at the end, this has to be done here. No moving to sqlalchemy object :( input_dict['submitter_id'] = re.sub(r"\s\(.+\)\s?$", "", str(input_dict['submitter_id'])).strip() try: input_dict['ww_processing_num'] = input_dict['sample_name_(lims)'] @@ -1988,7 +2029,11 @@ class WastewaterArtic(BasicSubmission): except AttributeError: plate_num = "1" plate_num = plate_num.strip("-") - repeat_num = re.search(r"R(?P\d)?$", "PBS20240426-2R").groups()[0] + # repeat_num = re.search(r"R(?P\d)?$", "PBS20240426-2R").groups()[0] + try: + repeat_num = re.search(r"R(?P\d)?$", processed).groups()[0] + except: + repeat_num = None if repeat_num is None and "R" in plate_num: repeat_num = "1" plate_num = re.sub(r"R", rf"R{repeat_num}", plate_num) @@ -2192,7 +2237,7 @@ class BasicSample(BaseClass, LogMixin): Base of basic sample which polymorphs into BCSample and WWSample """ - searchables = ['submitter_id'] + searchables = [dict(label="Submitter ID", field="submitter_id")] id = Column(INTEGER, primary_key=True) #: primary key submitter_id = Column(String(64), nullable=False, unique=True) #: identification from submitter @@ -2242,7 +2287,7 @@ class BasicSample(BaseClass, LogMixin): except AttributeError: return f" List[str]: """ Constructs a list of all attributes stored as SQL Timestamps @@ -2252,7 +2297,7 @@ class BasicSample(BaseClass, LogMixin): """ output = [item.name for item in cls.__table__.columns if isinstance(item.type, TIMESTAMP)] if issubclass(cls, BasicSample) and not cls.__name__ == "BasicSample": - output += BasicSample.timestamps() + output += BasicSample.timestamps return output def to_sub_dict(self, full_data: bool = False) -> dict: @@ -2293,7 +2338,7 @@ class BasicSample(BaseClass, LogMixin): @classmethod def find_polymorphic_subclass(cls, polymorphic_identity: str | None = None, - attrs: dict | None = None) -> BasicSample: + attrs: dict | None = None) -> Type[BasicSample]: """ Retrieves subclasses of BasicSample based on type name. @@ -2340,8 +2385,8 @@ class BasicSample(BaseClass, LogMixin): """ return input_dict - @classmethod - def get_details_template(cls) -> Template: + @classproperty + def details_template(cls) -> Template: """ Get the details jinja template for the correct class @@ -2458,15 +2503,15 @@ class BasicSample(BaseClass, LogMixin): def delete(self): raise AttributeError(f"Delete not implemented for {self.__class__}") - @classmethod - def get_searchables(cls) -> List[dict]: - """ - Delivers a list of fields that can be used in fuzzy search. - - Returns: - List[str]: List of fields. - """ - return [dict(label="Submitter ID", field="submitter_id")] + # @classmethod + # def get_searchables(cls) -> List[dict]: + # """ + # Delivers a list of fields that can be used in fuzzy search. + # + # Returns: + # List[str]: List of fields. + # """ + # return [dict(label="Submitter ID", field="submitter_id")] @classmethod def samples_to_df(cls, sample_list: List[BasicSample], **kwargs) -> pd.DataFrame: @@ -2504,6 +2549,16 @@ class BasicSample(BaseClass, LogMixin): pass def edit_from_search(self, obj, **kwargs): + """ + Function called form search. "Edit" is dependent on function as this one just shows details. + + Args: + obj (__type__): Parent widget. + **kwargs (): Required for all edit from search functions. + + Returns: + + """ self.show_details(obj) @@ -2514,7 +2569,7 @@ class WastewaterSample(BasicSample): Derivative wastewater sample """ - searchables = BasicSample.searchables + ['ww_processing_num', 'ww_full_sample_id', 'rsl_number'] + # searchables = BasicSample.searchables + ['ww_processing_num', 'ww_full_sample_id', 'rsl_number'] id = Column(INTEGER, ForeignKey('_basicsample.id'), primary_key=True) ww_processing_num = Column(String(64)) #: wastewater processing number @@ -2594,15 +2649,15 @@ class WastewaterSample(BasicSample): # logger.debug(pformat(output_dict, indent=4)) return output_dict - @classmethod - def get_searchables(cls) -> List[str]: + @classproperty + def searchables(cls) -> List[dict]: """ Delivers a list of fields that can be used in fuzzy search. Extends parent. Returns: List[str]: List of fields. """ - searchables = super().get_searchables() + searchables = super().searchables for item in ["ww_processing_num", "ww_full_sample_id", "rsl_number"]: label = item.strip("ww_").replace("_", " ").replace("rsl", "RSL").title() searchables.append(dict(label=label, field=item)) @@ -2726,7 +2781,8 @@ class SubmissionSampleAssociation(BaseClass): from backend.validators import PydSample return PydSample(**self.to_sub_dict()) - def to_hitpick(self) -> dict | None: + @property + def hitpicked(self) -> dict | None: """ Outputs a dictionary usable for html plate maps. @@ -2948,14 +3004,15 @@ class WastewaterAssociation(SubmissionSampleAssociation): logger.error(f"Couldn't check positives for {self.sample.rsl_number}. Looks like there isn't PCR data.") return sample - def to_hitpick(self) -> dict | None: + @property + def hitpicked(self) -> dict | None: """ Outputs a dictionary usable for html plate maps. Extends parent Returns: dict: dictionary of sample id, row and column in elution plate """ - sample = super().to_hitpick() + sample = super().hitpicked try: scaler = max([self.ct_n1, self.ct_n2]) except TypeError: diff --git a/src/submissions/backend/excel/parser.py b/src/submissions/backend/excel/parser.py index 6b5817a..6844de6 100644 --- a/src/submissions/backend/excel/parser.py +++ b/src/submissions/backend/excel/parser.py @@ -59,25 +59,27 @@ class SheetParser(object): Pulls basic information from the excel sheet """ parser = InfoParser(xl=self.xl, submission_type=self.submission_type, sub_object=self.sub_object) - info = parser.parse_info() - self.info_map = parser.map - # NOTE: in order to accommodate generic submission types we have to check for the type in the excel sheet and - # rerun accordingly + # info = parser.parsed_info + self.info_map = parser.info_map + # NOTE: in order to accommodate generic submission types we have to check for the type in the excel sheet and rerun accordingly try: - check = info['submission_type']['value'] not in [None, "None", "", " "] - except KeyError: + check = parser.parsed_info['submission_type']['value'] not in [None, "None", "", " "] + except KeyError as e: + logger.error(f"Couldn't check submission type due to KeyError: {e}") return logger.info( - f"Checking for updated submission type: {self.submission_type.name} against new: {info['submission_type']['value']}") - if self.submission_type.name != info['submission_type']['value']: + f"Checking for updated submission type: {self.submission_type.name} against new: {parser.parsed_info['submission_type']['value']}") + if self.submission_type.name != parser.parsed_info['submission_type']['value']: if check: - self.submission_type = SubmissionType.query(name=info['submission_type']['value']) + # NOTE: If initial submission type doesn't match parsed submission type, defer to parsed submission type. + self.submission_type = SubmissionType.query(name=parser.parsed_info['submission_type']['value']) logger.info(f"Updated self.submission_type to {self.submission_type}. Rerunning parse.") self.parse_info() else: self.submission_type = RSLNamer.retrieve_submission_type(filename=self.filepath) self.parse_info() - [self.sub.__setitem__(k, v) for k, v in info.items()] + for k, v in parser.parsed_info.items(): + self.sub.__setitem__(k, v) def parse_reagents(self, extraction_kit: str | None = None): """ @@ -90,28 +92,28 @@ class SheetParser(object): extraction_kit = self.sub['extraction_kit'] parser = ReagentParser(xl=self.xl, submission_type=self.submission_type, extraction_kit=extraction_kit) - self.sub['reagents'] = parser.parse_reagents() + self.sub['reagents'] = parser.parsed_reagents def parse_samples(self): """ Calls sample parser to pull info from the excel sheet """ parser = SampleParser(xl=self.xl, submission_type=self.submission_type) - self.sub['samples'] = parser.parse_samples() + self.sub['samples'] = parser.parsed_samples def parse_equipment(self): """ Calls equipment parser to pull info from the excel sheet """ parser = EquipmentParser(xl=self.xl, submission_type=self.submission_type) - self.sub['equipment'] = parser.parse_equipment() + self.sub['equipment'] = parser.parsed_equipment def parse_tips(self): """ Calls tips parser to pull info from the excel sheet """ parser = TipParser(xl=self.xl, submission_type=self.submission_type) - self.sub['tips'] = parser.parse_tips() + self.sub['tips'] = parser.parsed_tips def import_kit_validation_check(self): """ @@ -156,23 +158,23 @@ class InfoParser(object): if sub_object is None: sub_object = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=submission_type.name) self.submission_type_obj = submission_type + self.submission_type = dict(value=self.submission_type_obj.name, missing=True) self.sub_object = sub_object - self.map = self.fetch_submission_info_map() self.xl = xl - def fetch_submission_info_map(self) -> dict: + @property + def info_map(self) -> dict: """ Gets location of basic info from the submission_type object in the database. Returns: dict: Location map of all info for this submission type """ - self.submission_type = dict(value=self.submission_type_obj.name, missing=True) - info_map = self.sub_object.construct_info_map(submission_type=self.submission_type_obj, mode="read") # NOTE: Get the parse_info method from the submission type specified - return info_map + return self.sub_object.construct_info_map(submission_type=self.submission_type_obj, mode="read") - def parse_info(self) -> dict: + @property + def parsed_info(self) -> dict: """ Pulls basic info from the excel sheet. @@ -184,7 +186,7 @@ class InfoParser(object): for sheet in self.xl.sheetnames: ws = self.xl[sheet] relevant = [] - for k, v in self.map.items(): + for k, v in self.info_map.items(): # NOTE: If the value is hardcoded put it in the dictionary directly. Ex. Artic kit if k == "custom": continue @@ -215,7 +217,7 @@ class InfoParser(object): case "submitted_date": value, missing = is_missing(value) # NOTE: is field a JSON? Includes: Extraction info, PCR info, comment, custom - case thing if thing in self.sub_object.jsons(): + case thing if thing in self.sub_object.jsons: value, missing = is_missing(value) if missing: continue value = dict(name=f"Parser_{sheet}", text=value, time=datetime.now()) @@ -232,7 +234,7 @@ class InfoParser(object): except (KeyError, IndexError): continue # NOTE: Return after running the parser components held in submission object. - return self.sub_object.custom_info_parser(input_dict=dicto, xl=self.xl, custom_fields=self.map['custom']) + return self.sub_object.custom_info_parser(input_dict=dicto, xl=self.xl, custom_fields=self.info_map['custom']) class ReagentParser(object): @@ -252,16 +254,17 @@ class ReagentParser(object): if isinstance(submission_type, str): submission_type = SubmissionType.query(name=submission_type) self.submission_type_obj = submission_type + if not sub_object: + sub_object = submission_type.submission_class self.sub_object = sub_object if isinstance(extraction_kit, dict): extraction_kit = extraction_kit['value'] self.kit_object = KitType.query(name=extraction_kit) - self.map = self.fetch_kit_info_map(submission_type=submission_type) - logger.debug(f"Setting map: {self.map}") + # self.kit_map = self.kit_map(submission_type=submission_type) self.xl = xl - # @report_result - def fetch_kit_info_map(self, submission_type: str | SubmissionType) -> Tuple[Report, dict]: + @property + def kit_map(self) -> dict: """ Gets location of kit reagents from database @@ -271,38 +274,41 @@ class ReagentParser(object): Returns: dict: locations of reagent info for the kit. """ - report = Report() - if isinstance(submission_type, dict): - submission_type = submission_type['value'] - if isinstance(submission_type, str): - submission_type = SubmissionType.query(name=submission_type) - reagent_map = {k: v for k, v in self.kit_object.construct_xl_map_for_use(submission_type)} + # report = Report() + # if isinstance(submission_type, dict): + # submission_type = submission_type['value'] + # if isinstance(submission_type, str): + # submission_type = SubmissionType.query(name=submission_type) + logger.debug("Running kit map") + associations, self.kit_object = self.kit_object.construct_xl_map_for_use(submission_type=self.submission_type_obj) + reagent_map = {k: v for k, v in associations.items() if k != 'info'} try: del reagent_map['info'] except KeyError: pass - # NOTE: If reagent map is empty, maybe the wrong kit was given, check if there's only one kit for that submission type and use it if so. - if not reagent_map: - temp_kit_object = self.submission_type_obj.get_default_kit() - if temp_kit_object: - self.kit_object = temp_kit_object - logger.warning(f"Attempting to salvage with default kit {self.kit_object} and submission_type: {self.submission_type_obj}") - return self.fetch_kit_info_map(submission_type=self.submission_type_obj) - else: - logger.error(f"Still no reagent map, displaying error.") - try: - ext_kit_loc = self.submission_type_obj.info_map['extraction_kit']['read'][0] - location_string = f"Sheet: {ext_kit_loc['sheet']}, Row: {ext_kit_loc['row']}, Column: {ext_kit_loc['column']}?" - except (IndexError, KeyError): - location_string = "" - report.add_result(Result(owner=__name__, code=0, - msg=f"No kit map found for {self.kit_object.name}.\n\n" - f"Are you sure you put the right kit in:\n\n{location_string}?", - status="Critical")) - logger.debug(f"Here is the map coming out: {reagent_map}") + # # NOTE: If reagent map is empty, maybe the wrong kit was given, check if there's only one kit for that submission type and use it if so. + # if not reagent_map: + # temp_kit_object = self.submission_type_obj.default_kit + # if temp_kit_object: + # self.kit_object = temp_kit_object + # logger.warning(f"Attempting to salvage with default kit {self.kit_object} and submission_type: {self.submission_type_obj}") + # return self.fetch_kit_map(submission_type=self.submission_type_obj) + # else: + # logger.error(f"Still no reagent map, displaying error.") + # try: + # ext_kit_loc = self.submission_type_obj.info_map['extraction_kit']['read'][0] + # location_string = f"Sheet: {ext_kit_loc['sheet']}, Row: {ext_kit_loc['row']}, Column: {ext_kit_loc['column']}?" + # except (IndexError, KeyError): + # location_string = "" + # report.add_result(Result(owner=__name__, code=0, + # msg=f"No kit map found for {self.kit_object.name}.\n\n" + # f"Are you sure you put the right kit in:\n\n{location_string}?", + # status="Critical")) + # logger.debug(f"Here is the map coming out: {reagent_map}") return reagent_map - def parse_reagents(self) -> Generator[dict, None, None]: + @property + def parsed_reagents(self) -> Generator[dict, None, None]: """ Extracts reagent information from the Excel form. @@ -311,7 +317,7 @@ class ReagentParser(object): """ for sheet in self.xl.sheetnames: ws = self.xl[sheet] - relevant = {k.strip(): v for k, v in self.map.items() if sheet in self.map[k]['sheet']} + relevant = {k.strip(): v for k, v in self.kit_map.items() if sheet in self.kit_map[k]['sheet']} if not relevant: continue for item in relevant: @@ -367,11 +373,14 @@ class SampleParser(object): f"Sample parser attempting to fetch submission class with polymorphic identity: {self.submission_type}") sub_object = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=self.submission_type) self.sub_object = sub_object - self.sample_info_map = self.fetch_sample_info_map(submission_type=submission_type, sample_map=sample_map) - self.plate_map_samples = self.parse_plate_map() - self.lookup_samples = self.parse_lookup_table() + self.sample_type = self.sub_object.get_default_info("sample_type", submission_type=submission_type) + self.samp_object = BasicSample.find_polymorphic_subclass(polymorphic_identity=self.sample_type) + # self.sample_map = self.sample_map(submission_type=submission_type, sample_map=sample_map) + # self.plate_map_samples = self.parse_plate_map() + # self.lookup_samples = self.parse_lookup_table() - def fetch_sample_info_map(self, submission_type: str, sample_map: dict | None = None) -> dict: + @property + def sample_map(self) -> dict: """ Gets info locations in excel book for submission type. @@ -381,15 +390,16 @@ class SampleParser(object): Returns: dict: Info locations. """ - self.sample_type = self.sub_object.get_default_info("sample_type", submission_type=submission_type) - self.samp_object = BasicSample.find_polymorphic_subclass(polymorphic_identity=self.sample_type) - if sample_map is None: - sample_info_map = self.sub_object.construct_sample_map(submission_type=self.submission_type_obj) - else: - sample_info_map = sample_map - return sample_info_map - def parse_plate_map(self) -> List[dict]: + # if sample_map is None: + # sample_info_map = self.sub_object.construct_sample_map(submission_type=self.submission_type_obj) + # else: + # sample_info_map = sample_map + # return sample_info_map + return self.sub_object.construct_sample_map(submission_type=self.submission_type_obj) + + @property + def plate_map_samples(self) -> List[dict]: """ Parse sample location/name from plate map @@ -397,7 +407,7 @@ class SampleParser(object): List[dict]: List of sample ids and locations. """ invalids = [0, "0", "EMPTY"] - smap = self.sample_info_map['plate_map'] + smap = self.sample_map['plate_map'] ws = self.xl[smap['sheet']] plate_map_samples = [] for ii, row in enumerate(range(smap['start_row'], smap['end_row'] + 1), start=1): @@ -414,7 +424,8 @@ class SampleParser(object): pass return plate_map_samples - def parse_lookup_table(self) -> List[dict]: + @property + def lookup_samples(self) -> List[dict]: """ Parse misc info from lookup table. @@ -422,7 +433,7 @@ class SampleParser(object): List[dict]: List of basic sample info. """ - lmap = self.sample_info_map['lookup_table'] + lmap = self.sample_map['lookup_table'] ws = self.xl[lmap['sheet']] lookup_samples = [] for ii, row in enumerate(range(lmap['start_row'], lmap['end_row'] + 1), start=1): @@ -441,7 +452,8 @@ class SampleParser(object): lookup_samples.append(self.samp_object.parse_sample(row_dict)) return lookup_samples - def parse_samples(self) -> Generator[dict, None, None]: + @property + def parsed_samples(self) -> Generator[dict, None, None]: """ Merges sample info from lookup table and plate map. @@ -461,7 +473,7 @@ class SampleParser(object): pass yield new else: - merge_on_id = self.sample_info_map['lookup_table']['merge_on_id'] + merge_on_id = self.sample_map['lookup_table']['merge_on_id'] logger.info(f"Merging sample info using {merge_on_id}") plate_map_samples = sorted(copy(self.plate_map_samples), key=itemgetter('id')) lookup_samples = sorted(copy(self.lookup_samples), key=itemgetter(merge_on_id)) @@ -507,9 +519,10 @@ class EquipmentParser(object): submission_type = SubmissionType.query(name=submission_type) self.submission_type = submission_type self.xl = xl - self.map = self.fetch_equipment_map() + # self.equipment_map = self.fetch_equipment_map() - def fetch_equipment_map(self) -> dict: + @property + def equipment_map(self) -> dict: """ Gets the map of equipment locations in the submission type's spreadsheet @@ -528,14 +541,15 @@ class EquipmentParser(object): Returns: str: asset number """ - regex = Equipment.get_regex() + regex = Equipment.manufacturer_regex try: return regex.search(input).group().strip("-") except AttributeError as e: logger.error(f"Error getting asset number for {input}: {e}") return input - def parse_equipment(self) -> Generator[dict, None, None]: + @property + def parsed_equipment(self) -> Generator[dict, None, None]: """ Scrapes equipment from xl sheet @@ -545,7 +559,7 @@ class EquipmentParser(object): for sheet in self.xl.sheetnames: ws = self.xl[sheet] try: - relevant = {k: v for k, v in self.map.items() if v['sheet'] == sheet} + relevant = {k: v for k, v in self.equipment_map.items() if v['sheet'] == sheet} except (TypeError, KeyError) as e: logger.error(f"Error creating relevant equipment list: {e}") continue @@ -566,7 +580,7 @@ class EquipmentParser(object): nickname=eq.nickname) except AttributeError: logger.error(f"Unable to add {eq} to list.") - + continue class TipParser(object): """ @@ -583,9 +597,10 @@ class TipParser(object): submission_type = SubmissionType.query(name=submission_type) self.submission_type = submission_type self.xl = xl - self.map = self.fetch_tip_map() + # self.map = self.fetch_tip_map() - def fetch_tip_map(self) -> dict: + @property + def tip_map(self) -> dict: """ Gets the map of equipment locations in the submission type's spreadsheet @@ -594,7 +609,8 @@ class TipParser(object): """ return {k: v for k, v in self.submission_type.construct_field_map("tip")} - def parse_tips(self) -> List[dict]: + @property + def parsed_tips(self) -> Generator[dict, None, None]: """ Scrapes equipment from xl sheet @@ -604,7 +620,7 @@ class TipParser(object): for sheet in self.xl.sheetnames: ws = self.xl[sheet] try: - relevant = {k: v for k, v in self.map.items() if v['sheet'] == sheet} + relevant = {k: v for k, v in self.tip_map.items() if v['sheet'] == sheet} except (TypeError, KeyError) as e: logger.error(f"Error creating relevant equipment list: {e}") continue @@ -653,11 +669,12 @@ class PCRParser(object): else: self.submission_obj = submission rsl_plate_num = self.submission_obj.rsl_plate_num - self.pcr = self.parse_general() + # self.pcr = self.parse_general() self.samples = self.submission_obj.parse_pcr(xl=self.xl, rsl_plate_num=rsl_plate_num) self.controls = self.submission_obj.parse_pcr_controls(xl=self.xl, rsl_plate_num=rsl_plate_num) - def parse_general(self): + @property + def pcr_info(self) -> dict: """ Parse general info rows for all types of PCR results diff --git a/src/submissions/backend/excel/reports.py b/src/submissions/backend/excel/reports.py index 0b57ca0..797d72f 100644 --- a/src/submissions/backend/excel/reports.py +++ b/src/submissions/backend/excel/reports.py @@ -1,6 +1,7 @@ ''' Contains functions for generating summary reports ''' +import sys from pprint import pformat from pandas import DataFrame, ExcelWriter import logging @@ -8,7 +9,7 @@ from pathlib import Path from datetime import date from typing import Tuple from backend.db.models import BasicSubmission -from tools import jinja_template_loading, get_first_blank_df_row, row_map +from tools import jinja_template_loading, get_first_blank_df_row, row_map, ctx from PyQt6.QtWidgets import QWidget from openpyxl.worksheet.worksheet import Worksheet @@ -18,6 +19,9 @@ env = jinja_template_loading() class ReportArchetype(object): + """ + Made for children to inherit 'write_report", etc. + """ def write_report(self, filename: Path | str, obj: QWidget | None = None): """ @@ -168,7 +172,21 @@ class TurnaroundMaker(ReportArchetype): Returns: """ - days, tat_ok = sub.get_turnaround_time() + if 'pytest' not in sys.modules: + from tools import ctx + else: + from test_settings import ctx + days = sub.turnaround_time + try: + tat = sub.get_default_info("turnaround_time") + except (AttributeError, KeyError): + tat = None + if not tat: + tat = ctx.TaT_threshold + try: + tat_ok = days <= tat + except TypeError: + return {} return dict(name=str(sub.rsl_plate_num), days=days, submitted_date=sub.submitted_date, completed_date=sub.completed_date, acceptable=tat_ok) @@ -179,5 +197,3 @@ class ChartReportMaker(ReportArchetype): self.df = df self.sheet_name = sheet_name - - diff --git a/src/submissions/backend/excel/writer.py b/src/submissions/backend/excel/writer.py index 20274ab..7e8b902 100644 --- a/src/submissions/backend/excel/writer.py +++ b/src/submissions/backend/excel/writer.py @@ -45,7 +45,7 @@ class SheetWriter(object): template = self.submission_type.template_file if not template: logger.error(f"No template file found, falling back to Bacterial Culture") - template = SubmissionType.retrieve_template_file() + template = SubmissionType.basic_template workbook = load_workbook(BytesIO(template)) self.xl = workbook self.write_info() @@ -155,8 +155,11 @@ class InfoWriter(object): """ final_info = {} for k, v in self.info: - if k == "custom": - continue + match k: + case "custom": + continue + # case "comment": + # NOTE: merge all comments to fit in single cell. if k == "comment" and isinstance(v['value'], list): json_join = [item['text'] for item in v['value'] if 'text' in item.keys()] @@ -170,6 +173,7 @@ class InfoWriter(object): for loc in locations: sheet = self.xl[loc['sheet']] try: + logger.debug(f"Writing {v['value']} to row {loc['row']} and column {loc['column']}") sheet.cell(row=loc['row'], column=loc['column'], value=v['value']) except AttributeError as e: logger.error(f"Can't write {k} to that cell due to AttributeError: {e}") @@ -196,9 +200,13 @@ class ReagentWriter(object): self.xl = xl if isinstance(submission_type, str): submission_type = SubmissionType.query(name=submission_type) + self.submission_type_obj = submission_type if isinstance(extraction_kit, str): extraction_kit = KitType.query(name=extraction_kit) - reagent_map = {k: v for k, v in extraction_kit.construct_xl_map_for_use(submission_type)} + self.kit_object = extraction_kit + associations, self.kit_object = self.kit_object.construct_xl_map_for_use( + submission_type=self.submission_type_obj) + reagent_map = {k: v for k, v in associations.items()} self.reagents = self.reconcile_map(reagent_list=reagent_list, reagent_map=reagent_map) def reconcile_map(self, reagent_list: List[dict], reagent_map: dict) -> Generator[dict, None, None]: @@ -264,7 +272,7 @@ class SampleWriter(object): submission_type = SubmissionType.query(name=submission_type) self.submission_type = submission_type self.xl = xl - self.sample_map = submission_type.construct_sample_map()['lookup_table'] + self.sample_map = submission_type.sample_map['lookup_table'] # NOTE: exclude any samples without a submission rank. samples = [item for item in self.reconcile_map(sample_list) if item['submission_rank'] > 0] self.samples = sorted(samples, key=itemgetter('submission_rank')) @@ -282,7 +290,7 @@ class SampleWriter(object): """ multiples = ['row', 'column', 'assoc_id', 'submission_rank'] for sample in sample_list: - sample = self.submission_type.get_submission_class().custom_sample_writer(sample) + sample = self.submission_type.submission_class.custom_sample_writer(sample) for assoc in zip(sample['row'], sample['column'], sample['submission_rank']): new = dict(row=assoc[0], column=assoc[1], submission_rank=assoc[2]) for k, v in sample.items(): @@ -354,7 +362,7 @@ class EquipmentWriter(object): equipment_map (dict): Dictionary of equipment locations Returns: - List[dict]: List of merged dictionaries + List[dict]: List of merged dictionaries """ if equipment_list is None: return diff --git a/src/submissions/backend/validators/__init__.py b/src/submissions/backend/validators/__init__.py index 394a118..9a92a0b 100644 --- a/src/submissions/backend/validators/__init__.py +++ b/src/submissions/backend/validators/__init__.py @@ -19,21 +19,22 @@ class RSLNamer(object): Object that will enforce proper formatting on RSL plate names. """ - def __init__(self, filename: str, sub_type: str | None = None, data: dict | None = None): + def __init__(self, filename: str, submission_type: str | None = None, data: dict | None = None): # NOTE: Preferred method is path retrieval, but might also need validation for just string. filename = Path(filename) if Path(filename).exists() else filename - self.submission_type = sub_type + self.submission_type = submission_type if not self.submission_type: self.submission_type = self.retrieve_submission_type(filename=filename) logger.info(f"got submission type: {self.submission_type}") if self.submission_type: self.sub_object = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=self.submission_type) - self.parsed_name = self.retrieve_rsl_number(filename=filename, regex=self.sub_object.get_regex(submission_type=sub_type)) + self.parsed_name = self.retrieve_rsl_number(filename=filename, regex=self.sub_object.get_regex(submission_type=submission_type)) if not data: data = dict(submission_type=self.submission_type) if "submission_type" not in data.keys(): data['submission_type'] = self.submission_type self.parsed_name = self.sub_object.enforce_name(instr=self.parsed_name, data=data) + logger.info(f"Parsed name: {self.parsed_name}") @classmethod def retrieve_submission_type(cls, filename: str | Path) -> str: @@ -57,7 +58,7 @@ class RSLNamer(object): categories = wb.properties.category.split(";") submission_type = next(item.strip().title() for item in categories) except (StopIteration, AttributeError): - sts = {item.name: item.get_template_file_sheets() for item in SubmissionType.query() if item.template_file} + sts = {item.name: item.template_file_sheets for item in SubmissionType.query() if item.template_file} try: submission_type = next(k.title() for k,v in sts.items() if wb.sheetnames==v) except StopIteration: @@ -69,7 +70,7 @@ class RSLNamer(object): def st_from_str(filename:str) -> str: if filename.startswith("tmp"): return "Bacterial Culture" - regex = BasicSubmission.construct_regex() + regex = BasicSubmission.regex m = regex.search(filename) try: submission_type = m.lastgroup @@ -94,14 +95,15 @@ class RSLNamer(object): raise ValueError("Submission Type came back as None.") from frontend.widgets import ObjectSelector dlg = ObjectSelector(title="Couldn't parse submission type.", - message="Please select submission type from list below.", obj_type=SubmissionType) + message="Please select submission type from list below.", + obj_type=SubmissionType) if dlg.exec(): submission_type = dlg.parse_form() submission_type = submission_type.replace("_", " ") return submission_type @classmethod - def retrieve_rsl_number(cls, filename: str | Path, regex: str | None = None): + def retrieve_rsl_number(cls, filename: str | Path, regex: re.Pattern | None = None): """ Uses regex to retrieve the plate number and submission type from an input string @@ -110,12 +112,7 @@ class RSLNamer(object): filename (str): string to be parsed """ if regex is None: - regex = BasicSubmission.construct_regex() - else: - try: - regex = re.compile(rf'{regex}', re.IGNORECASE | re.VERBOSE) - except re.error as e: - regex = BasicSubmission.construct_regex() + regex = BasicSubmission.regex match filename: case Path(): m = regex.search(filename.stem) @@ -135,7 +132,7 @@ class RSLNamer(object): @classmethod def construct_new_plate_name(cls, data: dict) -> str: """ - Make a brand new plate name from submission data. + Make a brand-new plate name from submission data. Args: data (dict): incoming submission data @@ -179,7 +176,13 @@ class RSLNamer(object): template = environment.from_string(template) return template.render(**kwargs) - def calculate_repeat(self): + def calculate_repeat(self) -> str: + """ + Determines what repeat number this plate is. + + Returns: + str: Repeat number. + """ regex = re.compile(r"-\d(?PR\d)") m = regex.search(self.parsed_name) if m is not None: diff --git a/src/submissions/backend/validators/pydant.py b/src/submissions/backend/validators/pydant.py index 38a5f84..13cfeb6 100644 --- a/src/submissions/backend/validators/pydant.py +++ b/src/submissions/backend/validators/pydant.py @@ -73,7 +73,7 @@ class PydReagent(BaseModel): if value is not None: match value: case int(): - return datetime.fromordinal(datetime(1900, 1, 1).toordinal() + value - 2).date() + return datetime.fromordinal(datetime(1900, 1, 1).toordinal() + value - 2) case 'NA': return value case str(): @@ -117,7 +117,8 @@ class PydReagent(BaseModel): fields = list(self.model_fields.keys()) + extras return {k: getattr(self, k) for k in fields} - def toSQL(self, submission: BasicSubmission | str = None) -> Tuple[Reagent, Report]: + @report_result + def to_sql(self, submission: BasicSubmission | str = None) -> Tuple[Reagent, Report]: """ Converts this instance into a backend.db.models.kit.Reagent instance @@ -128,6 +129,7 @@ class PydReagent(BaseModel): if self.model_extra is not None: self.__dict__.update(self.model_extra) reagent = Reagent.query(lot=self.lot, name=self.name) + logger.debug(f"Reagent: {reagent}") if reagent is None: reagent = Reagent() for key, value in self.__dict__.items(): @@ -140,7 +142,6 @@ class PydReagent(BaseModel): assoc.comments = self.comment else: assoc = None - report.add_result(Result(owner=__name__, code=0, msg="New reagent created.", status="Information")) else: if submission is not None and reagent not in submission.reagents: submission.update_reagentassoc(reagent=reagent, role=self.role) @@ -160,7 +161,7 @@ class PydSample(BaseModel, extra='allow'): def validate_model(cls, data): model = BasicSample.find_polymorphic_subclass(polymorphic_identity=data.sample_type) for k, v in data.model_extra.items(): - if k in model.timestamps(): + if k in model.timestamps: if isinstance(v, str): v = datetime.strptime(v, "%Y-%m-%d") data.__setattr__(k, v) @@ -202,7 +203,7 @@ class PydSample(BaseModel, extra='allow'): fields = list(self.model_fields.keys()) + list(self.model_extra.keys()) return {k: getattr(self, k) for k in fields} - def toSQL(self, submission: BasicSubmission | str = None) -> Tuple[ + def to_sql(self, submission: BasicSubmission | str = None) -> Tuple[ BasicSample, List[SubmissionSampleAssociation], Result | None]: """ Converts this instance into a backend.db.models.submissions.Sample object @@ -271,7 +272,7 @@ class PydTips(BaseModel): def to_sql(self, submission: BasicSubmission) -> SubmissionTipsAssociation: """ - Con + Convert this object to the SQL version for database storage. Args: submission (BasicSubmission): A submission object to associate tips represented here. @@ -280,10 +281,10 @@ class PydTips(BaseModel): SubmissionTipsAssociation: Association between queried tips and submission """ tips = Tips.query(name=self.name, limit=1) - logger.debug(f"Tips query has yielded: {tips}") - assoc = SubmissionTipsAssociation.query(tip_id=tips.id, submission_id=submission.id, role=self.role, limit=1) - if assoc is None: - assoc = SubmissionTipsAssociation(submission=submission, tips=tips, role_name=self.role) + # logger.debug(f"Tips query has yielded: {tips}") + assoc = SubmissionTipsAssociation.query_or_create(tips=tips, submission=submission, role=self.role, limit=1) + # if assoc is None: + # assoc = SubmissionTipsAssociation(submission=submission, tips=tips, role_name=self.role) return assoc @@ -316,7 +317,7 @@ class PydEquipment(BaseModel, extra='ignore'): pass return value - def toSQL(self, submission: BasicSubmission | str = None) -> Tuple[Equipment, SubmissionEquipmentAssociation]: + def to_sql(self, submission: BasicSubmission | str = None, extraction_kit: KitType | str = None) -> Tuple[Equipment, SubmissionEquipmentAssociation]: """ Creates Equipment and SubmssionEquipmentAssociations for this PydEquipment @@ -328,6 +329,8 @@ class PydEquipment(BaseModel, extra='ignore'): """ if isinstance(submission, str): submission = BasicSubmission.query(rsl_plate_num=submission) + if isinstance(extraction_kit, str): + extraction_kit = KitType.query(name=extraction_kit) equipment = Equipment.query(asset_number=self.asset_number) if equipment is None: logger.error("No equipment found. Returning None.") @@ -343,7 +346,12 @@ class PydEquipment(BaseModel, extra='ignore'): if assoc is None: assoc = SubmissionEquipmentAssociation(submission=submission, equipment=equipment) # TODO: This seems precarious. What if there is more than one process? - process = Process.query(name=self.processes[0]) + # NOTE: It looks like the way fetching the processes is done in the SQL model, this shouldn't be a problem, but I'll include a failsafe. + # NOTE: I need to find a way to filter this by the kit involved. + if len(self.processes) > 1: + process = Process.query(submission_type=submission.get_submission_type(), extraction_kit=extraction_kit, equipment_role=self.role) + else: + process = Process.query(name=self.processes[0]) if process is None: logger.error(f"Found unknown process: {process}.") assoc.process = process @@ -405,10 +413,12 @@ class PydSubmission(BaseModel, extra='allow'): @field_validator('equipment', mode='before') @classmethod def convert_equipment_dict(cls, value): - if isinstance(value, Generator): - return [PydEquipment(**equipment) for equipment in value] if isinstance(value, dict): return value['value'] + if isinstance(value, Generator): + return [PydEquipment(**equipment) for equipment in value] + if not value: + return [] return value @field_validator('comment', mode='before') @@ -443,12 +453,11 @@ class PydSubmission(BaseModel, extra='allow'): def strip_datetime_string(cls, value): match value['value']: case date(): - return value + output = datetime.combine(value['value'], datetime.min.time()) case datetime(): - return value.date() + pass case int(): - return dict(value=datetime.fromordinal(datetime(1900, 1, 1).toordinal() + value['value'] - 2).date(), - missing=True) + output = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + value['value'] - 2) case str(): string = re.sub(r"(_|-)\d(R\d)?$", "", value['value']) try: @@ -456,12 +465,15 @@ class PydSubmission(BaseModel, extra='allow'): except ParserError as e: logger.error(f"Problem parsing date: {e}") try: - output = dict(value=parse(string.replace("-", "")).date(), missing=True) + output = parse(string.replace("-", "")).date() except Exception as e: logger.error(f"Problem with parse fallback: {e}") - return output + return value case _: raise ValueError(f"Could not get datetime from {value['value']}") + value['value'] = output.replace(tzinfo=timezone) + return value + @field_validator("submitting_lab", mode="before") @classmethod @@ -511,7 +523,7 @@ class PydSubmission(BaseModel, extra='allow'): if "pytest" in sys.modules and sub_type.replace(" ", "") == "BasicSubmission": output = "RSL-BS-Test001" else: - output = RSLNamer(filename=values.data['filepath'].__str__(), sub_type=sub_type, + output = RSLNamer(filename=values.data['filepath'].__str__(), submission_type=sub_type, data=values.data).parsed_name return dict(value=output, missing=True) @@ -653,9 +665,9 @@ class PydSubmission(BaseModel, extra='allow'): return value if isinstance(contact, tuple): contact = contact[0] - value = dict(value=f"Defaulted to: {contact}", missing=True) + value = dict(value=f"Defaulted to: {contact}", missing=False) logger.debug(f"Value after query: {value}") - return + return value else: logger.debug(f"Value after bypass check: {value}") return value @@ -665,7 +677,7 @@ class PydSubmission(BaseModel, extra='allow'): # NOTE: this could also be done with default_factory self.submission_object = BasicSubmission.find_polymorphic_subclass( polymorphic_identity=self.submission_type['value']) - self.namer = RSLNamer(self.rsl_plate_num['value'], sub_type=self.submission_type['value']) + self.namer = RSLNamer(self.rsl_plate_num['value'], submission_type=self.submission_type['value']) if run_custom: self.submission_object.custom_validation(pyd=self) @@ -777,10 +789,10 @@ class PydSubmission(BaseModel, extra='allow'): match key: case "reagents": for reagent in self.reagents: - reagent, _ = reagent.toSQL(submission=instance) + reagent = reagent.to_sql(submission=instance) case "samples": for sample in self.samples: - sample, associations, _ = sample.toSQL(submission=instance) + sample, associations, _ = sample.to_sql(submission=instance) for assoc in associations: if assoc is not None: if assoc not in instance.submission_sample_associations: @@ -791,7 +803,7 @@ class PydSubmission(BaseModel, extra='allow'): for equip in self.equipment: if equip is None: continue - equip, association = equip.toSQL(submission=instance) + equip, association = equip.to_sql(submission=instance, extraction_kit=self.extraction_kit) if association is not None: instance.submission_equipment_associations.append(association) case "tips": @@ -807,7 +819,7 @@ class PydSubmission(BaseModel, extra='allow'): instance.submission_tips_associations.append(association) else: logger.warning(f"Tips association {association} is already present in {instance}") - case item if item in instance.timestamps(): + case item if item in instance.timestamps: logger.warning(f"Incoming timestamp key: {item}, with value: {value}") if isinstance(value, date): value = datetime.combine(value, datetime.now().time()) @@ -818,7 +830,7 @@ class PydSubmission(BaseModel, extra='allow'): else: value = value instance.set_attribute(key=key, value=value) - case item if item in instance.jsons(): + case item if item in instance.jsons: try: ii = value.items() except AttributeError: @@ -989,7 +1001,7 @@ class PydContact(BaseModel): logger.debug(f"Output phone: {value}") return value - def toSQL(self) -> Tuple[Contact, Report]: + def to_sql(self) -> Tuple[Contact, Report]: """ Converts this instance into a backend.db.models.organization. Contact instance. Does not query for existing contacts. @@ -1024,7 +1036,7 @@ class PydOrganization(BaseModel): cost_centre: str contacts: List[PydContact] | None - def toSQL(self) -> Organization: + def to_sql(self) -> Organization: """ Converts this instance into a backend.db.models.organization.Organization instance. @@ -1055,7 +1067,7 @@ class PydReagentRole(BaseModel): return timedelta(days=value) return value - def toSQL(self, kit: KitType) -> ReagentRole: + def to_sql(self, kit: KitType) -> ReagentRole: """ Converts this instance into a backend.db.models.ReagentType instance @@ -1082,7 +1094,7 @@ class PydKit(BaseModel): name: str reagent_roles: List[PydReagentRole] = [] - def toSQL(self) -> Tuple[KitType, Report]: + def to_sql(self) -> Tuple[KitType, Report]: """ Converts this instance into a backend.db.models.kits.KitType instance @@ -1093,7 +1105,7 @@ class PydKit(BaseModel): instance = KitType.query(name=self.name) if instance is None: instance = KitType(name=self.name) - [item.toSQL(instance) for item in self.reagent_roles] + [item.to_sql(instance) for item in self.reagent_roles] return instance, report diff --git a/src/submissions/frontend/widgets/app.py b/src/submissions/frontend/widgets/app.py index f29e63d..96aceac 100644 --- a/src/submissions/frontend/widgets/app.py +++ b/src/submissions/frontend/widgets/app.py @@ -193,7 +193,7 @@ class App(QMainWindow): @check_authorization def edit_reagent(self, *args, **kwargs): - dlg = SearchBox(parent=self, object_type=Reagent, extras=['role']) + dlg = SearchBox(parent=self, object_type=Reagent, extras=[dict(name='Role', field="role")]) dlg.exec() @check_authorization diff --git a/src/submissions/frontend/widgets/controls_chart.py b/src/submissions/frontend/widgets/controls_chart.py index 0aa6045..14d6b5b 100644 --- a/src/submissions/frontend/widgets/controls_chart.py +++ b/src/submissions/frontend/widgets/controls_chart.py @@ -30,8 +30,7 @@ class ControlsViewer(InfoPane): self.control_sub_typer.addItems(con_sub_types) # NOTE: create custom widget to get types of analysis -- disabled by PCR control self.mode_typer = QComboBox() - mode_types = IridaControl.get_modes() - self.mode_typer.addItems(mode_types) + self.mode_typer.addItems(IridaControl.modes) # NOTE: create custom widget to get subtypes of analysis -- disabled by PCR control self.mode_sub_typer = QComboBox() self.mode_sub_typer.setEnabled(False) @@ -43,7 +42,7 @@ class ControlsViewer(InfoPane): self.layout.addWidget(self.control_sub_typer, 1, 0, 1, 4) self.layout.addWidget(self.mode_typer, 2, 0, 1, 4) self.layout.addWidget(self.mode_sub_typer, 3, 0, 1, 4) - self.archetype.get_instance_class().make_parent_buttons(parent=self) + self.archetype.instance_class.make_parent_buttons(parent=self) self.update_data() self.control_sub_typer.currentIndexChanged.connect(self.update_data) self.mode_typer.currentIndexChanged.connect(self.update_data) @@ -70,7 +69,7 @@ class ControlsViewer(InfoPane): except AttributeError: sub_types = [] # NOTE: added in allowed to have subtypes in case additions made in future. - if sub_types and self.mode.lower() in self.archetype.get_instance_class().subtyping_allowed: + if sub_types and self.mode.lower() in self.archetype.instance_class.subtyping_allowed: # NOTE: block signal that will rerun controls getter and update mode_sub_typer with QSignalBlocker(self.mode_sub_typer) as blocker: self.mode_sub_typer.addItems(sub_types) @@ -103,7 +102,7 @@ class ControlsViewer(InfoPane): chart_settings = dict(sub_type=self.con_sub_type, start_date=self.start_date, end_date=self.end_date, mode=self.mode, sub_mode=self.mode_sub_type, parent=self, months=months) - self.fig = self.archetype.get_instance_class().make_chart(chart_settings=chart_settings, parent=self, ctx=self.app.ctx) + self.fig = self.archetype.instance_class.make_chart(chart_settings=chart_settings, parent=self, ctx=self.app.ctx) self.report_obj = ChartReportMaker(df=self.fig.df, sheet_name=self.archetype.name) if issubclass(self.fig.__class__, CustomFigure): self.save_button.setEnabled(True) diff --git a/src/submissions/frontend/widgets/equipment_usage.py b/src/submissions/frontend/widgets/equipment_usage.py index 891a623..3b059d1 100644 --- a/src/submissions/frontend/widgets/equipment_usage.py +++ b/src/submissions/frontend/widgets/equipment_usage.py @@ -19,7 +19,7 @@ class EquipmentUsage(QDialog): super().__init__(parent) self.submission = submission self.setWindowTitle(f"Equipment Checklist - {submission.rsl_plate_num}") - self.used_equipment = self.submission.get_used_equipment() + self.used_equipment = self.submission.used_equipment self.kit = self.submission.extraction_kit self.opt_equipment = submission.submission_type.get_equipment() self.layout = QVBoxLayout() diff --git a/src/submissions/frontend/widgets/omni_add_edit.py b/src/submissions/frontend/widgets/omni_add_edit.py index 54b94f7..1da01d2 100644 --- a/src/submissions/frontend/widgets/omni_add_edit.py +++ b/src/submissions/frontend/widgets/omni_add_edit.py @@ -65,11 +65,11 @@ class AddEdit(QDialog): report = Report() parsed = {result[0].strip(":"): result[1] for result in [item.parse_form() for item in self.findChildren(EditProperty)] if result[0]} logger.debug(parsed) - model = self.object_type.get_pydantic_model() + model = self.object_type.pydantic_model # NOTE: Hand-off to pydantic model for validation. # NOTE: Also, why am I not just using the toSQL method here. I could write one for contacts. model = model(**parsed) - # output, result = model.toSQL() + # output, result = model.to_sql() # report.add_result(result) # if len(report.results) < 1: # report.add_result(Result(msg="Added new regeant.", icon="Information", owner=__name__)) diff --git a/src/submissions/frontend/widgets/omni_manager.py b/src/submissions/frontend/widgets/omni_manager.py index 6fab916..b4a4a27 100644 --- a/src/submissions/frontend/widgets/omni_manager.py +++ b/src/submissions/frontend/widgets/omni_manager.py @@ -188,7 +188,7 @@ class EditRelationship(QWidget): dlg = AddEdit(self, instance=instance, manager=self.parent().object_type.__name__.lower()) if dlg.exec(): new_instance = dlg.parse_form() - new_instance, result = new_instance.toSQL() + new_instance, result = new_instance.to_sql() logger.debug(f"New instance: {new_instance}") addition = getattr(self.parent().instance, self.objectName()) if isinstance(addition, InstrumentedList): @@ -213,7 +213,7 @@ class EditRelationship(QWidget): sets data in model """ # logger.debug(self.data) - self.data = DataFrame.from_records([item.to_omnigui_dict() for item in self.data]) + self.data = DataFrame.from_records([item.omnigui_dict for item in self.data]) try: self.columns_of_interest = [dict(name=item, column=self.data.columns.get_loc(item)) for item in self.extras] except (KeyError, AttributeError): diff --git a/src/submissions/frontend/widgets/omni_search.py b/src/submissions/frontend/widgets/omni_search.py index cbb0e7c..ebd0081 100644 --- a/src/submissions/frontend/widgets/omni_search.py +++ b/src/submissions/frontend/widgets/omni_search.py @@ -20,7 +20,7 @@ class SearchBox(QDialog): The full search widget. """ - def __init__(self, parent, object_type: Any, extras: List[str], returnable: bool = False, **kwargs): + def __init__(self, parent, object_type: Any, extras: List[dict], returnable: bool = False, **kwargs): super().__init__(parent) self.object_type = self.original_type = object_type self.extras = extras @@ -73,8 +73,9 @@ class SearchBox(QDialog): except AttributeError: search_fields = [] for iii, searchable in enumerate(search_fields): - widget = FieldSearch(parent=self, label=searchable, field_name=searchable) - widget.setObjectName(searchable) + widget = FieldSearch(parent=self, label=searchable['label'], field_name=searchable['field']) + # widget = FieldSearch(parent=self, label=k, field_name=v) + widget.setObjectName(searchable['field']) self.layout.addWidget(widget, 1 + iii, 0) widget.search_widget.textChanged.connect(self.update_data) self.update_data() @@ -150,14 +151,16 @@ class SearchResults(QTableView): self.extras = extras + self.object_type.searchables except AttributeError: self.extras = extras + logger.debug(f"Extras: {self.extras}") def setData(self, df: DataFrame) -> None: """ sets data in model """ + self.data = df try: - self.columns_of_interest = [dict(name=item, column=self.data.columns.get_loc(item)) for item in self.extras] + self.columns_of_interest = [dict(name=item['field'], column=self.data.columns.get_loc(item['field'])) for item in self.extras] except KeyError: self.columns_of_interest = [] try: diff --git a/src/submissions/frontend/widgets/submission_details.py b/src/submissions/frontend/widgets/submission_details.py index e34086b..82e272e 100644 --- a/src/submissions/frontend/widgets/submission_details.py +++ b/src/submissions/frontend/widgets/submission_details.py @@ -93,7 +93,7 @@ class SubmissionDetails(QDialog): base_dict = sample.to_sub_dict(full_data=True) exclude = ['submissions', 'excluded', 'colour', 'tooltip'] base_dict['excluded'] = exclude - template = sample.get_details_template() + template = sample.details_template template_path = Path(template.environment.loader.__getattribute__("searchpath")[0]) with open(template_path.joinpath("css", "styles.css"), "r") as f: css = f.read() @@ -147,7 +147,7 @@ class SubmissionDetails(QDialog): self.rsl_plate_num = submission.rsl_plate_num self.base_dict = submission.to_dict(full_data=True) # NOTE: don't want id - self.base_dict['platemap'] = submission.make_plate_map(sample_list=submission.hitpick_plate()) + self.base_dict['platemap'] = submission.make_plate_map(sample_list=submission.hitpicked) self.base_dict['excluded'] = submission.get_default_info("details_ignore") self.base_dict, self.template = submission.get_details_template(base_dict=self.base_dict) template_path = Path(self.template.environment.loader.__getattribute__("searchpath")[0]) diff --git a/src/submissions/frontend/widgets/submission_widget.py b/src/submissions/frontend/widgets/submission_widget.py index b469eff..e3778a6 100644 --- a/src/submissions/frontend/widgets/submission_widget.py +++ b/src/submissions/frontend/widgets/submission_widget.py @@ -147,14 +147,16 @@ class SubmissionFormContainer(QWidget): instance = Reagent() dlg = AddEdit(parent=self, instance=instance) if dlg.exec(): - reagent, result = dlg.parse_form() + reagent = dlg.parse_form() reagent.missing = False - logger.debug(f"Reagent: {reagent}, result: {result}") - report.add_result(result) + # logger.debug(f"Reagent: {reagent}, result: {result}") + # report.add_result(result) # NOTE: send reagent to db - sqlobj, result = reagent.toSQL() + sqlobj = reagent.to_sql() sqlobj.save() - report.add_result(result) + logger.debug(f"Reagent added!") + report.add_result(Result(owner=__name__, code=0, msg="New reagent created.", status="Information")) + # report.add_result(result) return reagent, report @report_result @@ -184,10 +186,10 @@ class SubmissionFormContainer(QWidget): # NOTE: create reagent object reagent = PydReagent(ctx=self.app.ctx, **info, missing=False) # NOTE: send reagent to db - sqlobj, result = reagent.toSQL() + sqlobj = reagent.to_sql() sqlobj.save() - report.add_result(result) - return reagent, report + # report.add_result(result) + return reagent class SubmissionFormWidget(QWidget): @@ -201,7 +203,7 @@ class SubmissionFormWidget(QWidget): self.pyd = submission self.missing_info = [] self.submission_type = SubmissionType.query(name=self.pyd.submission_type['value']) - st = self.submission_type.get_submission_class() + st = self.submission_type.submission_class defaults = st.get_default_info("form_recover", "form_ignore", submission_type=self.pyd.submission_type['value']) self.recover = defaults['form_recover'] self.ignore = defaults['form_ignore'] @@ -443,6 +445,9 @@ class SubmissionFormWidget(QWidget): reagent = widget.parse_form() if reagent is not None: reagents.append(reagent) + else: + report.add_result(Result(msg="Failed integrity check", status="Critical")) + return report case self.InfoItem(): field, value = widget.parse_form() if field is not None: @@ -523,7 +528,7 @@ class SubmissionFormWidget(QWidget): if isinstance(submission_type, str): submission_type = SubmissionType.query(name=submission_type) if sub_obj is None: - sub_obj = submission_type.get_submission_class() + sub_obj = submission_type.submission_class try: value = value['value'] except (TypeError, KeyError): @@ -585,7 +590,7 @@ class SubmissionFormWidget(QWidget): add_widget.addItems(cats) add_widget.setToolTip("Enter submission category or select from list.") case _: - if key in sub_obj.timestamps(): + if key in sub_obj.timestamps: add_widget = MyQDateEdit(calendarPopup=True, scrollWidget=parent) # NOTE: sets submitted date based on date found in excel sheet try: @@ -696,7 +701,7 @@ class SubmissionFormWidget(QWidget): if not self.lot.isEnabled(): return None, report lot = self.lot.currentText() - wanted_reagent, new = Reagent.query_or_create(lot=lot, role=self.reagent.role) + wanted_reagent, new = Reagent.query_or_create(lot=lot, role=self.reagent.role, expiry=self.reagent.expiry) # NOTE: if reagent doesn't exist in database, offer to add it (uses App.add_reagent) logger.debug(f"Wanted reagent: {wanted_reagent}, New: {new}") # if wanted_reagent is None: @@ -705,18 +710,13 @@ class SubmissionFormWidget(QWidget): message=f"Couldn't find reagent type {self.reagent.role}: {lot} in the database.\n\nWould you like to add it?") if dlg.exec(): - # wanted_reagent = self.parent().parent().add_reagent(reagent_lot=lot, - # reagent_role=self.reagent.role, - # expiry=self.reagent.expiry, - # name=self.reagent.name, - # kit=self.extraction_kit - # ) wanted_reagent = self.parent().parent().new_add_reagent(instance=wanted_reagent) - + logger.debug(f"Reagent added!") + report.add_result(Result(owner=__name__, code=0, msg="New reagent created.", status="Information")) return wanted_reagent, report else: # NOTE: In this case we will have an empty reagent and the submission will fail kit integrity check - report.add_result(Result(msg="Failed integrity check", status="Critical")) + return None, report else: # NOTE: Since this now gets passed in directly from the parser -> pyd -> form and the parser gets the name diff --git a/src/submissions/tools/__init__.py b/src/submissions/tools/__init__.py index 656e19a..d80f472 100644 --- a/src/submissions/tools/__init__.py +++ b/src/submissions/tools/__init__.py @@ -53,7 +53,6 @@ main_form_style = ''' QComboBox:!editable, QDateEdit { background-color:light gray; } - ''' page_size = 250