diff --git a/CHANGELOG.md b/CHANGELOG.md index e2d952e..df0b156 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 202311.01 + +- Kit integrity is now checked before creation of sql object to improve reagent type lookups. + ## 202310.03 - Better flexibility with parsers pulling methods from database objects. diff --git a/TODO.md b/TODO.md index 9b26cee..29e5a49 100644 --- a/TODO.md +++ b/TODO.md @@ -1,10 +1,14 @@ -- [ ] Make the kit verifier make more sense. -- [ ] Slim down the Import and Submit functions in main_window_functions. +- [ ] Update artic submission type database entry to +- [ ] Document code +- [x] Rewrite tests... again. +- [x] Have InfoItem change status self.missing to True if value changed. +- [x] Make the kit verifier make more sense. +- [x] Slim down the Import and Submit functions in main_window_functions. - [x] Create custom store methods for submission, reagent and sample. - [x] Make pydantic models for other things that use constructors. - [x] Move backend.db.functions.constructor functions into Pydantic models. - This will allow for better data validation. - - Parser -> Pydantic(validation) -> Form(user input) -> Pydantic(validation) -> SQL + - Parser(client input) -> Pydantic(validation) -> Form(user input) -> Pydantic(validation) -> SQL - [x] Rebuild RSLNamer and fix circular imports - Should be used when coming in to parser and when leaving form. NO OTHER PLACES. - [x] Change 'check_is_power_user' to decorator. diff --git a/alembic/versions/8a5bc2924ef9_adding_artic_technician_to_artic.py b/alembic/versions/8a5bc2924ef9_adding_artic_technician_to_artic.py new file mode 100644 index 0000000..ef68d31 --- /dev/null +++ b/alembic/versions/8a5bc2924ef9_adding_artic_technician_to_artic.py @@ -0,0 +1,32 @@ +"""adding artic_technician to Artic + +Revision ID: 8a5bc2924ef9 +Revises: b95478ffb4a3 +Create Date: 2023-10-31 13:59:47.746122 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '8a5bc2924ef9' +down_revision = 'b95478ffb4a3' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('_submissions', schema=None) as batch_op: + batch_op.add_column(sa.Column('artic_technician', sa.String(length=64), nullable=True)) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('_submissions', schema=None) as batch_op: + batch_op.drop_column('artic_technician') + + # ### end Alembic commands ### diff --git a/source/submissions.backend.validators.rst b/source/submissions.backend.validators.rst new file mode 100644 index 0000000..b37f176 --- /dev/null +++ b/source/submissions.backend.validators.rst @@ -0,0 +1,21 @@ +submissions.backend.validators package +====================================== + +Submodules +---------- + +submissions.backend.validators.pydant module +-------------------------------------------- + +.. automodule:: submissions.backend.validators.pydant + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: submissions.backend.validators + :members: + :undoc-members: + :show-inheritance: diff --git a/src/submissions/__init__.py b/src/submissions/__init__.py index c87c8ad..661356f 100644 --- a/src/submissions/__init__.py +++ b/src/submissions/__init__.py @@ -4,7 +4,7 @@ from pathlib import Path # Version of the realpython-reader package __project__ = "submissions" -__version__ = "202310.4b" +__version__ = "202311.1b" __author__ = {"name":"Landon Wark", "email":"Landon.Wark@phac-aspc.gc.ca"} __copyright__ = "2022-2023, Government of Canada" diff --git a/src/submissions/backend/db/__init__.py b/src/submissions/backend/db/__init__.py index 5c4e6d5..e4b232c 100644 --- a/src/submissions/backend/db/__init__.py +++ b/src/submissions/backend/db/__init__.py @@ -1,4 +1,4 @@ ''' All database related operations. ''' -from .functions import * \ No newline at end of file +# from .functions import * \ No newline at end of file diff --git a/src/submissions/backend/db/functions/constructions.py b/src/submissions/backend/db/functions/constructions.py deleted file mode 100644 index b9ade0e..0000000 --- a/src/submissions/backend/db/functions/constructions.py +++ /dev/null @@ -1,280 +0,0 @@ -''' -Used to construct models from input dictionaries. -''' - -from tools import Settings, check_regex_match, check_authorization, massage_common_reagents -from .. import models -from .lookups import * -import logging -from datetime import date, timedelta -from dateutil.parser import parse -from typing import Tuple -from sqlalchemy.exc import IntegrityError, SAWarning -from . import store_object -from backend.validators import RSLNamer - -logger = logging.getLogger(f"submissions.{__name__}") - -# def construct_reagent(ctx:Settings, info_dict:dict) -> models.Reagent: -# """ -# Construct reagent object from dictionary -# NOTE: Depreciated in favour of Pydantic model .toSQL method - -# Args: -# ctx (Settings): settings object passed down from gui -# info_dict (dict): dictionary to be converted - -# Returns: -# models.Reagent: Constructed reagent object -# """ -# reagent = models.Reagent() -# for item in info_dict: -# logger.debug(f"Reagent info item for {item}: {info_dict[item]}") -# # set fields based on keys in dictionary -# match item: -# case "lot": -# reagent.lot = info_dict[item].upper() -# case "expiry": -# if isinstance(info_dict[item], date): -# reagent.expiry = info_dict[item] -# else: -# reagent.expiry = parse(info_dict[item]).date() -# case "type": -# reagent_type = lookup_reagent_types(ctx=ctx, name=info_dict[item]) -# if reagent_type != None: -# reagent.type.append(reagent_type) -# case "name": -# if item == None: -# reagent.name = reagent.type.name -# else: -# reagent.name = info_dict[item] -# # add end-of-life extension from reagent type to expiry date -# # NOTE: this will now be done only in the reporting phase to account for potential changes in end-of-life extensions -# return reagent - -# def construct_submission_info(ctx:Settings, info_dict:dict) -> Tuple[models.BasicSubmission, dict]: -# """ -# Construct submission object from dictionary pulled from gui form -# NOTE: Depreciated in favour of Pydantic model .toSQL method - -# Args: -# ctx (Settings): settings object passed down from gui -# info_dict (dict): dictionary to be transformed - -# Returns: -# models.BasicSubmission: Constructed submission object -# """ -# # convert submission type into model name -# # model = get_polymorphic_subclass(polymorphic_identity=info_dict['submission_type']) -# model = models.BasicSubmission.find_polymorphic_subclass(polymorphic_identity=info_dict['submission_type']) -# logger.debug(f"We've got the model: {type(model)}") -# # Ensure an rsl plate number exists for the plate -# if not check_regex_match("^RSL", info_dict["rsl_plate_num"]): -# instance = None -# msg = "A proper RSL plate number is required." -# return instance, {'code': 2, 'message': "A proper RSL plate number is required."} -# else: -# # # enforce conventions on the rsl plate number from the form -# # # info_dict['rsl_plate_num'] = RSLNamer(ctx=ctx, instr=info_dict["rsl_plate_num"]).parsed_name -# info_dict['rsl_plate_num'] = RSLNamer(ctx=ctx, instr=info_dict["rsl_plate_num"], sub_type=info_dict['submission_type']).parsed_name -# # check database for existing object -# instance = lookup_submissions(ctx=ctx, rsl_number=info_dict['rsl_plate_num']) -# # get model based on submission type converted above -# # logger.debug(f"Looking at models for submission type: {query}") - -# # if query return nothing, ie doesn't already exist in db -# if instance == None: -# instance = model() -# logger.debug(f"Submission doesn't exist yet, creating new instance: {instance}") -# msg = None -# code = 0 -# else: -# code = 1 -# msg = "This submission already exists.\nWould you like to overwrite?" -# for item in info_dict: -# value = info_dict[item] -# logger.debug(f"Setting {item} to {value}") -# # set fields based on keys in dictionary -# match item: -# case "extraction_kit": -# logger.debug(f"Looking up kit {value}") -# field_value = lookup_kit_types(ctx=ctx, name=value) -# logger.debug(f"Got {field_value} for kit {value}") -# case "submitting_lab": -# logger.debug(f"Looking up organization: {value}") -# field_value = lookup_organizations(ctx=ctx, name=value) -# logger.debug(f"Got {field_value} for organization {value}") -# case "submitter_plate_num": -# logger.debug(f"Submitter plate id: {value}") -# field_value = value -# case "samples": -# instance = construct_samples(ctx=ctx, instance=instance, samples=value) -# continue -# case "submission_type": -# field_value = lookup_submission_type(ctx=ctx, name=value) -# case _: -# field_value = value -# # insert into field -# try: -# setattr(instance, item, field_value) -# except AttributeError: -# logger.debug(f"Could not set attribute: {item} to {info_dict[item]}") -# continue -# except KeyError: -# continue -# # calculate cost of the run: immutable cost + mutable times number of columns -# # This is now attached to submission upon creation to preserve at-run costs incase of cost increase in the future. -# try: -# logger.debug(f"Calculating costs for procedure...") -# instance.calculate_base_cost() -# except (TypeError, AttributeError) as e: -# logger.debug(f"Looks like that kit doesn't have cost breakdown yet due to: {e}, using full plate cost.") -# instance.run_cost = instance.extraction_kit.cost_per_run -# logger.debug(f"Calculated base run cost of: {instance.run_cost}") -# # Apply any discounts that are applicable for client and kit. -# try: -# logger.debug("Checking and applying discounts...") -# discounts = [item.amount for item in lookup_discounts(ctx=ctx, kit_type=instance.extraction_kit, organization=instance.submitting_lab)] -# logger.debug(f"We got discounts: {discounts}") -# if len(discounts) > 0: -# discounts = sum(discounts) -# instance.run_cost = instance.run_cost - discounts -# except Exception as e: -# logger.error(f"An unknown exception occurred when calculating discounts: {e}") -# # We need to make sure there's a proper rsl plate number -# logger.debug(f"We've got a total cost of {instance.run_cost}") -# try: -# logger.debug(f"Constructed instance: {instance.to_string()}") -# except AttributeError as e: -# logger.debug(f"Something went wrong constructing instance {info_dict['rsl_plate_num']}: {e}") -# logger.debug(f"Constructed submissions message: {msg}") -# return instance, {'code':code, 'message':msg} - -# def construct_samples(ctx:Settings, instance:models.BasicSubmission, samples:List[dict]) -> models.BasicSubmission: -# """ -# constructs sample objects and adds to submission -# NOTE: Depreciated in favour of Pydantic model .toSQL method - -# Args: -# ctx (Settings): settings passed down from gui -# instance (models.BasicSubmission): Submission samples scraped from. -# samples (List[dict]): List of parsed samples - -# Returns: -# models.BasicSubmission: Updated submission object. -# """ -# for sample in samples: -# sample_instance = lookup_samples(ctx=ctx, submitter_id=str(sample['sample'].submitter_id)) -# if sample_instance == None: -# sample_instance = sample['sample'] -# else: -# logger.warning(f"Sample {sample} already exists, creating association.") -# logger.debug(f"Adding {sample_instance.__dict__}") -# if sample_instance in instance.samples: -# logger.error(f"Looks like there's a duplicate sample on this plate: {sample_instance.submitter_id}!") -# continue -# try: -# with ctx.database_session.no_autoflush: -# try: -# sample_query = sample_instance.sample_type.replace('Sample', '').strip() -# logger.debug(f"Here is the sample instance type: {sample_instance}") -# try: -# assoc = getattr(models, f"{sample_query}Association") -# except AttributeError as e: -# logger.error(f"Couldn't get type specific association using {sample_instance.sample_type.replace('Sample', '').strip()}. Getting generic.") -# assoc = models.SubmissionSampleAssociation -# assoc = assoc(submission=instance, sample=sample_instance, row=sample['row'], column=sample['column']) -# instance.submission_sample_associations.append(assoc) -# except IntegrityError: -# logger.error(f"Hit integrity error for: {sample}") -# continue -# except SAWarning: -# logger.error(f"Looks like the association already exists for submission: {instance} and sample: {sample_instance}") -# continue -# except IntegrityError as e: -# logger.critical(e) -# continue -# return instance - -# @check_authorization -# def construct_kit_from_yaml(ctx:Settings, kit_dict:dict) -> dict: -# """ -# Create and store a new kit in the database based on a .yml file -# TODO: split into create and store functions - -# Args: -# ctx (Settings): Context object passed down from frontend -# kit_dict (dict): Experiment dictionary created from yaml file - -# Returns: -# dict: a dictionary containing results of db addition -# """ -# # from tools import check_is_power_user, massage_common_reagents -# # Don't want just anyone adding kits -# # if not check_is_power_user(ctx=ctx): -# # logger.debug(f"{getuser()} does not have permission to add kits.") -# # return {'code':1, 'message':"This user does not have permission to add kits.", "status":"warning"} -# submission_type = lookup_submission_type(ctx=ctx, name=kit_dict['used_for']) -# logger.debug(f"Looked up submission type: {kit_dict['used_for']} and got {submission_type}") -# kit = models.KitType(name=kit_dict["kit_name"]) -# kt_st_assoc = models.SubmissionTypeKitTypeAssociation(kit_type=kit, submission_type=submission_type) -# for k,v in kit_dict.items(): -# if k not in ["reagent_types", "kit_name", "used_for"]: -# kt_st_assoc.set_attrib(k, v) -# kit.kit_submissiontype_associations.append(kt_st_assoc) -# # A kit contains multiple reagent types. -# for r in kit_dict['reagent_types']: -# logger.debug(f"Constructing reagent type: {r}") -# rtname = massage_common_reagents(r['rtname']) -# look_up = lookup_reagent_types(name=rtname) -# if look_up == None: -# rt = models.ReagentType(name=rtname.strip(), eol_ext=timedelta(30*r['eol'])) -# else: -# rt = look_up -# uses = {kit_dict['used_for']:{k:v for k,v in r.items() if k not in ['eol']}} -# assoc = models.KitTypeReagentTypeAssociation(kit_type=kit, reagent_type=rt, uses=uses) -# # ctx.database_session.add(rt) -# store_object(ctx=ctx, object=rt) -# kit.kit_reagenttype_associations.append(assoc) -# logger.debug(f"Kit construction reagent type: {rt.__dict__}") -# logger.debug(f"Kit construction kit: {kit.__dict__}") -# store_object(ctx=ctx, object=kit) -# return {'code':0, 'message':'Kit has been added', 'status': 'information'} - -# @check_authorization -# def construct_org_from_yaml(ctx:Settings, org:dict) -> dict: -# """ -# Create and store a new organization based on a .yml file - -# Args: -# ctx (Settings): Context object passed down from frontend -# org (dict): Dictionary containing organization info. - -# Returns: -# dict: dictionary containing results of db addition -# """ -# # from tools import check_is_power_user -# # # Don't want just anyone adding in clients -# # if not check_is_power_user(ctx=ctx): -# # logger.debug(f"{getuser()} does not have permission to add kits.") -# # return {'code':1, 'message':"This user does not have permission to add organizations."} -# # the yml can contain multiple clients -# for client in org: -# cli_org = models.Organization(name=client.replace(" ", "_").lower(), cost_centre=org[client]['cost centre']) -# # a client can contain multiple contacts -# for contact in org[client]['contacts']: -# cont_name = list(contact.keys())[0] -# # check if contact already exists -# look_up = ctx.database_session.query(models.Contact).filter(models.Contact.name==cont_name).first() -# if look_up == None: -# cli_cont = models.Contact(name=cont_name, phone=contact[cont_name]['phone'], email=contact[cont_name]['email'], organization=[cli_org]) -# else: -# cli_cont = look_up -# cli_cont.organization.append(cli_org) -# ctx.database_session.add(cli_cont) -# logger.debug(f"Client creation contact: {cli_cont.__dict__}") -# logger.debug(f"Client creation client: {cli_org.__dict__}") -# ctx.database_session.add(cli_org) -# ctx.database_session.commit() -# return {"code":0, "message":"Organization has been added."} - diff --git a/src/submissions/backend/db/functions/lookups.py b/src/submissions/backend/db/functions/lookups.py index ff247fd..a42be60 100644 --- a/src/submissions/backend/db/functions/lookups.py +++ b/src/submissions/backend/db/functions/lookups.py @@ -141,7 +141,10 @@ def lookup_reagent_types(ctx:Settings, # logger.debug(f"Reagent reagent types: {reagent._sa_instance_state}") result = list(set(kit_type.reagent_types).intersection(reagent.type)) logger.debug(f"Result: {result}") - return result[0] + try: + return result[0] + except IndexError: + return result match name: case str(): logger.debug(f"Looking up reagent type by name: {name}") @@ -249,7 +252,7 @@ def lookup_submissions(ctx:Settings, if chronologic: # query.order_by(models.BasicSubmission.submitted_date) query.order_by(model.submitted_date) - logger.debug(f"At the end of the search, the query gets: {query.all()}") + # logger.debug(f"At the end of the search, the query gets: {query.all()}") return query_return(query=query, limit=limit) def lookup_submission_type(ctx:Settings, diff --git a/src/submissions/backend/db/functions/misc.py b/src/submissions/backend/db/functions/misc.py index 1bae27f..1d78777 100644 --- a/src/submissions/backend/db/functions/misc.py +++ b/src/submissions/backend/db/functions/misc.py @@ -1,6 +1,7 @@ ''' Contains convenience functions for using database ''' +import sys from tools import Settings from .lookups import * import pandas as pd @@ -13,6 +14,8 @@ from sqlalchemy.exc import OperationalError as AlcOperationalError, IntegrityErr from sqlite3 import OperationalError as SQLOperationalError, IntegrityError as SQLIntegrityError from pprint import pformat import logging +from backend.validators import pydant + logger = logging.getLogger(f"submissions.{__name__}") @@ -172,7 +175,7 @@ def update_ww_sample(ctx:Settings, sample_obj:dict) -> dict|None: result = store_object(ctx=ctx, object=assoc) return result -def check_kit_integrity(sub:models.BasicSubmission|models.KitType, reagenttypes:list|None=None) -> dict|None: +def check_kit_integrity(ctx:Settings, sub:models.BasicSubmission|models.KitType|pydant.PydSubmission, reagenttypes:list=[]) -> dict|None: """ Ensures all reagents expected in kit are listed in Submission @@ -185,20 +188,30 @@ def check_kit_integrity(sub:models.BasicSubmission|models.KitType, reagenttypes: """ logger.debug(type(sub)) # What type is sub? - reagenttypes = [] + # reagenttypes = [] match sub: + case pydant.PydSubmission(): + ext_kit = lookup_kit_types(ctx=ctx, name=sub.extraction_kit['value']) + ext_kit_rtypes = [item.name for item in ext_kit.get_reagents(required=True, submission_type=sub.submission_type['value'])] + reagenttypes = [item.type for item in sub.reagents] case models.BasicSubmission(): # Get all required reagent types for this kit. ext_kit_rtypes = [item.name for item in sub.extraction_kit.get_reagents(required=True, submission_type=sub.submission_type_name)] # Overwrite function parameter reagenttypes for reagent in sub.reagents: + logger.debug(f"For kit integrity, looking up reagent: {reagent}") try: - rt = list(set(reagent.type).intersection(sub.extraction_kit.reagent_types))[0].name + # rt = list(set(reagent.type).intersection(sub.extraction_kit.reagent_types))[0].name + rt = lookup_reagent_types(ctx=ctx, kit_type=sub.extraction_kit, reagent=reagent) logger.debug(f"Got reagent type: {rt}") - reagenttypes.append(rt) + if isinstance(rt, models.ReagentType): + reagenttypes.append(rt.name) except AttributeError as e: logger.error(f"Problem parsing reagents: {[f'{reagent.lot}, {reagent.type}' for reagent in sub.reagents]}") reagenttypes.append(reagent.type[0].name) + except IndexError: + logger.error(f"No intersection of {reagent} type {reagent.type} and {sub.extraction_kit.reagent_types}") + raise ValueError(f"No intersection of {reagent} type {reagent.type} and {sub.extraction_kit.reagent_types}") case models.KitType(): ext_kit_rtypes = [item.name for item in sub.get_reagents(required=True)] case _: diff --git a/src/submissions/backend/db/models/kits.py b/src/submissions/backend/db/models/kits.py index 01c9465..10e62df 100644 --- a/src/submissions/backend/db/models/kits.py +++ b/src/submissions/backend/db/models/kits.py @@ -248,7 +248,7 @@ class Reagent(Base): "expiry": place_holder.strftime("%Y-%m-%d") } - def to_reagent_dict(self, extraction_kit:KitType=None) -> dict: + def to_reagent_dict(self, extraction_kit:KitType|str=None) -> dict: """ Returns basic reagent dictionary. @@ -314,6 +314,7 @@ class SubmissionType(Base): name = Column(String(128), unique=True) #: name of submission type info_map = Column(JSON) #: Where basic information is found in the excel workbook corresponding to this type. instances = relationship("BasicSubmission", backref="submission_type") + # regex = Column(String(512)) submissiontype_kit_associations = relationship( "SubmissionTypeKitTypeAssociation", @@ -325,6 +326,7 @@ class SubmissionType(Base): def __repr__(self) -> str: return f"" + class SubmissionTypeKitTypeAssociation(Base): """ diff --git a/src/submissions/backend/db/models/submissions.py b/src/submissions/backend/db/models/submissions.py index 1b7a555..aec5071 100644 --- a/src/submissions/backend/db/models/submissions.py +++ b/src/submissions/backend/db/models/submissions.py @@ -47,6 +47,7 @@ class BasicSubmission(Base): reagents = relationship("Reagent", back_populates="submissions", secondary=reagents_submissions) #: relationship to reagents reagents_id = Column(String, ForeignKey("_reagents.id", ondelete="SET NULL", name="fk_BS_reagents_id")) #: id of used reagents extraction_info = Column(JSON) #: unstructured output from the extraction table logger. + pcr_info = Column(JSON) #: unstructured output from pcr table logger or user(Artic) run_cost = Column(FLOAT(2)) #: total cost of running the plate. Set from constant and mutable kit costs at time of creation. uploaded_by = Column(String(32)) #: user name of person who submitted the submission to the database. comment = Column(JSON) @@ -211,12 +212,12 @@ class BasicSubmission(Base): Calculate the number of columns in this submission Returns: - int: largest column number + int: Number of unique columns. """ logger.debug(f"Here's the samples: {self.samples}") - columns = [assoc.column for assoc in self.submission_sample_associations] + columns = set([assoc.column for assoc in self.submission_sample_associations]) logger.debug(f"Here are the columns for {self.rsl_plate_num}: {columns}") - return max(columns) + return len(columns) def hitpick_plate(self, plate_number:int|None=None) -> list: """ @@ -281,7 +282,7 @@ class BasicSubmission(Base): Returns: dict: Updated sample dictionary """ - logger.debug(f"Called {cls.__name__} sample parser") + # logger.debug(f"Called {cls.__name__} sample parser") return input_dict @classmethod @@ -461,7 +462,7 @@ class Wastewater(BasicSubmission): """ derivative submission type from BasicSubmission """ - pcr_info = Column(JSON) + # pcr_info = Column(JSON) ext_technician = Column(String(64)) pcr_technician = Column(String(64)) __mapper_args__ = {"polymorphic_identity": "Wastewater", "polymorphic_load": "inline"} @@ -570,13 +571,16 @@ class Wastewater(BasicSubmission): @classmethod def get_regex(cls): - return "(?PRSL(?:-|_)?WW(?:-|_)?20\d{2}-?\d{2}-?\d{2}(?:(_|-)\d?(\D|$)R?\d?)?)" + # return "(?PRSL(?:-|_)?WW(?:-|_)?20\d{2}-?\d{2}-?\d{2}(?:(_|-)\d?(\D|$)R?\d?)?)" + # return "(?PRSL(?:-|_)?WW(?:-|_)?20\d{2}-?\d{2}-?\d{2}(?:(_|-)\d?([^_|\D]|$)R?\d?)?)" + return "(?PRSL(?:-|_)?WW(?:-|_)?20\d{2}-?\d{2}-?\d{2}(?:(_|-)?\d?([^_0123456789]|$)R?\d?)?)" class WastewaterArtic(BasicSubmission): """ derivative submission type for artic wastewater """ __mapper_args__ = {"polymorphic_identity": "Wastewater Artic", "polymorphic_load": "inline"} + artic_technician = Column(String(64)) def calculate_base_cost(self): """ @@ -752,7 +756,7 @@ class BasicSample(Base): @classmethod def parse_sample(cls, input_dict:dict) -> dict: - logger.debug(f"Called {cls.__name__} sample parser") + # logger.debug(f"Called {cls.__name__} sample parser") return input_dict class WastewaterSample(BasicSample): diff --git a/src/submissions/backend/excel/parser.py b/src/submissions/backend/excel/parser.py index 99ce76a..ec8fe05 100644 --- a/src/submissions/backend/excel/parser.py +++ b/src/submissions/backend/excel/parser.py @@ -7,7 +7,8 @@ from typing import List import pandas as pd import numpy as np from pathlib import Path -from backend.db import models, lookup_kit_types, lookup_submission_type, lookup_samples +from backend.db import models +from backend.db.functions import lookup_kit_types, lookup_submission_type, lookup_samples from backend.validators import PydSubmission, PydReagent, RSLNamer, PydSample import logging from collections import OrderedDict @@ -32,7 +33,7 @@ class SheetParser(object): filepath (Path | None, optional): file path to excel sheet. Defaults to None. """ self.ctx = ctx - logger.debug(f"Parsing {filepath.__str__()}") + logger.debug(f"\n\nParsing {filepath.__str__()}\n\n") match filepath: case Path(): self.filepath = filepath @@ -48,7 +49,7 @@ class SheetParser(object): raise FileNotFoundError(f"Couldn't parse file {self.filepath}") self.sub = OrderedDict() # make decision about type of sample we have - self.sub['submission_type'] = dict(value=RSLNamer.retrieve_submission_type(ctx=self.ctx, instr=self.filepath), parsed=False) + self.sub['submission_type'] = dict(value=RSLNamer.retrieve_submission_type(ctx=self.ctx, instr=self.filepath), missing=True) # # grab the info map from the submission type in database self.parse_info() self.import_kit_validation_check() @@ -98,12 +99,12 @@ class SheetParser(object): if not check_not_nan(self.sub['extraction_kit']['value']): dlg = KitSelector(ctx=self.ctx, title="Kit Needed", message="At minimum a kit is needed. Please select one.") if dlg.exec(): - self.sub['extraction_kit'] = dict(value=dlg.getValues(), parsed=False) + self.sub['extraction_kit'] = dict(value=dlg.getValues(), missing=True) else: raise ValueError("Extraction kit needed.") else: if isinstance(self.sub['extraction_kit'], str): - self.sub['extraction_kit'] = dict(value=self.sub['extraction_kit'], parsed=False) + self.sub['extraction_kit'] = dict(value=self.sub['extraction_kit'], missing=True) def import_reagent_validation_check(self): """ @@ -130,6 +131,7 @@ class SheetParser(object): class InfoParser(object): def __init__(self, ctx:Settings, xl:pd.ExcelFile, submission_type:str): + logger.debug(f"\n\nHello from InfoParser!") self.ctx = ctx self.map = self.fetch_submission_info_map(submission_type=submission_type) self.xl = xl @@ -147,7 +149,7 @@ class InfoParser(object): dict: Location map of all info for this submission type """ if isinstance(submission_type, str): - submission_type = dict(value=submission_type, parsed=False) + submission_type = dict(value=submission_type, missing=True) logger.debug(f"Looking up submission type: {submission_type['value']}") submission_type = lookup_submission_type(ctx=self.ctx, name=submission_type['value']) info_map = submission_type.info_map @@ -168,7 +170,7 @@ class InfoParser(object): relevant = {} for k, v in self.map.items(): if isinstance(v, str): - dicto[k] = dict(value=v, parsed=True) + dicto[k] = dict(value=v, missing=False) continue if k == "samples": continue @@ -183,16 +185,16 @@ class InfoParser(object): if check_not_nan(value): if value != "None": try: - dicto[item] = dict(value=value, parsed=True) + dicto[item] = dict(value=value, missing=False) except (KeyError, IndexError): continue else: try: - dicto[item] = dict(value=value, parsed=False) + dicto[item] = dict(value=value, missing=True) except (KeyError, IndexError): continue else: - dicto[item] = dict(value=convert_nans_to_nones(value), parsed=False) + dicto[item] = dict(value=convert_nans_to_nones(value), missing=True) try: check = dicto['submission_category'] not in ["", None] except KeyError: @@ -202,6 +204,7 @@ class InfoParser(object): class ReagentParser(object): def __init__(self, ctx:Settings, xl:pd.ExcelFile, submission_type:str, extraction_kit:str): + logger.debug("\n\nHello from ReagentParser!\n\n") self.ctx = ctx self.map = self.fetch_kit_info_map(extraction_kit=extraction_kit, submission_type=submission_type) self.xl = xl @@ -232,18 +235,18 @@ class ReagentParser(object): lot = df.iat[relevant[item]['lot']['row']-1, relevant[item]['lot']['column']-1] expiry = df.iat[relevant[item]['expiry']['row']-1, relevant[item]['expiry']['column']-1] except (KeyError, IndexError): - listo.append(PydReagent(ctx=self.ctx, type=item.strip(), lot=None, exp=None, name=None, parsed=False)) + listo.append(PydReagent(ctx=self.ctx, type=item.strip(), lot=None, expiry=None, name=None, missing=True)) continue # If the cell is blank tell the PydReagent if check_not_nan(lot): - parsed = True + missing = False else: - parsed = False + missing = True # logger.debug(f"Got lot for {item}-{name}: {lot} as {type(lot)}") lot = str(lot) logger.debug(f"Going into pydantic: name: {name}, lot: {lot}, expiry: {expiry}, type: {item.strip()}") - listo.append(PydReagent(ctx=self.ctx, type=item.strip(), lot=lot, expiry=expiry, name=name, parsed=parsed)) - logger.debug(f"Returning listo: {listo}") + listo.append(PydReagent(ctx=self.ctx, type=item.strip(), lot=lot, expiry=expiry, name=name, missing=missing)) + # logger.debug(f"Returning listo: {listo}") return listo class SampleParser(object): @@ -260,6 +263,7 @@ class SampleParser(object): df (pd.DataFrame): input sample dataframe elution_map (pd.DataFrame | None, optional): optional map of elution plate. Defaults to None. """ + logger.debug("\n\nHello from SampleParser!") self.samples = [] self.ctx = ctx self.xl = xl @@ -310,6 +314,7 @@ class SampleParser(object): # custom_mapper = get_polymorphic_subclass(models.BasicSubmission, self.submission_type) custom_mapper = models.BasicSubmission.find_polymorphic_subclass(polymorphic_identity=self.submission_type) df = custom_mapper.custom_platemap(self.xl, df) + logger.debug(f"Custom platemap:\n{df}") return df def construct_lookup_table(self, lookup_table_location:dict) -> pd.DataFrame: @@ -369,10 +374,10 @@ class SampleParser(object): for sample in self.samples: # addition = self.lookup_table[self.lookup_table.isin([sample['submitter_id']]).any(axis=1)].squeeze().to_dict() addition = self.lookup_table[self.lookup_table.isin([sample['submitter_id']]).any(axis=1)].squeeze() - logger.debug(addition) + # logger.debug(addition) if isinstance(addition, pd.DataFrame) and not addition.empty: addition = addition.iloc[0] - logger.debug(f"Lookuptable info: {addition.to_dict()}") + # logger.debug(f"Lookuptable info: {addition.to_dict()}") for k,v in addition.to_dict().items(): # logger.debug(f"Checking {k} in lookup table.") if check_not_nan(k) and isinstance(k, str): @@ -395,7 +400,7 @@ class SampleParser(object): self.lookup_table.loc[self.lookup_table['Well']==addition['Well']] = np.nan except (ValueError, KeyError): pass - logger.debug(f"Output sample dict: {sample}") + # logger.debug(f"Output sample dict: {sample}") logger.debug(f"Final lookup_table: \n\n {self.lookup_table}") def parse_samples(self, generate:bool=True) -> List[dict]|List[models.BasicSample]: @@ -432,11 +437,7 @@ class SampleParser(object): translated_dict['sample_type'] = f"{self.submission_type} Sample" translated_dict = self.custom_sub_parser(translated_dict) translated_dict = self.custom_sample_parser(translated_dict) - logger.debug(f"Here is the output of the custom parser: \n\n{translated_dict}\n\n") - # if generate: - # new_samples.append(self.generate_sample_object(translated_dict)) - # else: - # new_samples.append(translated_dict) + # logger.debug(f"Here is the output of the custom parser:\n{translated_dict}") new_samples.append(PydSample(**translated_dict)) return result, new_samples diff --git a/src/submissions/backend/validators/__init__.py b/src/submissions/backend/validators/__init__.py index df96626..0fdd715 100644 --- a/src/submissions/backend/validators/__init__.py +++ b/src/submissions/backend/validators/__init__.py @@ -18,7 +18,7 @@ class RSLNamer(object): if self.submission_type == None: self.submission_type = self.retrieve_submission_type(ctx=self.ctx, instr=instr) - print(self.submission_type) + logger.debug(f"got submission type: {self.submission_type}") if self.submission_type != None: enforcer = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=self.submission_type) self.parsed_name = self.retrieve_rsl_number(instr=instr, regex=enforcer.get_regex()) @@ -67,10 +67,12 @@ class RSLNamer(object): Args: in_str (str): string to be parsed """ + logger.debug(f"Input string to be parsed: {instr}") if regex == None: regex = BasicSubmission.construct_regex() else: regex = re.compile(rf'{regex}', re.IGNORECASE | re.VERBOSE) + logger.debug(f"Using regex: {regex}") match instr: case Path(): m = regex.search(instr.stem) diff --git a/src/submissions/backend/validators/pydant.py b/src/submissions/backend/validators/pydant.py index 0a16c8a..7cb134e 100644 --- a/src/submissions/backend/validators/pydant.py +++ b/src/submissions/backend/validators/pydant.py @@ -20,6 +20,8 @@ from backend.db.functions import (lookup_submissions, lookup_reagent_types, look from backend.db.models import * from sqlalchemy.exc import InvalidRequestError, StatementError from PyQt6.QtWidgets import QComboBox, QWidget, QLabel, QVBoxLayout +from pprint import pformat +from openpyxl import load_workbook logger = logging.getLogger(f"submissions.{__name__}") @@ -29,7 +31,7 @@ class PydReagent(BaseModel): type: str|None expiry: date|None name: str|None - parsed: bool = Field(default=False) + missing: bool = Field(default=True) @field_validator("type", mode='before') @classmethod @@ -134,6 +136,11 @@ class PydSample(BaseModel, extra='allow'): return [value] return value + @field_validator("submitter_id", mode="before") + @classmethod + def int_to_str(cls, value): + return str(value) + def toSQL(self, ctx:Settings, submission): result = None self.__dict__.update(self.model_extra) @@ -165,14 +172,14 @@ class PydSubmission(BaseModel, extra='allow'): filepath: Path submission_type: dict|None # For defaults - submitter_plate_num: dict|None = Field(default=dict(value=None, parsed=False), validate_default=True) - rsl_plate_num: dict|None = Field(default=dict(value=None, parsed=False), validate_default=True) + submitter_plate_num: dict|None = Field(default=dict(value=None, missing=True), validate_default=True) + rsl_plate_num: dict|None = Field(default=dict(value=None, missing=True), validate_default=True) submitted_date: dict|None submitting_lab: dict|None sample_count: dict|None extraction_kit: dict|None technician: dict|None - submission_category: dict|None = Field(default=dict(value=None, parsed=False), validate_default=True) + submission_category: dict|None = Field(default=dict(value=None, missing=True), validate_default=True) reagents: List[dict]|List[PydReagent] = [] samples: List[Any] @@ -181,7 +188,7 @@ class PydSubmission(BaseModel, extra='allow'): def enforce_with_uuid(cls, value): logger.debug(f"submitter plate id: {value}") if value['value'] == None or value['value'] == "None": - return dict(value=uuid.uuid4().hex.upper(), parsed=False) + return dict(value=uuid.uuid4().hex.upper(), missing=True) else: return value @@ -189,7 +196,7 @@ class PydSubmission(BaseModel, extra='allow'): @classmethod def rescue_date(cls, value): if value == None: - return dict(value=date.today(), parsed=False) + return dict(value=date.today(), missing=True) return value @field_validator("submitted_date") @@ -200,14 +207,14 @@ class PydSubmission(BaseModel, extra='allow'): if isinstance(value['value'], date): return value if isinstance(value['value'], int): - return dict(value=datetime.fromordinal(datetime(1900, 1, 1).toordinal() + value['value'] - 2).date(), parsed=False) + return dict(value=datetime.fromordinal(datetime(1900, 1, 1).toordinal() + value['value'] - 2).date(), missing=True) string = re.sub(r"(_|-)\d$", "", value['value']) try: - output = dict(value=parse(string).date(), parsed=False) + output = dict(value=parse(string).date(), missing=True) except ParserError as e: logger.error(f"Problem parsing date: {e}") try: - output = dict(value=parse(string.replace("-","")).date(), parsed=False) + output = dict(value=parse(string.replace("-","")).date(), missing=True) except Exception as e: logger.error(f"Problem with parse fallback: {e}") return output @@ -216,14 +223,14 @@ class PydSubmission(BaseModel, extra='allow'): @classmethod def rescue_submitting_lab(cls, value): if value == None: - return dict(value=None, parsed=False) + return dict(value=None, missing=True) return value @field_validator("rsl_plate_num", mode='before') @classmethod def rescue_rsl_number(cls, value): if value == None: - return dict(value=None, parsed=False) + return dict(value=None, missing=True) return value @field_validator("rsl_plate_num") @@ -233,21 +240,21 @@ class PydSubmission(BaseModel, extra='allow'): sub_type = values.data['submission_type']['value'] if check_not_nan(value['value']): if lookup_submissions(ctx=values.data['ctx'], rsl_number=value['value']) == None: - return dict(value=value['value'], parsed=True) + return dict(value=value['value'], missing=False) else: logger.warning(f"Submission number {value} already exists in DB, attempting salvage with filepath") # output = RSLNamer(ctx=values.data['ctx'], instr=values.data['filepath'].__str__(), sub_type=sub_type).parsed_name output = RSLNamer(ctx=values.data['ctx'], instr=values.data['filepath'].__str__(), sub_type=sub_type).parsed_name - return dict(value=output, parsed=False) + return dict(value=output, missing=True) else: output = RSLNamer(ctx=values.data['ctx'], instr=values.data['filepath'].__str__(), sub_type=sub_type).parsed_name - return dict(value=output, parsed=False) + return dict(value=output, missing=True) @field_validator("technician", mode="before") @classmethod def rescue_tech(cls, value): if value == None: - return dict(value=None, parsed=False) + return dict(value=None, missing=True) return value @field_validator("technician") @@ -257,14 +264,14 @@ class PydSubmission(BaseModel, extra='allow'): value['value'] = re.sub(r"\: \d", "", value['value']) return value else: - return dict(value=convert_nans_to_nones(value['value']), parsed=False) + return dict(value=convert_nans_to_nones(value['value']), missing=True) return value @field_validator("sample_count", mode='before') @classmethod def rescue_sample_count(cls, value): if value == None: - return dict(value=None, parsed=False) + return dict(value=None, missing=True) return value @field_validator("extraction_kit", mode='before') @@ -273,13 +280,13 @@ class PydSubmission(BaseModel, extra='allow'): if check_not_nan(value): if isinstance(value, str): - return dict(value=value, parsed=True) + return dict(value=value, missing=False) elif isinstance(value, dict): return value else: raise ValueError(f"No extraction kit found.") if value == None: - return dict(value=None, parsed=False) + return dict(value=None, missing=True) return value @field_validator("submission_type", mode='before') @@ -289,11 +296,11 @@ class PydSubmission(BaseModel, extra='allow'): value = {"value": value} if check_not_nan(value['value']): value = value['value'].title() - return dict(value=value, parsed=True) + return dict(value=value, missing=False) # else: # return dict(value="RSL Name not found.") else: - return dict(value=RSLNamer(ctx=values.data['ctx'], instr=values.data['filepath'].__str__()).submission_type.title(), parsed=False) + return dict(value=RSLNamer(ctx=values.data['ctx'], instr=values.data['filepath'].__str__()).submission_type.title(), missing=True) @field_validator("submission_category") @classmethod @@ -318,9 +325,21 @@ class PydSubmission(BaseModel, extra='allow'): output.append(dummy) self.samples = output + def improved_dict(self): + fields = list(self.model_fields.keys()) + list(self.model_extra.keys()) + output = {k:getattr(self, k) for k in fields} + return output + + def find_missing(self): + info = {k:v for k,v in self.improved_dict().items() if isinstance(v, dict)} + missing_info = {k:v for k,v in info.items() if v['missing']} + missing_reagents = [reagent for reagent in self.reagents if reagent.missing] + return missing_info, missing_reagents + def toSQL(self): code = 0 msg = None + status = None self.__dict__.update(self.model_extra) instance = lookup_submissions(ctx=self.ctx, rsl_number=self.rsl_plate_num['value']) if instance == None: @@ -358,6 +377,11 @@ class PydSubmission(BaseModel, extra='allow'): field_value = [reagent['value'].toSQL()[0] if isinstance(reagent, dict) else reagent.toSQL()[0] for reagent in value] case "submission_type": field_value = lookup_submission_type(ctx=self.ctx, name=value) + case "sample_count": + if value == None: + field_value = len(self.samples) + else: + field_value = value case "ctx" | "csv" | "filepath": continue case _: @@ -394,9 +418,85 @@ class PydSubmission(BaseModel, extra='allow'): except AttributeError as e: logger.debug(f"Something went wrong constructing instance {self.rsl_plate_num}: {e}") logger.debug(f"Constructed submissions message: {msg}") - return instance, {'code':code, 'message':msg} + return instance, {'code':code, 'message':msg, 'status':"Information"} - def toForm(self): + def toForm(self, parent:QWidget): + from frontend.custom_widgets.misc import SubmissionFormWidget + return SubmissionFormWidget(parent=parent, **self.improved_dict()) + + def autofill_excel(self, missing_only:bool=True): + if missing_only: + info, reagents = self.find_missing() + else: + info = {k:v for k,v in self.improved_dict().items() if isinstance(v, dict)} + reagents = self.reagents + if len(reagents + list(info.keys())) == 0: + return None + logger.debug(f"We have blank info and/or reagents in the excel sheet.\n\tLet's try to fill them in.") + extraction_kit = lookup_kit_types(ctx=self.ctx, name=self.extraction_kit['value']) + logger.debug(f"We have the extraction kit: {extraction_kit.name}") + excel_map = extraction_kit.construct_xl_map_for_use(self.submission_type['value']) + logger.debug(f"Extraction kit map:\n\n{pformat(excel_map)}") + logger.debug(f"Missing reagents going into autofile: {pformat(reagents)}") + logger.debug(f"Missing info going into autofile: {pformat(info)}") + new_reagents = [] + for reagent in reagents: + new_reagent = {} + new_reagent['type'] = reagent.type + new_reagent['lot'] = excel_map[new_reagent['type']]['lot'] + new_reagent['lot']['value'] = reagent.lot + new_reagent['expiry'] = excel_map[new_reagent['type']]['expiry'] + new_reagent['expiry']['value'] = reagent.expiry + new_reagent['sheet'] = excel_map[new_reagent['type']]['sheet'] + # name is only present for Bacterial Culture + try: + new_reagent['name'] = excel_map[new_reagent['type']]['name'] + new_reagent['name']['value'] = reagent.name + except Exception as e: + logger.error(f"Couldn't get name due to {e}") + new_reagents.append(new_reagent) + new_info = [] + for k,v in info.items(): + try: + new_item = {} + new_item['type'] = k + new_item['location'] = excel_map['info'][k] + new_item['value'] = v['value'] + new_info.append(new_item) + except KeyError: + logger.error(f"Unable to fill in {k}, not found in relevant info.") + logger.debug(f"New reagents: {new_reagents}") + logger.debug(f"New info: {new_info}") + # open a new workbook using openpyxl + workbook = load_workbook(self.filepath) + # get list of sheet names + sheets = workbook.sheetnames + # logger.debug(workbook.sheetnames) + for sheet in sheets: + # open sheet + worksheet=workbook[sheet] + # Get relevant reagents for that sheet + sheet_reagents = [item for item in new_reagents if sheet in item['sheet']] + for reagent in sheet_reagents: + # logger.debug(f"Attempting to write lot {reagent['lot']['value']} in: row {reagent['lot']['row']}, column {reagent['lot']['column']}") + worksheet.cell(row=reagent['lot']['row'], column=reagent['lot']['column'], value=reagent['lot']['value']) + # logger.debug(f"Attempting to write expiry {reagent['expiry']['value']} in: row {reagent['expiry']['row']}, column {reagent['expiry']['column']}") + worksheet.cell(row=reagent['expiry']['row'], column=reagent['expiry']['column'], value=reagent['expiry']['value']) + try: + # logger.debug(f"Attempting to write name {reagent['name']['value']} in: row {reagent['name']['row']}, column {reagent['name']['column']}") + worksheet.cell(row=reagent['name']['row'], column=reagent['name']['column'], value=reagent['name']['value']) + except Exception as e: + logger.error(f"Could not write name {reagent['name']['value']} due to {e}") + # Get relevant info for that sheet + sheet_info = [item for item in new_info if sheet in item['location']['sheets']] + for item in sheet_info: + logger.debug(f"Attempting: {item['type']} in row {item['location']['row']}, column {item['location']['column']}") + worksheet.cell(row=item['location']['row'], column=item['location']['column'], value=item['value']) + # Hacky way to pop in 'signed by' + # custom_parser = get_polymorphic_subclass(BasicSubmission, info['submission_type']) + custom_parser = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=self.submission_type['value']) + workbook = custom_parser.custom_autofill(workbook) + return workbook class PydContact(BaseModel): diff --git a/src/submissions/frontend/__init__.py b/src/submissions/frontend/__init__.py index 6ccf3a9..c719b8d 100644 --- a/src/submissions/frontend/__init__.py +++ b/src/submissions/frontend/__init__.py @@ -13,7 +13,7 @@ from PyQt6.QtCore import pyqtSignal from PyQt6.QtGui import QAction from PyQt6.QtWebEngineWidgets import QWebEngineView from pathlib import Path -from backend.db import ( +from backend.db.functions import ( lookup_control_types, lookup_modes ) from backend.validators import PydSubmission, PydReagent @@ -22,6 +22,7 @@ from frontend.custom_widgets import SubmissionsSheet, AlertPop, AddReagentForm, import logging from datetime import date import webbrowser +from pathlib import Path logger = logging.getLogger(f'submissions.{__name__}') logger.info("Hello, I am a logger") @@ -32,6 +33,7 @@ class App(QMainWindow): logger.debug(f"Initializing main window...") super().__init__() self.ctx = ctx + self.last_dir = ctx.directory_path # indicate version and connected database in title bar try: self.title = f"Submissions App (v{ctx.package.__version__}) - {ctx.database_path}" @@ -156,6 +158,7 @@ class App(QMainWindow): Args: result (dict | None, optional): The result from a function. Defaults to None. """ + logger.info(f"We got the result: {result}") if result != None: msg = AlertPop(message=result['message'], status=result['status']) msg.exec() @@ -399,7 +402,7 @@ class SubmissionFormContainer(QWidget): def __init__(self, parent: QWidget) -> None: logger.debug(f"Setting form widget...") super().__init__(parent) - self.parent = parent + # self.parent = parent self.setAcceptDrops(True) @@ -411,37 +414,8 @@ class SubmissionFormContainer(QWidget): def dropEvent(self, event): fname = Path([u.toLocalFile() for u in event.mimeData().urls()][0]) + app = self.parent().parent().parent().parent().parent().parent().parent + logger.debug(f"App: {app}") + app.last_dir = fname.parent self.import_drag.emit(fname) - def clear_form(self): - for item in self.findChildren(QWidget): - item.setParent(None) - - def parse_form(self) -> PydSubmission: - logger.debug(f"Hello from form parser!") - info = {} - reagents = [] - samples = self.parent.parent.samples - logger.debug(f"Using samples: {pformat(samples)}") - widgets = [widget for widget in self.findChildren(QWidget) if widget.objectName() not in self.ignore] - # widgets = [widget for widget in self.findChildren(QWidget)] - for widget in widgets: - logger.debug(f"Parsed widget: {widget.objectName()} of type {type(widget)}") - match widget: - case ReagentFormWidget(): - reagent, _ = widget.parse_form() - reagents.append(reagent) - case ImportReagent(): - reagent = dict(name=widget.objectName().replace("lot_", ""), lot=widget.currentText(), type=None, expiry=None) - reagents.append(PydReagent(ctx=self.parent.parent.ctx, **reagent)) - case QLineEdit(): - info[widget.objectName()] = dict(value=widget.text()) - case QComboBox(): - info[widget.objectName()] = dict(value=widget.currentText()) - case QDateEdit(): - info[widget.objectName()] = dict(value=widget.date().toPyDate()) - logger.debug(f"Info: {pformat(info)}") - logger.debug(f"Reagents: {pformat(reagents)}") - # sys.exit("Hi Landon. Check the reagents! frontend.__init__ line 442") - submission = PydSubmission(ctx=self.parent.parent.ctx, filepath=self.parent.parent.current_file, reagents=reagents, samples=samples, **info) - return submission diff --git a/src/submissions/frontend/all_window_functions.py b/src/submissions/frontend/all_window_functions.py index e6e959c..7a75b94 100644 --- a/src/submissions/frontend/all_window_functions.py +++ b/src/submissions/frontend/all_window_functions.py @@ -23,11 +23,13 @@ def select_open_file(obj:QMainWindow, file_extension:str) -> Path: Path: Path of file to be opened """ try: - home_dir = Path(obj.ctx.directory_path).resolve().__str__() + # home_dir = Path(obj.ctx.directory_path).resolve().__str__() + home_dir = obj.last_dir.resolve().__str__() except FileNotFoundError: home_dir = Path.home().resolve().__str__() fname = Path(QFileDialog.getOpenFileName(obj, 'Open file', home_dir, filter = f"{file_extension}(*.{file_extension})")[0]) # fname = Path(QFileDialog.getOpenFileName(obj, 'Open file', filter = f"{file_extension}(*.{file_extension})")[0]) + obj.last_file = fname return fname def select_save_file(obj:QMainWindow, default_name:str, extension:str) -> Path: @@ -43,12 +45,13 @@ def select_save_file(obj:QMainWindow, default_name:str, extension:str) -> Path: Path: Path of file to be opened """ try: - home_dir = Path(obj.ctx.directory_path).joinpath(default_name).resolve().__str__() + # home_dir = Path(obj.ctx.directory_path).joinpath(default_name).resolve().__str__() + home_dir = obj.last_dir.joinpath(default_name).resolve().__str__() except FileNotFoundError: home_dir = Path.home().joinpath(default_name).resolve().__str__() fname = Path(QFileDialog.getSaveFileName(obj, "Save File", home_dir, filter = f"{extension}(*.{extension})")[0]) # fname = Path(QFileDialog.getSaveFileName(obj, "Save File", filter = f"{extension}(*.{extension})")[0]) - + obj.last_dir = fname.parent return fname def extract_form_info(object) -> dict: diff --git a/src/submissions/frontend/custom_widgets/misc.py b/src/submissions/frontend/custom_widgets/misc.py index 9a2bb3b..41ce586 100644 --- a/src/submissions/frontend/custom_widgets/misc.py +++ b/src/submissions/frontend/custom_widgets/misc.py @@ -11,19 +11,20 @@ from PyQt6.QtWidgets import ( QGridLayout, QPushButton, QSpinBox, QDoubleSpinBox, QHBoxLayout, QScrollArea, QFormLayout ) -from PyQt6.QtCore import Qt, QDate, QSize +from PyQt6.QtCore import Qt, QDate, QSize, pyqtSignal from tools import check_not_nan, jinja_template_loading, Settings from backend.db.functions import \ lookup_reagent_types, lookup_reagents, lookup_submission_type, lookup_reagenttype_kittype_association, \ - lookup_submissions#, construct_kit_from_yaml + lookup_submissions, lookup_organizations, lookup_kit_types from backend.db.models import SubmissionTypeKitTypeAssociation from sqlalchemy import FLOAT, INTEGER import logging import numpy as np from .pop_ups import AlertPop, QuestionAsker from backend.validators import PydReagent, PydKit, PydReagentType, PydSubmission -from typing import Tuple +from typing import Tuple, List from pprint import pformat +import difflib logger = logging.getLogger(f"submissions.{__name__}") @@ -388,6 +389,10 @@ class ControlsDatePicker(QWidget): class ImportReagent(QComboBox): + """ + NOTE: Depreciated in favour of ReagentFormWidget + """ + def __init__(self, ctx:Settings, reagent:dict|PydReagent, extraction_kit:str): super().__init__() self.setEditable(True) @@ -442,25 +447,6 @@ class ImportReagent(QComboBox): self.setObjectName(f"lot_{reagent.type}") self.addItems(relevant_reagents) -class ParsedQLabel(QLabel): - - def __init__(self, input_object, field_name, title:bool=True, label_name:str|None=None): - super().__init__() - try: - check = input_object['parsed'] - except: - return - if label_name != None: - self.setObjectName(label_name) - if title: - output = field_name.replace('_', ' ').title() - else: - output = field_name.replace('_', ' ') - if check: - self.setText(f"Parsed {output}") - else: - self.setText(f"MISSING {output}") - class FirstStrandSalvage(QDialog): def __init__(self, ctx:Settings, submitter_id:str, rsl_plate_num:str|None=None) -> None: @@ -526,10 +512,8 @@ class FirstStrandPlateList(QDialog): class ReagentFormWidget(QWidget): def __init__(self, parent:QWidget, reagent:PydReagent, extraction_kit:str): - super().__init__() - self.setParent(parent) - logger.debug(f"Reagent form widget parent is: {self.parent()}") - logger.debug(f"It's great grandparent is {self.parent().parent.parent} which has a method [add_reagent]: {hasattr(self.parent().parent.parent, 'add_reagent')}") + super().__init__(parent) + # self.setParent(parent) self.reagent = reagent self.extraction_kit = extraction_kit self.ctx = reagent.ctx @@ -538,49 +522,57 @@ class ReagentFormWidget(QWidget): layout.addWidget(self.label) self.lot = self.ReagentLot(reagent=reagent, extraction_kit=extraction_kit) layout.addWidget(self.lot) + # Remove spacing between reagents + layout.setContentsMargins(0,0,0,0) self.setLayout(layout) self.setObjectName(reagent.name) - self.missing = not reagent.parsed + self.missing = reagent.missing + # If changed set self.missing to True and update self.label + self.lot.currentTextChanged.connect(self.updated) def parse_form(self) -> Tuple[PydReagent, dict]: lot = self.lot.currentText() - # type = self.label.text().replace("_label") wanted_reagent = lookup_reagents(ctx=self.ctx, lot_number=lot, reagent_type=self.reagent.type) + # if reagent doesn't exist in database, off to add it (uses App.add_reagent) if wanted_reagent == None: dlg = QuestionAsker(title=f"Add {lot}?", message=f"Couldn't find reagent type {self.reagent.type}: {lot} in the database.\n\nWould you like to add it?") if dlg.exec(): - # logger.debug(f"Looking through {pformat(self.parent.reagents)} for reagent {reagent.name}") - # try: - # picked_reagent = [item for item in obj.reagents if item.type == reagent.name][0] - # except IndexError: - # logger.error(f"Couldn't find {reagent.name} in obj.reagents. Checking missing reagents {pprint.pformat(obj.missing_reagents)}") - # picked_reagent = [item for item in obj.missing_reagents if item.type == reagent.name][0] - # logger.debug(f"checking reagent: {reagent.name} in obj.reagents. Result: {picked_reagent}") - # expiry_date = picked_reagent.expiry - wanted_reagent = self.parent().parent.parent.add_reagent(reagent_lot=lot, reagent_type=self.reagent.type, expiry=self.reagent.expiry, name=self.reagent.name) + wanted_reagent = self.parent().parent().parent().parent().parent().parent().parent().parent().parent.add_reagent(reagent_lot=lot, reagent_type=self.reagent.type, expiry=self.reagent.expiry, name=self.reagent.name) return wanted_reagent, None else: # In this case we will have an empty reagent and the submission will fail kit integrity check logger.debug("Will not add reagent.") return None, dict(message="Failed integrity check", status="critical") else: - rt = lookup_reagent_types(ctx=self.ctx, kit_type=self.extraction_kit, reagent=wanted_reagent) + # Since this now gets passed in directly from the parser -> pyd -> form and the parser gets the name + # from the db, it should no longer be necessary to query the db with reagent/kit, but with rt name directly. + rt = lookup_reagent_types(ctx=self.ctx, name=self.reagent.type) + # rt = lookup_reagent_types(ctx=self.ctx, kit_type=self.extraction_kit, reagent=wanted_reagent) + if rt == None: + rt = lookup_reagent_types(ctx=self.ctx, kit_type=self.extraction_kit, reagent=wanted_reagent) return PydReagent(ctx=self.ctx, name=wanted_reagent.name, lot=wanted_reagent.lot, type=rt.name, expiry=wanted_reagent.expiry, parsed=not self.missing), None + def updated(self): + self.missing = True + self.label.updated(self.reagent.type) + class ReagentParsedLabel(QLabel): def __init__(self, reagent:PydReagent): super().__init__() try: - check = reagent.parsed + check = not reagent.missing except: - return + check = False self.setObjectName(f"{reagent.type}_label") if check: self.setText(f"Parsed {reagent.type}") else: self.setText(f"MISSING {reagent.type}") + + def updated(self, reagent_type:str): + self.setText(f"UPDATED {reagent_type}") class ReagentLot(QComboBox): @@ -588,8 +580,8 @@ class ReagentFormWidget(QWidget): super().__init__() self.ctx = reagent.ctx self.setEditable(True) - if reagent.parsed: - pass + # if reagent.parsed: + # pass logger.debug(f"Attempting lookup of reagents by type: {reagent.type}") # below was lookup_reagent_by_type_name_and_kit_name, but I couldn't get it to work. lookup = lookup_reagents(ctx=self.ctx, reagent_type=reagent.type) @@ -611,8 +603,11 @@ class ReagentFormWidget(QWidget): else: # TODO: look up the last used reagent of this type in the database looked_up_rt = lookup_reagenttype_kittype_association(ctx=self.ctx, reagent_type=reagent.type, kit_type=extraction_kit) - looked_up_reg = lookup_reagents(ctx=self.ctx, lot_number=looked_up_rt.last_used) - logger.debug(f"Because there was no reagent listed for {reagent}, we will insert the last lot used: {looked_up_reg}") + try: + looked_up_reg = lookup_reagents(ctx=self.ctx, lot_number=looked_up_rt.last_used) + except AttributeError: + looked_up_reg = None + logger.debug(f"Because there was no reagent listed for {reagent.lot}, we will insert the last lot used: {looked_up_reg}") if looked_up_reg != None: relevant_reagents.remove(str(looked_up_reg.lot)) relevant_reagents.insert(0, str(looked_up_reg.lot)) @@ -631,41 +626,212 @@ class ReagentFormWidget(QWidget): class SubmissionFormWidget(QWidget): - def __init__(self, parent: QWidget) -> None: + def __init__(self, parent: QWidget, **kwargs) -> None: super().__init__(parent) - self.ignore = [None, "", "qt_spinbox_lineedit", "qt_scrollarea_viewport", "qt_scrollarea_hcontainer", - "qt_scrollarea_vcontainer", "submit_btn" - ] + # self.ignore = [None, "", "qt_spinbox_lineedit", "qt_scrollarea_viewport", "qt_scrollarea_hcontainer", + # "qt_scrollarea_vcontainer", "submit_btn" + # ] + self.ignore = ['filepath', 'samples', 'reagents', 'csv', 'ctx'] + layout = QVBoxLayout() + for k, v in kwargs.items(): + if k not in self.ignore: + add_widget = self.create_widget(key=k, value=v, submission_type=kwargs['submission_type']) + if add_widget != None: + layout.addWidget(add_widget) + else: + setattr(self, k, v) + self.setLayout(layout) + + def create_widget(self, key:str, value:dict, submission_type:str|None=None): + if key not in self.ignore: + return self.InfoItem(self, key=key, value=value, submission_type=submission_type) + return None def clear_form(self): for item in self.findChildren(QWidget): item.setParent(None) + def find_widgets(self, object_name:str|None=None) -> List[QWidget]: + query = self.findChildren(QWidget) + if object_name != None: + query = [widget for widget in query if widget.objectName()==object_name] + return query + def parse_form(self) -> PydSubmission: logger.debug(f"Hello from form parser!") info = {} reagents = [] - samples = self.parent.parent.samples - logger.debug(f"Using samples: {pformat(samples)}") - widgets = [widget for widget in self.findChildren(QWidget) if widget.objectName() not in self.ignore] + if hasattr(self, 'csv'): + info['csv'] = self.csv + # samples = self.parent().parent.parent.samples + # filepath = self.parent().parent.parent.pyd.filepath + # logger.debug(f"Using samples: {pformat(samples)}") + # widgets = [widget for widget in self.findChildren(QWidget) if widget.objectName() not in self.ignore] # widgets = [widget for widget in self.findChildren(QWidget)] - for widget in widgets: - logger.debug(f"Parsed widget: {widget.objectName()} of type {type(widget)}") + # for widget in widgets: + for widget in self.findChildren(QWidget): + logger.debug(f"Parsed widget of type {type(widget)}") match widget: case ReagentFormWidget(): reagent, _ = widget.parse_form() reagents.append(reagent) - case ImportReagent(): - reagent = dict(name=widget.objectName().replace("lot_", ""), lot=widget.currentText(), type=None, expiry=None) - reagents.append(PydReagent(ctx=self.parent.parent.ctx, **reagent)) - case QLineEdit(): - info[widget.objectName()] = dict(value=widget.text()) - case QComboBox(): - info[widget.objectName()] = dict(value=widget.currentText()) - case QDateEdit(): - info[widget.objectName()] = dict(value=widget.date().toPyDate()) + case self.InfoItem(): + field, value = widget.parse_form() + if field != None: + info[field] = value + # case ImportReagent(): + # reagent = dict(name=widget.objectName().replace("lot_", ""), lot=widget.currentText(), type=None, expiry=None) + # # ctx: self.SubmissionContinerWidget.AddSubForm + # reagents.append(PydReagent(ctx=self.parent.parent.ctx, **reagent)) + # case QLineEdit(): + # info[widget.objectName()] = dict(value=widget.text()) + # case QComboBox(): + # info[widget.objectName()] = dict(value=widget.currentText()) + # case QDateEdit(): + # info[widget.objectName()] = dict(value=widget.date().toPyDate()) logger.debug(f"Info: {pformat(info)}") logger.debug(f"Reagents: {pformat(reagents)}") - # sys.exit("Hi Landon. Check the reagents! frontend.__init__ line 442") - submission = PydSubmission(ctx=self.parent.parent.ctx, filepath=self.parent.parent.current_file, reagents=reagents, samples=samples, **info) + app = self.parent().parent().parent().parent().parent().parent().parent().parent + submission = PydSubmission(ctx=app.ctx, filepath=self.filepath, reagents=reagents, samples=self.samples, **info) return submission + + class InfoItem(QWidget): + + def __init__(self, parent: QWidget, key:str, value:dict, submission_type:str|None=None) -> None: + super().__init__(parent) + layout = QVBoxLayout() + self.label = self.ParsedQLabel(key=key, value=value) + self.input: QWidget = self.set_widget(parent=self, key=key, value=value, submission_type=submission_type['value']) + self.setObjectName(key) + try: + self.missing:bool = value['missing'] + except (TypeError, KeyError): + self.missing:bool = False + if self.input != None: + layout.addWidget(self.label) + layout.addWidget(self.input) + layout.setContentsMargins(0,0,0,0) + self.setLayout(layout) + match self.input: + case QComboBox(): + self.input.currentTextChanged.connect(self.update_missing) + case QDateEdit(): + self.input.dateChanged.connect(self.update_missing) + case QLineEdit(): + self.input.textChanged.connect(self.update_missing) + + def parse_form(self): + match self.input: + case QLineEdit(): + value = self.input.text() + case QComboBox(): + value = self.input.currentText() + case QDateEdit(): + value = self.input.date().toPyDate() + case _: + return None, None + return self.input.objectName(), dict(value=value, missing=self.missing) + + def set_widget(self, parent: QWidget, key:str, value:dict, submission_type:str|None=None) -> QWidget: + try: + value = value['value'] + except (TypeError, KeyError): + pass + obj = parent.parent().parent() + logger.debug(f"Creating widget for: {key}") + match key: + case 'submitting_lab': + add_widget = QComboBox() + # lookup organizations suitable for submitting_lab (ctx: self.InfoItem.SubmissionFormWidget.SubmissionFormContainer.AddSubForm ) + labs = [item.__str__() for item in lookup_organizations(ctx=obj.ctx)] + # try to set closest match to top of list + try: + labs = difflib.get_close_matches(value, labs, len(labs), 0) + except (TypeError, ValueError): + pass + # set combobox values to lookedup values + add_widget.addItems(labs) + case 'extraction_kit': + # if extraction kit not available, all other values fail + if not check_not_nan(value): + msg = AlertPop(message="Make sure to check your extraction kit in the excel sheet!", status="warning") + msg.exec() + # create combobox to hold looked up kits + add_widget = QComboBox() + # lookup existing kits by 'submission_type' decided on by sheetparser + logger.debug(f"Looking up kits used for {submission_type}") + uses = [item.__str__() for item in lookup_kit_types(ctx=obj.ctx, used_for=submission_type)] + obj.uses = uses + logger.debug(f"Kits received for {submission_type}: {uses}") + if check_not_nan(value): + logger.debug(f"The extraction kit in parser was: {value}") + uses.insert(0, uses.pop(uses.index(value))) + obj.ext_kit = value + else: + logger.error(f"Couldn't find {obj.prsr.sub['extraction_kit']}") + obj.ext_kit = uses[0] + add_widget.addItems(uses) + + # Run reagent scraper whenever extraction kit is changed. + # add_widget.currentTextChanged.connect(obj.scrape_reagents) + case 'submitted_date': + # uses base calendar + add_widget = QDateEdit(calendarPopup=True) + # sets submitted date based on date found in excel sheet + try: + add_widget.setDate(value) + # if not found, use today + except: + add_widget.setDate(date.today()) + case 'submission_category': + add_widget = QComboBox() + cats = ['Diagnostic', "Surveillance", "Research"] + cats += [item.name for item in lookup_submission_type(ctx=obj.ctx)] + try: + cats.insert(0, cats.pop(cats.index(value))) + except ValueError: + cats.insert(0, cats.pop(cats.index(submission_type))) + add_widget.addItems(cats) + case _: + # anything else gets added in as a line edit + add_widget = QLineEdit() + logger.debug(f"Setting widget text to {str(value).replace('_', ' ')}") + add_widget.setText(str(value).replace("_", " ")) + if add_widget != None: + add_widget.setObjectName(key) + add_widget.setParent(parent) + + return add_widget + + def update_missing(self): + self.missing = True + self.label.updated(self.objectName()) + + class ParsedQLabel(QLabel): + + def __init__(self, key:str, value:dict, title:bool=True, label_name:str|None=None): + super().__init__() + try: + check = not value['missing'] + except: + check = True + if label_name != None: + self.setObjectName(label_name) + else: + self.setObjectName(f"{key}_label") + if title: + output = key.replace('_', ' ').title() + else: + output = key.replace('_', ' ') + if check: + self.setText(f"Parsed {output}") + else: + self.setText(f"MISSING {output}") + + def updated(self, key:str, title:bool=True): + if title: + output = key.replace('_', ' ').title() + else: + output = key.replace('_', ' ') + self.setText(f"UPDATED {output}") + diff --git a/src/submissions/frontend/main_window_functions.py b/src/submissions/frontend/main_window_functions.py index ac2399f..4029946 100644 --- a/src/submissions/frontend/main_window_functions.py +++ b/src/submissions/frontend/main_window_functions.py @@ -7,6 +7,7 @@ from getpass import getuser import inspect import pprint import re +import sys import yaml import json from typing import Tuple, List @@ -35,7 +36,6 @@ from backend.validators import PydSubmission, PydSample, PydReagent from tools import check_not_nan, convert_well_to_row_column from .custom_widgets.pop_ups import AlertPop, QuestionAsker from .custom_widgets import ReportDatePicker -from .custom_widgets.misc import ImportReagent, ParsedQLabel from .visualizations.control_charts import create_charts, construct_html from pathlib import Path from frontend.custom_widgets.misc import FirstStrandSalvage, FirstStrandPlateList, ReagentFormWidget @@ -54,8 +54,12 @@ def import_submission_function(obj:QMainWindow, fname:Path|None=None) -> Tuple[Q """ logger.debug(f"\n\nStarting Import...\n\n") result = None - logger.debug(obj.ctx) + # logger.debug(obj.ctx) # initialize samples + try: + obj.form.setParent(None) + except AttributeError: + pass obj.samples = [] obj.missing_info = [] # set file dialog @@ -73,106 +77,114 @@ def import_submission_function(obj:QMainWindow, fname:Path|None=None) -> Tuple[Q return obj, result try: logger.debug(f"Submission dictionary:\n{pprint.pformat(obj.prsr.sub)}") - pyd = obj.prsr.to_pydantic() - logger.debug(f"Pydantic result: \n\n{pprint.pformat(pyd)}\n\n") + obj.pyd = obj.prsr.to_pydantic() + logger.debug(f"Pydantic result: \n\n{pprint.pformat(obj.pyd)}\n\n") except Exception as e: return obj, dict(message= f"Problem creating pydantic model:\n\n{e}", status="critical") # destroy any widgets from previous imports - obj.table_widget.formwidget.clear_form() - obj.current_submission_type = pyd.submission_type['value'] - obj.current_file = pyd.filepath + # obj.table_widget.formwidget.set_parent(None) + # obj.current_submission_type = pyd.submission_type['value'] + # obj.current_file = pyd.filepath # Get list of fields from pydantic model. - fields = list(pyd.model_fields.keys()) + list(pyd.model_extra.keys()) - fields.remove('filepath') - logger.debug(f"pydantic fields: {fields}") - for field in fields: - value = getattr(pyd, field) - logger.debug(f"Checking: {field}: {value}") - # Get from pydantic model whether field was completed in the form - if isinstance(value, dict) and field != 'ctx': - logger.debug(f"The field {field} is a dictionary: {value}") - if not value['parsed']: - obj.missing_info.append(field) - label = ParsedQLabel(value, field) - match field: - case 'submitting_lab': - logger.debug(f"{field}: {value['value']}") - # create combobox to hold looked up submitting labs - add_widget = QComboBox() - labs = [item.__str__() for item in lookup_organizations(ctx=obj.ctx)] - # try to set closest match to top of list - try: - labs = difflib.get_close_matches(value['value'], labs, len(labs), 0) - except (TypeError, ValueError): - pass - # set combobox values to lookedup values - add_widget.addItems(labs) - case 'extraction_kit': - # if extraction kit not available, all other values fail - if not check_not_nan(value['value']): - msg = AlertPop(message="Make sure to check your extraction kit in the excel sheet!", status="warning") - msg.exec() - # create combobox to hold looked up kits - add_widget = QComboBox() - # lookup existing kits by 'submission_type' decided on by sheetparser - logger.debug(f"Looking up kits used for {pyd.submission_type['value']}") - uses = [item.__str__() for item in lookup_kit_types(ctx=obj.ctx, used_for=pyd.submission_type['value'])] - logger.debug(f"Kits received for {pyd.submission_type['value']}: {uses}") - if check_not_nan(value['value']): - logger.debug(f"The extraction kit in parser was: {value['value']}") - uses.insert(0, uses.pop(uses.index(value['value']))) - obj.ext_kit = value['value'] - else: - logger.error(f"Couldn't find {obj.prsr.sub['extraction_kit']}") - obj.ext_kit = uses[0] - # Run reagent scraper whenever extraction kit is changed. - add_widget.currentTextChanged.connect(obj.scrape_reagents) - case 'submitted_date': - # uses base calendar - add_widget = QDateEdit(calendarPopup=True) - # sets submitted date based on date found in excel sheet - try: - add_widget.setDate(value['value']) - # if not found, use today - except: - add_widget.setDate(date.today()) - case 'samples': - # hold samples in 'obj' until form submitted - logger.debug(f"{field}:\n\t{value}") - obj.samples = value - continue - case 'submission_category': - add_widget = QComboBox() - cats = ['Diagnostic', "Surveillance", "Research"] - cats += [item.name for item in lookup_submission_type(ctx=obj.ctx)] - try: - cats.insert(0, cats.pop(cats.index(value['value']))) - except ValueError: - cats.insert(0, cats.pop(cats.index(pyd.submission_type['value']))) - add_widget.addItems(cats) - case "ctx" | 'reagents' | 'csv' | 'filepath': - continue - case _: - # anything else gets added in as a line edit - add_widget = QLineEdit() - logger.debug(f"Setting widget text to {str(value['value']).replace('_', ' ')}") - add_widget.setText(str(value['value']).replace("_", " ")) - try: - add_widget.setObjectName(field) - logger.debug(f"Widget name set to: {add_widget.objectName()}") - obj.table_widget.formlayout.addWidget(label) - obj.table_widget.formlayout.addWidget(add_widget) - except AttributeError as e: - logger.error(e) - kit_widget = obj.table_widget.formlayout.parentWidget().findChild(QComboBox, 'extraction_kit') - kit_widget.addItems(uses) + # fields = list(pyd.model_fields.keys()) + list(pyd.model_extra.keys()) + # fields.remove('filepath') + # logger.debug(f"pydantic fields: {fields}") + # for field in fields: + # value = getattr(pyd, field) + # logger.debug(f"Checking: {field}: {value}") + # # Get from pydantic model whether field was completed in the form + # if isinstance(value, dict) and field != 'ctx': + # logger.debug(f"The field {field} is a dictionary: {value}") + # if not value['parsed']: + # obj.missing_info.append(field) + # label = ParsedQLabel(value, field) + # match field: + # case 'submitting_lab': + # logger.debug(f"{field}: {value['value']}") + # # create combobox to hold looked up submitting labs + # add_widget = QComboBox() + # labs = [item.__str__() for item in lookup_organizations(ctx=obj.ctx)] + # # try to set closest match to top of list + # try: + # labs = difflib.get_close_matches(value['value'], labs, len(labs), 0) + # except (TypeError, ValueError): + # pass + # # set combobox values to lookedup values + # add_widget.addItems(labs) + # case 'extraction_kit': + # # if extraction kit not available, all other values fail + # if not check_not_nan(value['value']): + # msg = AlertPop(message="Make sure to check your extraction kit in the excel sheet!", status="warning") + # msg.exec() + # # create combobox to hold looked up kits + # add_widget = QComboBox() + # # lookup existing kits by 'submission_type' decided on by sheetparser + # logger.debug(f"Looking up kits used for {pyd.submission_type['value']}") + # uses = [item.__str__() for item in lookup_kit_types(ctx=obj.ctx, used_for=pyd.submission_type['value'])] + # logger.debug(f"Kits received for {pyd.submission_type['value']}: {uses}") + # if check_not_nan(value['value']): + # logger.debug(f"The extraction kit in parser was: {value['value']}") + # uses.insert(0, uses.pop(uses.index(value['value']))) + # obj.ext_kit = value['value'] + # else: + # logger.error(f"Couldn't find {obj.prsr.sub['extraction_kit']}") + # obj.ext_kit = uses[0] + # # Run reagent scraper whenever extraction kit is changed. + # add_widget.currentTextChanged.connect(obj.scrape_reagents) + # case 'submitted_date': + # # uses base calendar + # add_widget = QDateEdit(calendarPopup=True) + # # sets submitted date based on date found in excel sheet + # try: + # add_widget.setDate(value['value']) + # # if not found, use today + # except: + # add_widget.setDate(date.today()) + # case 'samples': + # # hold samples in 'obj' until form submitted + # logger.debug(f"{field}:\n\t{value}") + # obj.samples = value + # continue + # case 'submission_category': + # add_widget = QComboBox() + # cats = ['Diagnostic', "Surveillance", "Research"] + # cats += [item.name for item in lookup_submission_type(ctx=obj.ctx)] + # try: + # cats.insert(0, cats.pop(cats.index(value['value']))) + # except ValueError: + # cats.insert(0, cats.pop(cats.index(pyd.submission_type['value']))) + # add_widget.addItems(cats) + # case "ctx" | 'reagents' | 'csv' | 'filepath': + # continue + # case _: + # # anything else gets added in as a line edit + # add_widget = QLineEdit() + # logger.debug(f"Setting widget text to {str(value['value']).replace('_', ' ')}") + # add_widget.setText(str(value['value']).replace("_", " ")) + # try: + # add_widget.setObjectName(field) + # logger.debug(f"Widget name set to: {add_widget.objectName()}") + # obj.table_widget.formlayout.addWidget(label) + # obj.table_widget.formlayout.addWidget(add_widget) + # except AttributeError as e: + # logger.error(e) + obj.form = obj.pyd.toForm(parent=obj) + obj.table_widget.formlayout.addWidget(obj.form) + # kit_widget = obj.table_widget.formlayout.parentWidget().findChild(QComboBox, 'extraction_kit') + kit_widget = obj.form.find_widgets(object_name="extraction_kit")[0].input + logger.debug(f"Kitwidget {kit_widget}") + # block + # with QSignalBlocker(kit_widget) as blocker: + # kit_widget.addItems(obj.uses) + obj.scrape_reagents(kit_widget.currentText()) + kit_widget.currentTextChanged.connect(obj.scrape_reagents) # compare obj.reagents with expected reagents in kit if obj.prsr.sample_result != None: msg = AlertPop(message=obj.prsr.sample_result, status="WARNING") msg.exec() - logger.debug(f"Pydantic extra fields: {pyd.model_extra}") - if "csv" in pyd.model_extra: - obj.csv = pyd.model_extra['csv'] + # logger.debug(f"Pydantic extra fields: {obj.pyd.model_extra}") + # if "csv" in pyd.model_extra: + # obj.csv = pyd.model_extra['csv'] logger.debug(f"All attributes of obj:\n{pprint.pformat(obj.__dict__)}") return obj, result @@ -187,15 +199,19 @@ def kit_reload_function(obj:QMainWindow) -> Tuple[QMainWindow, dict]: Tuple[QMainWindow, dict]: Collection of new main app window and result dict """ result = None - for item in obj.table_widget.formlayout.parentWidget().findChildren(QWidget): - if isinstance(item, QLabel): - if item.text().startswith("Lot"): - item.setParent(None) - else: - logger.debug(f"Type of {item.objectName()} is {type(item)}") - if item.objectName().startswith("lot_"): - item.setParent(None) - obj.kit_integrity_completion_function() + # for item in obj.table_widget.formlayout.parentWidget().findChildren(QWidget): + logger.debug(f"Attempting to clear {obj.form.find_widgets()}") + + for item in obj.form.find_widgets(): + if isinstance(item, ReagentFormWidget): + item.setParent(None) + # if item.text().startswith("Lot"): + # item.setParent(None) + # else: + # logger.debug(f"Type of {item.objectName()} is {type(item)}") + # if item.objectName().startswith("lot_"): + # item.setParent(None) + kit_integrity_completion_function(obj) return obj, result def kit_integrity_completion_function(obj:QMainWindow) -> Tuple[QMainWindow, dict]: @@ -209,23 +225,31 @@ def kit_integrity_completion_function(obj:QMainWindow) -> Tuple[QMainWindow, dic Tuple[QMainWindow, dict]: Collection of new main app window and result dict """ result = None + missing_reagents = [] + # kit_reload_function(obj=obj) logger.debug(inspect.currentframe().f_back.f_code.co_name) # find the widget that contains kit info - kit_widget = obj.table_widget.formlayout.parentWidget().findChild(QComboBox, 'extraction_kit') + kit_widget = obj.form.find_widgets(object_name="extraction_kit")[0].input logger.debug(f"Kit selector: {kit_widget}") # get current kit being used obj.ext_kit = kit_widget.currentText() - for reagent in obj.reagents: + # for reagent in obj.pyd.reagents: + for reagent in obj.form.reagents: # obj.table_widget.formlayout.addWidget(ParsedQLabel({'parsed':True}, item.type, title=False, label_name=f"lot_{item.type}")) # reagent = dict(type=item.type, lot=item.lot, expiry=item.expiry, name=item.name) # add_widget = ImportReagent(ctx=obj.ctx, reagent=reagent, extraction_kit=obj.ext_kit) # obj.table_widget.formlayout.addWidget(add_widget) add_widget = ReagentFormWidget(parent=obj.table_widget.formwidget, reagent=reagent, extraction_kit=obj.ext_kit) - obj.table_widget.formlayout.addWidget(add_widget) + add_widget.setParent(obj.form) + # obj.table_widget.formlayout.addWidget(add_widget) + obj.form.layout().addWidget(add_widget) + if reagent.missing: + missing_reagents.append(reagent) logger.debug(f"Checking integrity of {obj.ext_kit}") + # TODO: put check_kit_integrity here instead of what's here? # see if there are any missing reagents - if len(obj.missing_reagents) > 0: - result = dict(message=f"The submission you are importing is missing some reagents expected by the kit.\n\nIt looks like you are missing: {[item.type.upper() for item in obj.missing_reagents]}\n\nAlternatively, you may have set the wrong extraction kit.\n\nThe program will populate lists using existing reagents.\n\nPlease make sure you check the lots carefully!", status="Warning") + if len(missing_reagents) > 0: + result = dict(message=f"The submission you are importing is missing some reagents expected by the kit.\n\nIt looks like you are missing: {[item.type.upper() for item in missing_reagents]}\n\nAlternatively, you may have set the wrong extraction kit.\n\nThe program will populate lists using existing reagents.\n\nPlease make sure you check the lots carefully!", status="Warning") # for item in obj.missing_reagents: # # Add label that has parsed as False to show "MISSING" label. # obj.table_widget.formlayout.addWidget(ParsedQLabel({'parsed':False}, item.type, title=False, label_name=f"missing_{item.type}")) @@ -238,7 +262,7 @@ def kit_integrity_completion_function(obj:QMainWindow) -> Tuple[QMainWindow, dic # Add submit button to the form. submit_btn = QPushButton("Submit") submit_btn.setObjectName("submit_btn") - obj.table_widget.formlayout.addWidget(submit_btn) + obj.form.layout().addWidget(submit_btn) submit_btn.clicked.connect(obj.submit_new_sample) return obj, result @@ -254,61 +278,25 @@ def submit_new_sample_function(obj:QMainWindow) -> Tuple[QMainWindow, dict]: """ logger.debug(f"\n\nBeginning Submission\n\n") result = None - # extract info from the form widgets - # info = extract_form_info(obj.table_widget.tab1) - # if isinstance(info, tuple): - # logger.warning(f"Got tuple for info for some reason.") - # info = info[0] - # # seperate out reagents - # reagents = {k.replace("lot_", ""):v for k,v in info.items() if k.startswith("lot_")} - # info = {k:v for k,v in info.items() if not k.startswith("lot_")} - # info, reagents = obj.table_widget.formwidget.parse_form() - submission: PydSubmission = obj.table_widget.formwidget.parse_form() - logger.debug(f"Submission: {pprint.pformat(submission)}") - # parsed_reagents = [] - # compare reagents in form to reagent database - # for reagent in submission.reagents: - # # Lookup any existing reagent of this type with this lot number - # wanted_reagent = lookup_reagents(ctx=obj.ctx, lot_number=reagent.lot, reagent_type=reagent.name) - # logger.debug(f"Looked up reagent: {wanted_reagent}") - # # if reagent not found offer to add to database - # if wanted_reagent == None: - # # r_lot = reagent[reagent] - # dlg = QuestionAsker(title=f"Add {reagent.lot}?", message=f"Couldn't find reagent type {reagent.name.strip('Lot')}: {reagent.lot} in the database.\n\nWould you like to add it?") - # if dlg.exec(): - # logger.debug(f"Looking through {pprint.pformat(obj.reagents)} for reagent {reagent.name}") - # try: - # picked_reagent = [item for item in obj.reagents if item.type == reagent.name][0] - # except IndexError: - # logger.error(f"Couldn't find {reagent.name} in obj.reagents. Checking missing reagents {pprint.pformat(obj.missing_reagents)}") - # picked_reagent = [item for item in obj.missing_reagents if item.type == reagent.name][0] - # logger.debug(f"checking reagent: {reagent.name} in obj.reagents. Result: {picked_reagent}") - # expiry_date = picked_reagent.expiry - # wanted_reagent = obj.add_reagent(reagent_lot=reagent.lot, reagent_type=reagent.name.replace("lot_", ""), expiry=expiry_date, name=picked_reagent.name) - # else: - # # In this case we will have an empty reagent and the submission will fail kit integrity check - # logger.debug("Will not add reagent.") - # return obj, dict(message="Failed integrity check", status="critical") - # # Append the PydReagent object o be added to the submission - # parsed_reagents.append(reagent) - # # move samples into preliminary submission dict - # submission.reagents = parsed_reagents - # submission.uploaded_by = getuser() - # construct submission object - # logger.debug(f"Here is the info_dict: {pprint.pformat(info)}") - # base_submission, result = construct_submission_info(ctx=obj.ctx, info_dict=info) - base_submission, result = submission.toSQL() - # delattr(base_submission, "ctx") - # raise ValueError(base_submission.__dict__) + obj.pyd: PydSubmission = obj.form.parse_form() + logger.debug(f"Submission: {pprint.pformat(obj.pyd)}") + logger.debug("Checking kit integrity...") + kit_integrity = check_kit_integrity(ctx=obj.ctx, sub=obj.pyd) + if kit_integrity != None: + return obj, dict(message=kit_integrity['message'], status="critical") + base_submission, result = obj.pyd.toSQL() # check output message for issues match result['code']: + # code 0: everything is fine. + case 0: + result = None # code 1: ask for overwrite case 1: dlg = QuestionAsker(title=f"Review {base_submission.rsl_plate_num}?", message=result['message']) if dlg.exec(): # Do not add duplicate reagents. # base_submission.reagents = [] - pass + result = None else: obj.ctx.database_session.rollback() return obj, dict(message="Overwrite cancelled", status="Information") @@ -321,31 +309,19 @@ def submit_new_sample_function(obj:QMainWindow) -> Tuple[QMainWindow, dict]: for reagent in base_submission.reagents: update_last_used(ctx=obj.ctx, reagent=reagent, kit=base_submission.extraction_kit) logger.debug(f"Here is the final submission: {pprint.pformat(base_submission.__dict__)}") - logger.debug(f"Parsed reagents: {pprint.pformat(base_submission.reagents)}") - logger.debug("Checking kit integrity...") - kit_integrity = check_kit_integrity(base_submission) - if kit_integrity != None: - return obj, dict(message=kit_integrity['message'], status="critical") + logger.debug(f"Parsed reagents: {pprint.pformat(base_submission.reagents)}") logger.debug(f"Sending submission: {base_submission.rsl_plate_num} to database.") - # result = store_object(ctx=obj.ctx, object=base_submission) base_submission.save(ctx=obj.ctx) # update summary sheet obj.table_widget.sub_wid.setData() # reset form - for item in obj.table_widget.formlayout.parentWidget().findChildren(QWidget): - item.setParent(None) + obj.form.setParent(None) logger.debug(f"All attributes of obj: {pprint.pformat(obj.__dict__)}") - if len(obj.missing_reagents + obj.missing_info) > 0: - logger.debug(f"We have blank reagents in the excel sheet.\n\tLet's try to fill them in.") - extraction_kit = lookup_kit_types(ctx=obj.ctx, name=obj.ext_kit) - logger.debug(f"We have the extraction kit: {extraction_kit.name}") - excel_map = extraction_kit.construct_xl_map_for_use(obj.current_submission_type) - logger.debug(f"Extraction kit map:\n\n{pprint.pformat(excel_map)}") - input_reagents = [item.to_reagent_dict(extraction_kit=base_submission.extraction_kit) for item in base_submission.reagents] - logger.debug(f"Parsed reagents going into autofile: {pprint.pformat(input_reagents)}") - # autofill_excel(obj=obj, xl_map=excel_map, reagents=input_reagents, missing_reagents=obj.missing_reagents, info=info, missing_info=obj.missing_info) - autofill_excel(obj=obj, xl_map=excel_map, reagents=input_reagents, missing_reagents=obj.missing_reagents, info=base_submission.__dict__, missing_info=obj.missing_info) - if hasattr(obj, 'csv'): + wkb = obj.pyd.autofill_excel() + if wkb != None: + fname = select_save_file(obj=obj, default_name=obj.pyd.rsl_plate_num['value'], extension="xlsx") + wkb.save(filename=fname.__str__()) + if hasattr(obj.pyd, 'csv'): dlg = QuestionAsker("Export CSV?", "Would you like to export the csv file?") if dlg.exec(): fname = select_save_file(obj, f"{base_submission.rsl_plate_num}.csv", extension="csv") @@ -353,10 +329,6 @@ def submit_new_sample_function(obj:QMainWindow) -> Tuple[QMainWindow, dict]: obj.csv.to_csv(fname.__str__(), index=False) except PermissionError: logger.debug(f"Could not get permissions to {fname}. Possibly the request was cancelled.") - try: - delattr(obj, "csv") - except AttributeError: - pass return obj, result def generate_report_function(obj:QMainWindow) -> Tuple[QMainWindow, dict]: @@ -905,6 +877,8 @@ def autofill_excel(obj:QMainWindow, xl_map:dict, reagents:List[dict], missing_re fname = select_save_file(obj=obj, default_name=info['rsl_plate_num'], extension="xlsx") workbook.save(filename=fname.__str__()) + + def construct_first_strand_function(obj:QMainWindow) -> Tuple[QMainWindow, dict]: """ Generates a csv file from client submitted xlsx file. @@ -1016,23 +990,29 @@ def scrape_reagents(obj:QMainWindow, extraction_kit:str) -> Tuple[QMainWindow, d Returns: Tuple[QMainWindow, dict]: Updated application and result """ - logger.debug("\n\nHello from reagent scraper!!\n\n") logger.debug(f"Extraction kit: {extraction_kit}") - obj.reagents = [] - obj.missing_reagents = [] + # obj.reagents = [] + # obj.missing_reagents = [] # Remove previous reagent widgets - # [item.setParent(None) for item in obj.table_widget.formlayout.parentWidget().findChildren(QWidget) if item.objectName().startswith("lot_") or item.objectName().startswith("missing_")] - # [item.setParent(None) for item in obj.table_widget.formlayout.parentWidget().findChildren(QPushButton)] - reagents = obj.prsr.parse_reagents(extraction_kit=extraction_kit) - logger.debug(f"Got reagents: {reagents}") + try: + old_reagents = obj.form.find_widgets() + except AttributeError: + logger.error(f"Couldn't find old reagents.") + old_reagents = [] + # logger.debug(f"\n\nAttempting to clear: {old_reagents}\n\n") + for reagent in old_reagents: + if isinstance(reagent, ReagentFormWidget) or isinstance(reagent, QPushButton): + reagent.setParent(None) + # reagents = obj.prsr.parse_reagents(extraction_kit=extraction_kit) + # logger.debug(f"Got reagents: {reagents}") # for reagent in obj.prsr.sub['reagents']: # # create label # if reagent.parsed: # obj.reagents.append(reagent) # else: # obj.missing_reagents.append(reagent) - obj.reagents = obj.prsr.sub['reagents'] - logger.debug(f"Imported reagents: {obj.reagents}") - logger.debug(f"Missing reagents: {obj.missing_reagents}") + obj.form.reagents = obj.prsr.sub['reagents'] + # logger.debug(f"Imported reagents: {obj.reagents}") + # logger.debug(f"Missing reagents: {obj.missing_reagents}") return obj, None