landon/Submissions-App
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pre-cleanup

Browse Source
This commit is contained in:
Landon Wark
2023-11-01 08:59:58 -05:00
parent f3a7d75c6a
commit 22a23b7838
18 changed files with 665 additions and 636 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@@ -1,3 +1,7 @@
## 202311.01
- Kit integrity is now checked before creation of sql object to improve reagent type lookups.
## 202310.03
- Better flexibility with parsers pulling methods from database objects.

10
TODO.md
View File

@@ -1,10 +1,14 @@
- [ ] Make the kit verifier make more sense.
- [ ] Slim down the Import and Submit functions in main_window_functions.
- [ ] Update artic submission type database entry to
- [ ] Document code
- [x] Rewrite tests... again.
- [x] Have InfoItem change status self.missing to True if value changed.
- [x] Make the kit verifier make more sense.
- [x] Slim down the Import and Submit functions in main_window_functions.
- [x] Create custom store methods for submission, reagent and sample.
- [x] Make pydantic models for other things that use constructors.
- [x] Move backend.db.functions.constructor functions into Pydantic models.
- This will allow for better data validation.
- Parser -> Pydantic(validation) -> Form(user input) -> Pydantic(validation) -> SQL
- Parser(client input) -> Pydantic(validation) -> Form(user input) -> Pydantic(validation) -> SQL
- [x] Rebuild RSLNamer and fix circular imports
- Should be used when coming in to parser and when leaving form. NO OTHER PLACES.
- [x] Change 'check_is_power_user' to decorator.

32
alembic/versions/8a5bc2924ef9_adding_artic_technician_to_artic.py Normal file
View File

@@ -0,0 +1,32 @@
"""adding artic_technician to Artic
Revision ID: 8a5bc2924ef9
Revises: b95478ffb4a3
Create Date: 2023-10-31 13:59:47.746122
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '8a5bc2924ef9'
down_revision = 'b95478ffb4a3'
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('_submissions', schema=None) as batch_op:
batch_op.add_column(sa.Column('artic_technician', sa.String(length=64), nullable=True))
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('_submissions', schema=None) as batch_op:
batch_op.drop_column('artic_technician')
# ### end Alembic commands ###

21
source/submissions.backend.validators.rst Normal file
View File

@@ -0,0 +1,21 @@
submissions.backend.validators package
======================================
Submodules
----------
submissions.backend.validators.pydant module
--------------------------------------------
.. automodule:: submissions.backend.validators.pydant
:members:
:undoc-members:
:show-inheritance:
Module contents
---------------
.. automodule:: submissions.backend.validators
:members:
:undoc-members:
:show-inheritance:

2
src/submissions/__init__.py
View File

@@ -4,7 +4,7 @@ from pathlib import Path
# Version of the realpython-reader package
__project__ = "submissions"
__version__ = "202310.4b"
__version__ = "202311.1b"
__author__ = {"name":"Landon Wark", "email":"Landon.Wark@phac-aspc.gc.ca"}
__copyright__ = "2022-2023, Government of Canada"

2
src/submissions/backend/db/__init__.py
View File

@@ -1,4 +1,4 @@
'''
All database related operations.
'''
from .functions import *
# from .functions import *

280
src/submissions/backend/db/functions/constructions.py
View File

@@ -1,280 +0,0 @@
'''
Used to construct models from input dictionaries.
'''
from tools import Settings, check_regex_match, check_authorization, massage_common_reagents
from .. import models
from .lookups import *
import logging
from datetime import date, timedelta
from dateutil.parser import parse
from typing import Tuple
from sqlalchemy.exc import IntegrityError, SAWarning
from . import store_object
from backend.validators import RSLNamer
logger = logging.getLogger(f"submissions.{__name__}")
# def construct_reagent(ctx:Settings, info_dict:dict) -> models.Reagent:
# """
# Construct reagent object from dictionary
# NOTE: Depreciated in favour of Pydantic model .toSQL method
# Args:
# ctx (Settings): settings object passed down from gui
# info_dict (dict): dictionary to be converted
# Returns:
# models.Reagent: Constructed reagent object
# """
# reagent = models.Reagent()
# for item in info_dict:
# logger.debug(f"Reagent info item for {item}: {info_dict[item]}")
# # set fields based on keys in dictionary
# match item:
# case "lot":
# reagent.lot = info_dict[item].upper()
# case "expiry":
# if isinstance(info_dict[item], date):
# reagent.expiry = info_dict[item]
# else:
# reagent.expiry = parse(info_dict[item]).date()
# case "type":
# reagent_type = lookup_reagent_types(ctx=ctx, name=info_dict[item])
# if reagent_type != None:
# reagent.type.append(reagent_type)
# case "name":
# if item == None:
# reagent.name = reagent.type.name
# else:
# reagent.name = info_dict[item]
# # add end-of-life extension from reagent type to expiry date
# # NOTE: this will now be done only in the reporting phase to account for potential changes in end-of-life extensions
# return reagent
# def construct_submission_info(ctx:Settings, info_dict:dict) -> Tuple[models.BasicSubmission, dict]:
# """
# Construct submission object from dictionary pulled from gui form
# NOTE: Depreciated in favour of Pydantic model .toSQL method
# Args:
# ctx (Settings): settings object passed down from gui
# info_dict (dict): dictionary to be transformed
# Returns:
# models.BasicSubmission: Constructed submission object
# """
# # convert submission type into model name
# # model = get_polymorphic_subclass(polymorphic_identity=info_dict['submission_type'])
# model = models.BasicSubmission.find_polymorphic_subclass(polymorphic_identity=info_dict['submission_type'])
# logger.debug(f"We've got the model: {type(model)}")
# # Ensure an rsl plate number exists for the plate
# if not check_regex_match("^RSL", info_dict["rsl_plate_num"]):
# instance = None
# msg = "A proper RSL plate number is required."
# return instance, {'code': 2, 'message': "A proper RSL plate number is required."}
# else:
# # # enforce conventions on the rsl plate number from the form
# # # info_dict['rsl_plate_num'] = RSLNamer(ctx=ctx, instr=info_dict["rsl_plate_num"]).parsed_name
# info_dict['rsl_plate_num'] = RSLNamer(ctx=ctx, instr=info_dict["rsl_plate_num"], sub_type=info_dict['submission_type']).parsed_name
# # check database for existing object
# instance = lookup_submissions(ctx=ctx, rsl_number=info_dict['rsl_plate_num'])
# # get model based on submission type converted above
# # logger.debug(f"Looking at models for submission type: {query}")
# # if query return nothing, ie doesn't already exist in db
# if instance == None:
# instance = model()
# logger.debug(f"Submission doesn't exist yet, creating new instance: {instance}")
# msg = None
# code = 0
# else:
# code = 1
# msg = "This submission already exists.\nWould you like to overwrite?"
# for item in info_dict:
# value = info_dict[item]
# logger.debug(f"Setting {item} to {value}")
# # set fields based on keys in dictionary
# match item:
# case "extraction_kit":
# logger.debug(f"Looking up kit {value}")
# field_value = lookup_kit_types(ctx=ctx, name=value)
# logger.debug(f"Got {field_value} for kit {value}")
# case "submitting_lab":
# logger.debug(f"Looking up organization: {value}")
# field_value = lookup_organizations(ctx=ctx, name=value)
# logger.debug(f"Got {field_value} for organization {value}")
# case "submitter_plate_num":
# logger.debug(f"Submitter plate id: {value}")
# field_value = value
# case "samples":
# instance = construct_samples(ctx=ctx, instance=instance, samples=value)
# continue
# case "submission_type":
# field_value = lookup_submission_type(ctx=ctx, name=value)
# case _:
# field_value = value
# # insert into field
# try:
# setattr(instance, item, field_value)
# except AttributeError:
# logger.debug(f"Could not set attribute: {item} to {info_dict[item]}")
# continue
# except KeyError:
# continue
# # calculate cost of the run: immutable cost + mutable times number of columns
# # This is now attached to submission upon creation to preserve at-run costs incase of cost increase in the future.
# try:
# logger.debug(f"Calculating costs for procedure...")
# instance.calculate_base_cost()
# except (TypeError, AttributeError) as e:
# logger.debug(f"Looks like that kit doesn't have cost breakdown yet due to: {e}, using full plate cost.")
# instance.run_cost = instance.extraction_kit.cost_per_run
# logger.debug(f"Calculated base run cost of: {instance.run_cost}")
# # Apply any discounts that are applicable for client and kit.
# try:
# logger.debug("Checking and applying discounts...")
# discounts = [item.amount for item in lookup_discounts(ctx=ctx, kit_type=instance.extraction_kit, organization=instance.submitting_lab)]
# logger.debug(f"We got discounts: {discounts}")
# if len(discounts) > 0:
# discounts = sum(discounts)
# instance.run_cost = instance.run_cost - discounts
# except Exception as e:
# logger.error(f"An unknown exception occurred when calculating discounts: {e}")
# # We need to make sure there's a proper rsl plate number
# logger.debug(f"We've got a total cost of {instance.run_cost}")
# try:
# logger.debug(f"Constructed instance: {instance.to_string()}")
# except AttributeError as e:
# logger.debug(f"Something went wrong constructing instance {info_dict['rsl_plate_num']}: {e}")
# logger.debug(f"Constructed submissions message: {msg}")
# return instance, {'code':code, 'message':msg}
# def construct_samples(ctx:Settings, instance:models.BasicSubmission, samples:List[dict]) -> models.BasicSubmission:
# """
# constructs sample objects and adds to submission
# NOTE: Depreciated in favour of Pydantic model .toSQL method
# Args:
# ctx (Settings): settings passed down from gui
# instance (models.BasicSubmission): Submission samples scraped from.
# samples (List[dict]): List of parsed samples
# Returns:
# models.BasicSubmission: Updated submission object.
# """
# for sample in samples:
# sample_instance = lookup_samples(ctx=ctx, submitter_id=str(sample['sample'].submitter_id))
# if sample_instance == None:
# sample_instance = sample['sample']
# else:
# logger.warning(f"Sample {sample} already exists, creating association.")
# logger.debug(f"Adding {sample_instance.__dict__}")
# if sample_instance in instance.samples:
# logger.error(f"Looks like there's a duplicate sample on this plate: {sample_instance.submitter_id}!")
# continue
# try:
# with ctx.database_session.no_autoflush:
# try:
# sample_query = sample_instance.sample_type.replace('Sample', '').strip()
# logger.debug(f"Here is the sample instance type: {sample_instance}")
# try:
# assoc = getattr(models, f"{sample_query}Association")
# except AttributeError as e:
# logger.error(f"Couldn't get type specific association using {sample_instance.sample_type.replace('Sample', '').strip()}. Getting generic.")
# assoc = models.SubmissionSampleAssociation
# assoc = assoc(submission=instance, sample=sample_instance, row=sample['row'], column=sample['column'])
# instance.submission_sample_associations.append(assoc)
# except IntegrityError:
# logger.error(f"Hit integrity error for: {sample}")
# continue
# except SAWarning:
# logger.error(f"Looks like the association already exists for submission: {instance} and sample: {sample_instance}")
# continue
# except IntegrityError as e:
# logger.critical(e)
# continue
# return instance
# @check_authorization
# def construct_kit_from_yaml(ctx:Settings, kit_dict:dict) -> dict:
# """
# Create and store a new kit in the database based on a .yml file
# TODO: split into create and store functions
# Args:
# ctx (Settings): Context object passed down from frontend
# kit_dict (dict): Experiment dictionary created from yaml file
# Returns:
# dict: a dictionary containing results of db addition
# """
# # from tools import check_is_power_user, massage_common_reagents
# # Don't want just anyone adding kits
# # if not check_is_power_user(ctx=ctx):
# # logger.debug(f"{getuser()} does not have permission to add kits.")
# # return {'code':1, 'message':"This user does not have permission to add kits.", "status":"warning"}
# submission_type = lookup_submission_type(ctx=ctx, name=kit_dict['used_for'])
# logger.debug(f"Looked up submission type: {kit_dict['used_for']} and got {submission_type}")
# kit = models.KitType(name=kit_dict["kit_name"])
# kt_st_assoc = models.SubmissionTypeKitTypeAssociation(kit_type=kit, submission_type=submission_type)
# for k,v in kit_dict.items():
# if k not in ["reagent_types", "kit_name", "used_for"]:
# kt_st_assoc.set_attrib(k, v)
# kit.kit_submissiontype_associations.append(kt_st_assoc)
# # A kit contains multiple reagent types.
# for r in kit_dict['reagent_types']:
# logger.debug(f"Constructing reagent type: {r}")
# rtname = massage_common_reagents(r['rtname'])
# look_up = lookup_reagent_types(name=rtname)
# if look_up == None:
# rt = models.ReagentType(name=rtname.strip(), eol_ext=timedelta(30*r['eol']))
# else:
# rt = look_up
# uses = {kit_dict['used_for']:{k:v for k,v in r.items() if k not in ['eol']}}
# assoc = models.KitTypeReagentTypeAssociation(kit_type=kit, reagent_type=rt, uses=uses)
# # ctx.database_session.add(rt)
# store_object(ctx=ctx, object=rt)
# kit.kit_reagenttype_associations.append(assoc)
# logger.debug(f"Kit construction reagent type: {rt.__dict__}")
# logger.debug(f"Kit construction kit: {kit.__dict__}")
# store_object(ctx=ctx, object=kit)
# return {'code':0, 'message':'Kit has been added', 'status': 'information'}
# @check_authorization
# def construct_org_from_yaml(ctx:Settings, org:dict) -> dict:
# """
# Create and store a new organization based on a .yml file
# Args:
# ctx (Settings): Context object passed down from frontend
# org (dict): Dictionary containing organization info.
# Returns:
# dict: dictionary containing results of db addition
# """
# # from tools import check_is_power_user
# # # Don't want just anyone adding in clients
# # if not check_is_power_user(ctx=ctx):
# # logger.debug(f"{getuser()} does not have permission to add kits.")
# # return {'code':1, 'message':"This user does not have permission to add organizations."}
# # the yml can contain multiple clients
# for client in org:
# cli_org = models.Organization(name=client.replace(" ", "_").lower(), cost_centre=org[client]['cost centre'])
# # a client can contain multiple contacts
# for contact in org[client]['contacts']:
# cont_name = list(contact.keys())[0]
# # check if contact already exists
# look_up = ctx.database_session.query(models.Contact).filter(models.Contact.name==cont_name).first()
# if look_up == None:
# cli_cont = models.Contact(name=cont_name, phone=contact[cont_name]['phone'], email=contact[cont_name]['email'], organization=[cli_org])
# else:
# cli_cont = look_up
# cli_cont.organization.append(cli_org)
# ctx.database_session.add(cli_cont)
# logger.debug(f"Client creation contact: {cli_cont.__dict__}")
# logger.debug(f"Client creation client: {cli_org.__dict__}")
# ctx.database_session.add(cli_org)
# ctx.database_session.commit()
# return {"code":0, "message":"Organization has been added."}

7
src/submissions/backend/db/functions/lookups.py
View File

@@ -141,7 +141,10 @@ def lookup_reagent_types(ctx:Settings,
# logger.debug(f"Reagent reagent types: {reagent._sa_instance_state}")
result = list(set(kit_type.reagent_types).intersection(reagent.type))
logger.debug(f"Result: {result}")
return result[0]
try:
return result[0]
except IndexError:
return result
match name:
case str():
logger.debug(f"Looking up reagent type by name: {name}")
@@ -249,7 +252,7 @@ def lookup_submissions(ctx:Settings,
if chronologic:
# query.order_by(models.BasicSubmission.submitted_date)
query.order_by(model.submitted_date)
logger.debug(f"At the end of the search, the query gets: {query.all()}")
# logger.debug(f"At the end of the search, the query gets: {query.all()}")
return query_return(query=query, limit=limit)
def lookup_submission_type(ctx:Settings,

21
src/submissions/backend/db/functions/misc.py
View File

@@ -1,6 +1,7 @@
'''
Contains convenience functions for using database
'''
import sys
from tools import Settings
from .lookups import *
import pandas as pd
@@ -13,6 +14,8 @@ from sqlalchemy.exc import OperationalError as AlcOperationalError, IntegrityErr
from sqlite3 import OperationalError as SQLOperationalError, IntegrityError as SQLIntegrityError
from pprint import pformat
import logging
from backend.validators import pydant
logger = logging.getLogger(f"submissions.{__name__}")
@@ -172,7 +175,7 @@ def update_ww_sample(ctx:Settings, sample_obj:dict) -> dict|None:
result = store_object(ctx=ctx, object=assoc)
return result
def check_kit_integrity(sub:models.BasicSubmission|models.KitType, reagenttypes:list|None=None) -> dict|None:
def check_kit_integrity(ctx:Settings, sub:models.BasicSubmission|models.KitType|pydant.PydSubmission, reagenttypes:list=[]) -> dict|None:
"""
Ensures all reagents expected in kit are listed in Submission
@@ -185,20 +188,30 @@ def check_kit_integrity(sub:models.BasicSubmission|models.KitType, reagenttypes:
"""
logger.debug(type(sub))
# What type is sub?
reagenttypes = []
# reagenttypes = []
match sub:
case pydant.PydSubmission():
ext_kit = lookup_kit_types(ctx=ctx, name=sub.extraction_kit['value'])
ext_kit_rtypes = [item.name for item in ext_kit.get_reagents(required=True, submission_type=sub.submission_type['value'])]
reagenttypes = [item.type for item in sub.reagents]
case models.BasicSubmission():
# Get all required reagent types for this kit.
ext_kit_rtypes = [item.name for item in sub.extraction_kit.get_reagents(required=True, submission_type=sub.submission_type_name)]
# Overwrite function parameter reagenttypes
for reagent in sub.reagents:
logger.debug(f"For kit integrity, looking up reagent: {reagent}")
try:
rt = list(set(reagent.type).intersection(sub.extraction_kit.reagent_types))[0].name
# rt = list(set(reagent.type).intersection(sub.extraction_kit.reagent_types))[0].name
rt = lookup_reagent_types(ctx=ctx, kit_type=sub.extraction_kit, reagent=reagent)
logger.debug(f"Got reagent type: {rt}")
reagenttypes.append(rt)
if isinstance(rt, models.ReagentType):
reagenttypes.append(rt.name)
except AttributeError as e:
logger.error(f"Problem parsing reagents: {[f'{reagent.lot}, {reagent.type}' for reagent in sub.reagents]}")
reagenttypes.append(reagent.type[0].name)
except IndexError:
logger.error(f"No intersection of {reagent} type {reagent.type} and {sub.extraction_kit.reagent_types}")
raise ValueError(f"No intersection of {reagent} type {reagent.type} and {sub.extraction_kit.reagent_types}")
case models.KitType():
ext_kit_rtypes = [item.name for item in sub.get_reagents(required=True)]
case _:

4
src/submissions/backend/db/models/kits.py
View File

@@ -248,7 +248,7 @@ class Reagent(Base):
"expiry": place_holder.strftime("%Y-%m-%d")
}
def to_reagent_dict(self, extraction_kit:KitType=None) -> dict:
def to_reagent_dict(self, extraction_kit:KitType|str=None) -> dict:
"""
Returns basic reagent dictionary.
@@ -314,6 +314,7 @@ class SubmissionType(Base):
name = Column(String(128), unique=True) #: name of submission type
info_map = Column(JSON) #: Where basic information is found in the excel workbook corresponding to this type.
instances = relationship("BasicSubmission", backref="submission_type")
# regex = Column(String(512))
submissiontype_kit_associations = relationship(
"SubmissionTypeKitTypeAssociation",
@@ -326,6 +327,7 @@ class SubmissionType(Base):
def __repr__(self) -> str:
return f"<SubmissionType({self.name})>"
class SubmissionTypeKitTypeAssociation(Base):
"""
Abstract of relationship between kits and their submission type.

18
src/submissions/backend/db/models/submissions.py
View File

@@ -47,6 +47,7 @@ class BasicSubmission(Base):
reagents = relationship("Reagent", back_populates="submissions", secondary=reagents_submissions) #: relationship to reagents
reagents_id = Column(String, ForeignKey("_reagents.id", ondelete="SET NULL", name="fk_BS_reagents_id")) #: id of used reagents
extraction_info = Column(JSON) #: unstructured output from the extraction table logger.
pcr_info = Column(JSON) #: unstructured output from pcr table logger or user(Artic)
run_cost = Column(FLOAT(2)) #: total cost of running the plate. Set from constant and mutable kit costs at time of creation.
uploaded_by = Column(String(32)) #: user name of person who submitted the submission to the database.
comment = Column(JSON)
@@ -211,12 +212,12 @@ class BasicSubmission(Base):
Calculate the number of columns in this submission
Returns:
int: largest column number
int: Number of unique columns.
"""
logger.debug(f"Here's the samples: {self.samples}")
columns = [assoc.column for assoc in self.submission_sample_associations]
columns = set([assoc.column for assoc in self.submission_sample_associations])
logger.debug(f"Here are the columns for {self.rsl_plate_num}: {columns}")
return max(columns)
return len(columns)
def hitpick_plate(self, plate_number:int|None=None) -> list:
"""
@@ -281,7 +282,7 @@ class BasicSubmission(Base):
Returns:
dict: Updated sample dictionary
"""
logger.debug(f"Called {cls.__name__} sample parser")
# logger.debug(f"Called {cls.__name__} sample parser")
return input_dict
@classmethod
@@ -461,7 +462,7 @@ class Wastewater(BasicSubmission):
"""
derivative submission type from BasicSubmission
"""
pcr_info = Column(JSON)
# pcr_info = Column(JSON)
ext_technician = Column(String(64))
pcr_technician = Column(String(64))
__mapper_args__ = {"polymorphic_identity": "Wastewater", "polymorphic_load": "inline"}
@@ -570,13 +571,16 @@ class Wastewater(BasicSubmission):
@classmethod
def get_regex(cls):
return "(?P<Wastewater>RSL(?:-|_)?WW(?:-|_)?20\d{2}-?\d{2}-?\d{2}(?:(_|-)\d?(\D|$)R?\d?)?)"
# return "(?P<Wastewater>RSL(?:-|_)?WW(?:-|_)?20\d{2}-?\d{2}-?\d{2}(?:(_|-)\d?(\D|$)R?\d?)?)"
# return "(?P<Wastewater>RSL(?:-|_)?WW(?:-|_)?20\d{2}-?\d{2}-?\d{2}(?:(_|-)\d?([^_|\D]|$)R?\d?)?)"
return "(?P<Wastewater>RSL(?:-|_)?WW(?:-|_)?20\d{2}-?\d{2}-?\d{2}(?:(_|-)?\d?([^_0123456789]|$)R?\d?)?)"
class WastewaterArtic(BasicSubmission):
"""
derivative submission type for artic wastewater
"""
__mapper_args__ = {"polymorphic_identity": "Wastewater Artic", "polymorphic_load": "inline"}
artic_technician = Column(String(64))
def calculate_base_cost(self):
"""
@@ -752,7 +756,7 @@ class BasicSample(Base):
@classmethod
def parse_sample(cls, input_dict:dict) -> dict:
logger.debug(f"Called {cls.__name__} sample parser")
# logger.debug(f"Called {cls.__name__} sample parser")
return input_dict
class WastewaterSample(BasicSample):

47
src/submissions/backend/excel/parser.py
View File

@@ -7,7 +7,8 @@ from typing import List
import pandas as pd
import numpy as np
from pathlib import Path
from backend.db import models, lookup_kit_types, lookup_submission_type, lookup_samples
from backend.db import models
from backend.db.functions import lookup_kit_types, lookup_submission_type, lookup_samples
from backend.validators import PydSubmission, PydReagent, RSLNamer, PydSample
import logging
from collections import OrderedDict
@@ -32,7 +33,7 @@ class SheetParser(object):
filepath (Path | None, optional): file path to excel sheet. Defaults to None.
"""
self.ctx = ctx
logger.debug(f"Parsing {filepath.__str__()}")
logger.debug(f"\n\nParsing {filepath.__str__()}\n\n")
match filepath:
case Path():
self.filepath = filepath
@@ -48,7 +49,7 @@ class SheetParser(object):
raise FileNotFoundError(f"Couldn't parse file {self.filepath}")
self.sub = OrderedDict()
# make decision about type of sample we have
self.sub['submission_type'] = dict(value=RSLNamer.retrieve_submission_type(ctx=self.ctx, instr=self.filepath), parsed=False)
self.sub['submission_type'] = dict(value=RSLNamer.retrieve_submission_type(ctx=self.ctx, instr=self.filepath), missing=True)
# # grab the info map from the submission type in database
self.parse_info()
self.import_kit_validation_check()
@@ -98,12 +99,12 @@ class SheetParser(object):
if not check_not_nan(self.sub['extraction_kit']['value']):
dlg = KitSelector(ctx=self.ctx, title="Kit Needed", message="At minimum a kit is needed. Please select one.")
if dlg.exec():
self.sub['extraction_kit'] = dict(value=dlg.getValues(), parsed=False)
self.sub['extraction_kit'] = dict(value=dlg.getValues(), missing=True)
else:
raise ValueError("Extraction kit needed.")
else:
if isinstance(self.sub['extraction_kit'], str):
self.sub['extraction_kit'] = dict(value=self.sub['extraction_kit'], parsed=False)
self.sub['extraction_kit'] = dict(value=self.sub['extraction_kit'], missing=True)
def import_reagent_validation_check(self):
"""
@@ -130,6 +131,7 @@ class SheetParser(object):
class InfoParser(object):
def __init__(self, ctx:Settings, xl:pd.ExcelFile, submission_type:str):
logger.debug(f"\n\nHello from InfoParser!")
self.ctx = ctx
self.map = self.fetch_submission_info_map(submission_type=submission_type)
self.xl = xl
@@ -147,7 +149,7 @@ class InfoParser(object):
dict: Location map of all info for this submission type
"""
if isinstance(submission_type, str):
submission_type = dict(value=submission_type, parsed=False)
submission_type = dict(value=submission_type, missing=True)
logger.debug(f"Looking up submission type: {submission_type['value']}")
submission_type = lookup_submission_type(ctx=self.ctx, name=submission_type['value'])
info_map = submission_type.info_map
@@ -168,7 +170,7 @@ class InfoParser(object):
relevant = {}
for k, v in self.map.items():
if isinstance(v, str):
dicto[k] = dict(value=v, parsed=True)
dicto[k] = dict(value=v, missing=False)
continue
if k == "samples":
continue
@@ -183,16 +185,16 @@ class InfoParser(object):
if check_not_nan(value):
if value != "None":
try:
dicto[item] = dict(value=value, parsed=True)
dicto[item] = dict(value=value, missing=False)
except (KeyError, IndexError):
continue
else:
try:
dicto[item] = dict(value=value, parsed=False)
dicto[item] = dict(value=value, missing=True)
except (KeyError, IndexError):
continue
else:
dicto[item] = dict(value=convert_nans_to_nones(value), parsed=False)
dicto[item] = dict(value=convert_nans_to_nones(value), missing=True)
try:
check = dicto['submission_category'] not in ["", None]
except KeyError:
@@ -202,6 +204,7 @@ class InfoParser(object):
class ReagentParser(object):
def __init__(self, ctx:Settings, xl:pd.ExcelFile, submission_type:str, extraction_kit:str):
logger.debug("\n\nHello from ReagentParser!\n\n")
self.ctx = ctx
self.map = self.fetch_kit_info_map(extraction_kit=extraction_kit, submission_type=submission_type)
self.xl = xl
@@ -232,18 +235,18 @@ class ReagentParser(object):
lot = df.iat[relevant[item]['lot']['row']-1, relevant[item]['lot']['column']-1]
expiry = df.iat[relevant[item]['expiry']['row']-1, relevant[item]['expiry']['column']-1]
except (KeyError, IndexError):
listo.append(PydReagent(ctx=self.ctx, type=item.strip(), lot=None, exp=None, name=None, parsed=False))
listo.append(PydReagent(ctx=self.ctx, type=item.strip(), lot=None, expiry=None, name=None, missing=True))
continue
# If the cell is blank tell the PydReagent
if check_not_nan(lot):
parsed = True
missing = False
else:
parsed = False
missing = True
# logger.debug(f"Got lot for {item}-{name}: {lot} as {type(lot)}")
lot = str(lot)
logger.debug(f"Going into pydantic: name: {name}, lot: {lot}, expiry: {expiry}, type: {item.strip()}")
listo.append(PydReagent(ctx=self.ctx, type=item.strip(), lot=lot, expiry=expiry, name=name, parsed=parsed))
logger.debug(f"Returning listo: {listo}")
listo.append(PydReagent(ctx=self.ctx, type=item.strip(), lot=lot, expiry=expiry, name=name, missing=missing))
# logger.debug(f"Returning listo: {listo}")
return listo
class SampleParser(object):
@@ -260,6 +263,7 @@ class SampleParser(object):
df (pd.DataFrame): input sample dataframe
elution_map (pd.DataFrame | None, optional): optional map of elution plate. Defaults to None.
"""
logger.debug("\n\nHello from SampleParser!")
self.samples = []
self.ctx = ctx
self.xl = xl
@@ -310,6 +314,7 @@ class SampleParser(object):
# custom_mapper = get_polymorphic_subclass(models.BasicSubmission, self.submission_type)
custom_mapper = models.BasicSubmission.find_polymorphic_subclass(polymorphic_identity=self.submission_type)
df = custom_mapper.custom_platemap(self.xl, df)
logger.debug(f"Custom platemap:\n{df}")
return df
def construct_lookup_table(self, lookup_table_location:dict) -> pd.DataFrame:
@@ -369,10 +374,10 @@ class SampleParser(object):
for sample in self.samples:
# addition = self.lookup_table[self.lookup_table.isin([sample['submitter_id']]).any(axis=1)].squeeze().to_dict()
addition = self.lookup_table[self.lookup_table.isin([sample['submitter_id']]).any(axis=1)].squeeze()
logger.debug(addition)
# logger.debug(addition)
if isinstance(addition, pd.DataFrame) and not addition.empty:
addition = addition.iloc[0]
logger.debug(f"Lookuptable info: {addition.to_dict()}")
# logger.debug(f"Lookuptable info: {addition.to_dict()}")
for k,v in addition.to_dict().items():
# logger.debug(f"Checking {k} in lookup table.")
if check_not_nan(k) and isinstance(k, str):
@@ -395,7 +400,7 @@ class SampleParser(object):
self.lookup_table.loc[self.lookup_table['Well']==addition['Well']] = np.nan
except (ValueError, KeyError):
pass
logger.debug(f"Output sample dict: {sample}")
# logger.debug(f"Output sample dict: {sample}")
logger.debug(f"Final lookup_table: \n\n {self.lookup_table}")
def parse_samples(self, generate:bool=True) -> List[dict]|List[models.BasicSample]:
@@ -432,11 +437,7 @@ class SampleParser(object):
translated_dict['sample_type'] = f"{self.submission_type} Sample"
translated_dict = self.custom_sub_parser(translated_dict)
translated_dict = self.custom_sample_parser(translated_dict)
logger.debug(f"Here is the output of the custom parser: \n\n{translated_dict}\n\n")
# if generate:
# new_samples.append(self.generate_sample_object(translated_dict))
# else:
# new_samples.append(translated_dict)
# logger.debug(f"Here is the output of the custom parser:\n{translated_dict}")
new_samples.append(PydSample(**translated_dict))
return result, new_samples

4
src/submissions/backend/validators/__init__.py
View File

@@ -18,7 +18,7 @@ class RSLNamer(object):
if self.submission_type == None:
self.submission_type = self.retrieve_submission_type(ctx=self.ctx, instr=instr)
print(self.submission_type)
logger.debug(f"got submission type: {self.submission_type}")
if self.submission_type != None:
enforcer = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=self.submission_type)
self.parsed_name = self.retrieve_rsl_number(instr=instr, regex=enforcer.get_regex())
@@ -67,10 +67,12 @@ class RSLNamer(object):
Args:
in_str (str): string to be parsed
"""
logger.debug(f"Input string to be parsed: {instr}")
if regex == None:
regex = BasicSubmission.construct_regex()
else:
regex = re.compile(rf'{regex}', re.IGNORECASE | re.VERBOSE)
logger.debug(f"Using regex: {regex}")
match instr:
case Path():
m = regex.search(instr.stem)

146
src/submissions/backend/validators/pydant.py
View File

@@ -20,6 +20,8 @@ from backend.db.functions import (lookup_submissions, lookup_reagent_types, look
from backend.db.models import *
from sqlalchemy.exc import InvalidRequestError, StatementError
from PyQt6.QtWidgets import QComboBox, QWidget, QLabel, QVBoxLayout
from pprint import pformat
from openpyxl import load_workbook
logger = logging.getLogger(f"submissions.{__name__}")
@@ -29,7 +31,7 @@ class PydReagent(BaseModel):
type: str|None
expiry: date|None
name: str|None
parsed: bool = Field(default=False)
missing: bool = Field(default=True)
@field_validator("type", mode='before')
@classmethod
@@ -134,6 +136,11 @@ class PydSample(BaseModel, extra='allow'):
return [value]
return value
@field_validator("submitter_id", mode="before")
@classmethod
def int_to_str(cls, value):
return str(value)
def toSQL(self, ctx:Settings, submission):
result = None
self.__dict__.update(self.model_extra)
@@ -165,14 +172,14 @@ class PydSubmission(BaseModel, extra='allow'):
filepath: Path
submission_type: dict|None
# For defaults
submitter_plate_num: dict|None = Field(default=dict(value=None, parsed=False), validate_default=True)
rsl_plate_num: dict|None = Field(default=dict(value=None, parsed=False), validate_default=True)
submitter_plate_num: dict|None = Field(default=dict(value=None, missing=True), validate_default=True)
rsl_plate_num: dict|None = Field(default=dict(value=None, missing=True), validate_default=True)
submitted_date: dict|None
submitting_lab: dict|None
sample_count: dict|None
extraction_kit: dict|None
technician: dict|None
submission_category: dict|None = Field(default=dict(value=None, parsed=False), validate_default=True)
submission_category: dict|None = Field(default=dict(value=None, missing=True), validate_default=True)
reagents: List[dict]|List[PydReagent] = []
samples: List[Any]
@@ -181,7 +188,7 @@ class PydSubmission(BaseModel, extra='allow'):
def enforce_with_uuid(cls, value):
logger.debug(f"submitter plate id: {value}")
if value['value'] == None or value['value'] == "None":
return dict(value=uuid.uuid4().hex.upper(), parsed=False)
return dict(value=uuid.uuid4().hex.upper(), missing=True)
else:
return value
@@ -189,7 +196,7 @@ class PydSubmission(BaseModel, extra='allow'):
@classmethod
def rescue_date(cls, value):
if value == None:
return dict(value=date.today(), parsed=False)
return dict(value=date.today(), missing=True)
return value
@field_validator("submitted_date")
@@ -200,14 +207,14 @@ class PydSubmission(BaseModel, extra='allow'):
if isinstance(value['value'], date):
return value
if isinstance(value['value'], int):
return dict(value=datetime.fromordinal(datetime(1900, 1, 1).toordinal() + value['value'] - 2).date(), parsed=False)
return dict(value=datetime.fromordinal(datetime(1900, 1, 1).toordinal() + value['value'] - 2).date(), missing=True)
string = re.sub(r"(_|-)\d$", "", value['value'])
try:
output = dict(value=parse(string).date(), parsed=False)
output = dict(value=parse(string).date(), missing=True)
except ParserError as e:
logger.error(f"Problem parsing date: {e}")
try:
output = dict(value=parse(string.replace("-","")).date(), parsed=False)
output = dict(value=parse(string.replace("-","")).date(), missing=True)
except Exception as e:
logger.error(f"Problem with parse fallback: {e}")
return output
@@ -216,14 +223,14 @@ class PydSubmission(BaseModel, extra='allow'):
@classmethod
def rescue_submitting_lab(cls, value):
if value == None:
return dict(value=None, parsed=False)
return dict(value=None, missing=True)
return value
@field_validator("rsl_plate_num", mode='before')
@classmethod
def rescue_rsl_number(cls, value):
if value == None:
return dict(value=None, parsed=False)
return dict(value=None, missing=True)
return value
@field_validator("rsl_plate_num")
@@ -233,21 +240,21 @@ class PydSubmission(BaseModel, extra='allow'):
sub_type = values.data['submission_type']['value']
if check_not_nan(value['value']):
if lookup_submissions(ctx=values.data['ctx'], rsl_number=value['value']) == None:
return dict(value=value['value'], parsed=True)
return dict(value=value['value'], missing=False)
else:
logger.warning(f"Submission number {value} already exists in DB, attempting salvage with filepath")
# output = RSLNamer(ctx=values.data['ctx'], instr=values.data['filepath'].__str__(), sub_type=sub_type).parsed_name
output = RSLNamer(ctx=values.data['ctx'], instr=values.data['filepath'].__str__(), sub_type=sub_type).parsed_name
return dict(value=output, parsed=False)
return dict(value=output, missing=True)
else:
output = RSLNamer(ctx=values.data['ctx'], instr=values.data['filepath'].__str__(), sub_type=sub_type).parsed_name
return dict(value=output, parsed=False)
return dict(value=output, missing=True)
@field_validator("technician", mode="before")
@classmethod
def rescue_tech(cls, value):
if value == None:
return dict(value=None, parsed=False)
return dict(value=None, missing=True)
return value
@field_validator("technician")
@@ -257,14 +264,14 @@ class PydSubmission(BaseModel, extra='allow'):
value['value'] = re.sub(r"\: \d", "", value['value'])
return value
else:
return dict(value=convert_nans_to_nones(value['value']), parsed=False)
return dict(value=convert_nans_to_nones(value['value']), missing=True)
return value
@field_validator("sample_count", mode='before')
@classmethod
def rescue_sample_count(cls, value):
if value == None:
return dict(value=None, parsed=False)
return dict(value=None, missing=True)
return value
@field_validator("extraction_kit", mode='before')
@@ -273,13 +280,13 @@ class PydSubmission(BaseModel, extra='allow'):
if check_not_nan(value):
if isinstance(value, str):
return dict(value=value, parsed=True)
return dict(value=value, missing=False)
elif isinstance(value, dict):
return value
else:
raise ValueError(f"No extraction kit found.")
if value == None:
return dict(value=None, parsed=False)
return dict(value=None, missing=True)
return value
@field_validator("submission_type", mode='before')
@@ -289,11 +296,11 @@ class PydSubmission(BaseModel, extra='allow'):
value = {"value": value}
if check_not_nan(value['value']):
value = value['value'].title()
return dict(value=value, parsed=True)
return dict(value=value, missing=False)
# else:
# return dict(value="RSL Name not found.")
else:
return dict(value=RSLNamer(ctx=values.data['ctx'], instr=values.data['filepath'].__str__()).submission_type.title(), parsed=False)
return dict(value=RSLNamer(ctx=values.data['ctx'], instr=values.data['filepath'].__str__()).submission_type.title(), missing=True)
@field_validator("submission_category")
@classmethod
@@ -318,9 +325,21 @@ class PydSubmission(BaseModel, extra='allow'):
output.append(dummy)
self.samples = output
def improved_dict(self):
fields = list(self.model_fields.keys()) + list(self.model_extra.keys())
output = {k:getattr(self, k) for k in fields}
return output
def find_missing(self):
info = {k:v for k,v in self.improved_dict().items() if isinstance(v, dict)}
missing_info = {k:v for k,v in info.items() if v['missing']}
missing_reagents = [reagent for reagent in self.reagents if reagent.missing]
return missing_info, missing_reagents
def toSQL(self):
code = 0
msg = None
status = None
self.__dict__.update(self.model_extra)
instance = lookup_submissions(ctx=self.ctx, rsl_number=self.rsl_plate_num['value'])
if instance == None:
@@ -358,6 +377,11 @@ class PydSubmission(BaseModel, extra='allow'):
field_value = [reagent['value'].toSQL()[0] if isinstance(reagent, dict) else reagent.toSQL()[0] for reagent in value]
case "submission_type":
field_value = lookup_submission_type(ctx=self.ctx, name=value)
case "sample_count":
if value == None:
field_value = len(self.samples)
else:
field_value = value
case "ctx" | "csv" | "filepath":
continue
case _:
@@ -394,9 +418,85 @@ class PydSubmission(BaseModel, extra='allow'):
except AttributeError as e:
logger.debug(f"Something went wrong constructing instance {self.rsl_plate_num}: {e}")
logger.debug(f"Constructed submissions message: {msg}")
return instance, {'code':code, 'message':msg}
return instance, {'code':code, 'message':msg, 'status':"Information"}
def toForm(self):
def toForm(self, parent:QWidget):
from frontend.custom_widgets.misc import SubmissionFormWidget
return SubmissionFormWidget(parent=parent, **self.improved_dict())
def autofill_excel(self, missing_only:bool=True):
if missing_only:
info, reagents = self.find_missing()
else:
info = {k:v for k,v in self.improved_dict().items() if isinstance(v, dict)}
reagents = self.reagents
if len(reagents + list(info.keys())) == 0:
return None
logger.debug(f"We have blank info and/or reagents in the excel sheet.\n\tLet's try to fill them in.")
extraction_kit = lookup_kit_types(ctx=self.ctx, name=self.extraction_kit['value'])
logger.debug(f"We have the extraction kit: {extraction_kit.name}")
excel_map = extraction_kit.construct_xl_map_for_use(self.submission_type['value'])
logger.debug(f"Extraction kit map:\n\n{pformat(excel_map)}")
logger.debug(f"Missing reagents going into autofile: {pformat(reagents)}")
logger.debug(f"Missing info going into autofile: {pformat(info)}")
new_reagents = []
for reagent in reagents:
new_reagent = {}
new_reagent['type'] = reagent.type
new_reagent['lot'] = excel_map[new_reagent['type']]['lot']
new_reagent['lot']['value'] = reagent.lot
new_reagent['expiry'] = excel_map[new_reagent['type']]['expiry']
new_reagent['expiry']['value'] = reagent.expiry
new_reagent['sheet'] = excel_map[new_reagent['type']]['sheet']
# name is only present for Bacterial Culture
try:
new_reagent['name'] = excel_map[new_reagent['type']]['name']
new_reagent['name']['value'] = reagent.name
except Exception as e:
logger.error(f"Couldn't get name due to {e}")
new_reagents.append(new_reagent)
new_info = []
for k,v in info.items():
try:
new_item = {}
new_item['type'] = k
new_item['location'] = excel_map['info'][k]
new_item['value'] = v['value']
new_info.append(new_item)
except KeyError:
logger.error(f"Unable to fill in {k}, not found in relevant info.")
logger.debug(f"New reagents: {new_reagents}")
logger.debug(f"New info: {new_info}")
# open a new workbook using openpyxl
workbook = load_workbook(self.filepath)
# get list of sheet names
sheets = workbook.sheetnames
# logger.debug(workbook.sheetnames)
for sheet in sheets:
# open sheet
worksheet=workbook[sheet]
# Get relevant reagents for that sheet
sheet_reagents = [item for item in new_reagents if sheet in item['sheet']]
for reagent in sheet_reagents:
# logger.debug(f"Attempting to write lot {reagent['lot']['value']} in: row {reagent['lot']['row']}, column {reagent['lot']['column']}")
worksheet.cell(row=reagent['lot']['row'], column=reagent['lot']['column'], value=reagent['lot']['value'])
# logger.debug(f"Attempting to write expiry {reagent['expiry']['value']} in: row {reagent['expiry']['row']}, column {reagent['expiry']['column']}")
worksheet.cell(row=reagent['expiry']['row'], column=reagent['expiry']['column'], value=reagent['expiry']['value'])
try:
# logger.debug(f"Attempting to write name {reagent['name']['value']} in: row {reagent['name']['row']}, column {reagent['name']['column']}")
worksheet.cell(row=reagent['name']['row'], column=reagent['name']['column'], value=reagent['name']['value'])
except Exception as e:
logger.error(f"Could not write name {reagent['name']['value']} due to {e}")
# Get relevant info for that sheet
sheet_info = [item for item in new_info if sheet in item['location']['sheets']]
for item in sheet_info:
logger.debug(f"Attempting: {item['type']} in row {item['location']['row']}, column {item['location']['column']}")
worksheet.cell(row=item['location']['row'], column=item['location']['column'], value=item['value'])
# Hacky way to pop in 'signed by'
# custom_parser = get_polymorphic_subclass(BasicSubmission, info['submission_type'])
custom_parser = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=self.submission_type['value'])
workbook = custom_parser.custom_autofill(workbook)
return workbook
class PydContact(BaseModel):

42
src/submissions/frontend/__init__.py
View File

@@ -13,7 +13,7 @@ from PyQt6.QtCore import pyqtSignal
from PyQt6.QtGui import QAction
from PyQt6.QtWebEngineWidgets import QWebEngineView
from pathlib import Path
from backend.db import (
from backend.db.functions import (
lookup_control_types, lookup_modes
)
from backend.validators import PydSubmission, PydReagent
@@ -22,6 +22,7 @@ from frontend.custom_widgets import SubmissionsSheet, AlertPop, AddReagentForm,
import logging
from datetime import date
import webbrowser
from pathlib import Path
logger = logging.getLogger(f'submissions.{__name__}')
logger.info("Hello, I am a logger")
@@ -32,6 +33,7 @@ class App(QMainWindow):
logger.debug(f"Initializing main window...")
super().__init__()
self.ctx = ctx
self.last_dir = ctx.directory_path
# indicate version and connected database in title bar
try:
self.title = f"Submissions App (v{ctx.package.__version__}) - {ctx.database_path}"
@@ -156,6 +158,7 @@ class App(QMainWindow):
Args:
result (dict | None, optional): The result from a function. Defaults to None.
"""
logger.info(f"We got the result: {result}")
if result != None:
msg = AlertPop(message=result['message'], status=result['status'])
msg.exec()
@@ -399,7 +402,7 @@ class SubmissionFormContainer(QWidget):
def __init__(self, parent: QWidget) -> None:
logger.debug(f"Setting form widget...")
super().__init__(parent)
self.parent = parent
# self.parent = parent
self.setAcceptDrops(True)
@@ -411,37 +414,8 @@ class SubmissionFormContainer(QWidget):
def dropEvent(self, event):
fname = Path([u.toLocalFile() for u in event.mimeData().urls()][0])
app = self.parent().parent().parent().parent().parent().parent().parent
logger.debug(f"App: {app}")
app.last_dir = fname.parent
self.import_drag.emit(fname)
def clear_form(self):
for item in self.findChildren(QWidget):
item.setParent(None)
def parse_form(self) -> PydSubmission:
logger.debug(f"Hello from form parser!")
info = {}
reagents = []
samples = self.parent.parent.samples
logger.debug(f"Using samples: {pformat(samples)}")
widgets = [widget for widget in self.findChildren(QWidget) if widget.objectName() not in self.ignore]
# widgets = [widget for widget in self.findChildren(QWidget)]
for widget in widgets:
logger.debug(f"Parsed widget: {widget.objectName()} of type {type(widget)}")
match widget:
case ReagentFormWidget():
reagent, _ = widget.parse_form()
reagents.append(reagent)
case ImportReagent():
reagent = dict(name=widget.objectName().replace("lot_", ""), lot=widget.currentText(), type=None, expiry=None)
reagents.append(PydReagent(ctx=self.parent.parent.ctx, **reagent))
case QLineEdit():
info[widget.objectName()] = dict(value=widget.text())
case QComboBox():
info[widget.objectName()] = dict(value=widget.currentText())
case QDateEdit():
info[widget.objectName()] = dict(value=widget.date().toPyDate())
logger.debug(f"Info: {pformat(info)}")
logger.debug(f"Reagents: {pformat(reagents)}")
# sys.exit("Hi Landon. Check the reagents! frontend.__init__ line 442")
submission = PydSubmission(ctx=self.parent.parent.ctx, filepath=self.parent.parent.current_file, reagents=reagents, samples=samples, **info)
return submission

9
src/submissions/frontend/all_window_functions.py
View File

@@ -23,11 +23,13 @@ def select_open_file(obj:QMainWindow, file_extension:str) -> Path:
Path: Path of file to be opened
"""
try:
home_dir = Path(obj.ctx.directory_path).resolve().__str__()
# home_dir = Path(obj.ctx.directory_path).resolve().__str__()
home_dir = obj.last_dir.resolve().__str__()
except FileNotFoundError:
home_dir = Path.home().resolve().__str__()
fname = Path(QFileDialog.getOpenFileName(obj, 'Open file', home_dir, filter = f"{file_extension}(*.{file_extension})")[0])
# fname = Path(QFileDialog.getOpenFileName(obj, 'Open file', filter = f"{file_extension}(*.{file_extension})")[0])
obj.last_file = fname
return fname
def select_save_file(obj:QMainWindow, default_name:str, extension:str) -> Path:
@@ -43,12 +45,13 @@ def select_save_file(obj:QMainWindow, default_name:str, extension:str) -> Path:
Path: Path of file to be opened
"""
try:
home_dir = Path(obj.ctx.directory_path).joinpath(default_name).resolve().__str__()
# home_dir = Path(obj.ctx.directory_path).joinpath(default_name).resolve().__str__()
home_dir = obj.last_dir.joinpath(default_name).resolve().__str__()
except FileNotFoundError:
home_dir = Path.home().joinpath(default_name).resolve().__str__()
fname = Path(QFileDialog.getSaveFileName(obj, "Save File", home_dir, filter = f"{extension}(*.{extension})")[0])
# fname = Path(QFileDialog.getSaveFileName(obj, "Save File", filter = f"{extension}(*.{extension})")[0])
obj.last_dir = fname.parent
return fname
def extract_form_info(object) -> dict:

294
src/submissions/frontend/custom_widgets/misc.py
View File

@@ -11,19 +11,20 @@ from PyQt6.QtWidgets import (
QGridLayout, QPushButton, QSpinBox, QDoubleSpinBox,
QHBoxLayout, QScrollArea, QFormLayout
)
from PyQt6.QtCore import Qt, QDate, QSize
from PyQt6.QtCore import Qt, QDate, QSize, pyqtSignal
from tools import check_not_nan, jinja_template_loading, Settings
from backend.db.functions import \
lookup_reagent_types, lookup_reagents, lookup_submission_type, lookup_reagenttype_kittype_association, \
lookup_submissions#, construct_kit_from_yaml
lookup_submissions, lookup_organizations, lookup_kit_types
from backend.db.models import SubmissionTypeKitTypeAssociation
from sqlalchemy import FLOAT, INTEGER
import logging
import numpy as np
from .pop_ups import AlertPop, QuestionAsker
from backend.validators import PydReagent, PydKit, PydReagentType, PydSubmission
from typing import Tuple
from typing import Tuple, List
from pprint import pformat
import difflib
logger = logging.getLogger(f"submissions.{__name__}")
@@ -388,6 +389,10 @@ class ControlsDatePicker(QWidget):
class ImportReagent(QComboBox):
"""
NOTE: Depreciated in favour of ReagentFormWidget
"""
def __init__(self, ctx:Settings, reagent:dict|PydReagent, extraction_kit:str):
super().__init__()
self.setEditable(True)
@@ -442,25 +447,6 @@ class ImportReagent(QComboBox):
self.setObjectName(f"lot_{reagent.type}")
self.addItems(relevant_reagents)
class ParsedQLabel(QLabel):
def __init__(self, input_object, field_name, title:bool=True, label_name:str|None=None):
super().__init__()
try:
check = input_object['parsed']
except:
return
if label_name != None:
self.setObjectName(label_name)
if title:
output = field_name.replace('_', ' ').title()
else:
output = field_name.replace('_', ' ')
if check:
self.setText(f"Parsed {output}")
else:
self.setText(f"MISSING {output}")
class FirstStrandSalvage(QDialog):
def __init__(self, ctx:Settings, submitter_id:str, rsl_plate_num:str|None=None) -> None:
@@ -526,10 +512,8 @@ class FirstStrandPlateList(QDialog):
class ReagentFormWidget(QWidget):
def __init__(self, parent:QWidget, reagent:PydReagent, extraction_kit:str):
super().__init__()
self.setParent(parent)
logger.debug(f"Reagent form widget parent is: {self.parent()}")
logger.debug(f"It's great grandparent is {self.parent().parent.parent} which has a method [add_reagent]: {hasattr(self.parent().parent.parent, 'add_reagent')}")
super().__init__(parent)
# self.setParent(parent)
self.reagent = reagent
self.extraction_kit = extraction_kit
self.ctx = reagent.ctx
@@ -538,58 +522,66 @@ class ReagentFormWidget(QWidget):
layout.addWidget(self.label)
self.lot = self.ReagentLot(reagent=reagent, extraction_kit=extraction_kit)
layout.addWidget(self.lot)
# Remove spacing between reagents
layout.setContentsMargins(0,0,0,0)
self.setLayout(layout)
self.setObjectName(reagent.name)
self.missing = not reagent.parsed
self.missing = reagent.missing
# If changed set self.missing to True and update self.label
self.lot.currentTextChanged.connect(self.updated)
def parse_form(self) -> Tuple[PydReagent, dict]:
lot = self.lot.currentText()
# type = self.label.text().replace("_label")
wanted_reagent = lookup_reagents(ctx=self.ctx, lot_number=lot, reagent_type=self.reagent.type)
# if reagent doesn't exist in database, off to add it (uses App.add_reagent)
if wanted_reagent == None:
dlg = QuestionAsker(title=f"Add {lot}?", message=f"Couldn't find reagent type {self.reagent.type}: {lot} in the database.\n\nWould you like to add it?")
if dlg.exec():
# logger.debug(f"Looking through {pformat(self.parent.reagents)} for reagent {reagent.name}")
# try:
# picked_reagent = [item for item in obj.reagents if item.type == reagent.name][0]
# except IndexError:
# logger.error(f"Couldn't find {reagent.name} in obj.reagents. Checking missing reagents {pprint.pformat(obj.missing_reagents)}")
# picked_reagent = [item for item in obj.missing_reagents if item.type == reagent.name][0]
# logger.debug(f"checking reagent: {reagent.name} in obj.reagents. Result: {picked_reagent}")
# expiry_date = picked_reagent.expiry
wanted_reagent = self.parent().parent.parent.add_reagent(reagent_lot=lot, reagent_type=self.reagent.type, expiry=self.reagent.expiry, name=self.reagent.name)
wanted_reagent = self.parent().parent().parent().parent().parent().parent().parent().parent().parent.add_reagent(reagent_lot=lot, reagent_type=self.reagent.type, expiry=self.reagent.expiry, name=self.reagent.name)
return wanted_reagent, None
else:
# In this case we will have an empty reagent and the submission will fail kit integrity check
logger.debug("Will not add reagent.")
return None, dict(message="Failed integrity check", status="critical")
else:
rt = lookup_reagent_types(ctx=self.ctx, kit_type=self.extraction_kit, reagent=wanted_reagent)
# Since this now gets passed in directly from the parser -> pyd -> form and the parser gets the name
# from the db, it should no longer be necessary to query the db with reagent/kit, but with rt name directly.
rt = lookup_reagent_types(ctx=self.ctx, name=self.reagent.type)
# rt = lookup_reagent_types(ctx=self.ctx, kit_type=self.extraction_kit, reagent=wanted_reagent)
if rt == None:
rt = lookup_reagent_types(ctx=self.ctx, kit_type=self.extraction_kit, reagent=wanted_reagent)
return PydReagent(ctx=self.ctx, name=wanted_reagent.name, lot=wanted_reagent.lot, type=rt.name, expiry=wanted_reagent.expiry, parsed=not self.missing), None
def updated(self):
self.missing = True
self.label.updated(self.reagent.type)
class ReagentParsedLabel(QLabel):
def __init__(self, reagent:PydReagent):
super().__init__()
try:
check = reagent.parsed
check = not reagent.missing
except:
return
check = False
self.setObjectName(f"{reagent.type}_label")
if check:
self.setText(f"Parsed {reagent.type}")
else:
self.setText(f"MISSING {reagent.type}")
def updated(self, reagent_type:str):
self.setText(f"UPDATED {reagent_type}")
class ReagentLot(QComboBox):
def __init__(self, reagent, extraction_kit:str) -> None:
super().__init__()
self.ctx = reagent.ctx
self.setEditable(True)
if reagent.parsed:
pass
# if reagent.parsed:
# pass
logger.debug(f"Attempting lookup of reagents by type: {reagent.type}")
# below was lookup_reagent_by_type_name_and_kit_name, but I couldn't get it to work.
lookup = lookup_reagents(ctx=self.ctx, reagent_type=reagent.type)
@@ -611,8 +603,11 @@ class ReagentFormWidget(QWidget):
else:
# TODO: look up the last used reagent of this type in the database
looked_up_rt = lookup_reagenttype_kittype_association(ctx=self.ctx, reagent_type=reagent.type, kit_type=extraction_kit)
looked_up_reg = lookup_reagents(ctx=self.ctx, lot_number=looked_up_rt.last_used)
logger.debug(f"Because there was no reagent listed for {reagent}, we will insert the last lot used: {looked_up_reg}")
try:
looked_up_reg = lookup_reagents(ctx=self.ctx, lot_number=looked_up_rt.last_used)
except AttributeError:
looked_up_reg = None
logger.debug(f"Because there was no reagent listed for {reagent.lot}, we will insert the last lot used: {looked_up_reg}")
if looked_up_reg != None:
relevant_reagents.remove(str(looked_up_reg.lot))
relevant_reagents.insert(0, str(looked_up_reg.lot))
@@ -631,41 +626,212 @@ class ReagentFormWidget(QWidget):
class SubmissionFormWidget(QWidget):
def __init__(self, parent: QWidget) -> None:
def __init__(self, parent: QWidget, **kwargs) -> None:
super().__init__(parent)
self.ignore = [None, "", "qt_spinbox_lineedit", "qt_scrollarea_viewport", "qt_scrollarea_hcontainer",
"qt_scrollarea_vcontainer", "submit_btn"
]
# self.ignore = [None, "", "qt_spinbox_lineedit", "qt_scrollarea_viewport", "qt_scrollarea_hcontainer",
# "qt_scrollarea_vcontainer", "submit_btn"
# ]
self.ignore = ['filepath', 'samples', 'reagents', 'csv', 'ctx']
layout = QVBoxLayout()
for k, v in kwargs.items():
if k not in self.ignore:
add_widget = self.create_widget(key=k, value=v, submission_type=kwargs['submission_type'])
if add_widget != None:
layout.addWidget(add_widget)
else:
setattr(self, k, v)
self.setLayout(layout)
def create_widget(self, key:str, value:dict, submission_type:str|None=None):
if key not in self.ignore:
return self.InfoItem(self, key=key, value=value, submission_type=submission_type)
return None
def clear_form(self):
for item in self.findChildren(QWidget):
item.setParent(None)
def find_widgets(self, object_name:str|None=None) -> List[QWidget]:
query = self.findChildren(QWidget)
if object_name != None:
query = [widget for widget in query if widget.objectName()==object_name]
return query
def parse_form(self) -> PydSubmission:
logger.debug(f"Hello from form parser!")
info = {}
reagents = []
samples = self.parent.parent.samples
logger.debug(f"Using samples: {pformat(samples)}")
widgets = [widget for widget in self.findChildren(QWidget) if widget.objectName() not in self.ignore]
if hasattr(self, 'csv'):
info['csv'] = self.csv
# samples = self.parent().parent.parent.samples
# filepath = self.parent().parent.parent.pyd.filepath
# logger.debug(f"Using samples: {pformat(samples)}")
# widgets = [widget for widget in self.findChildren(QWidget) if widget.objectName() not in self.ignore]
# widgets = [widget for widget in self.findChildren(QWidget)]
for widget in widgets:
logger.debug(f"Parsed widget: {widget.objectName()} of type {type(widget)}")
# for widget in widgets:
for widget in self.findChildren(QWidget):
logger.debug(f"Parsed widget of type {type(widget)}")
match widget:
case ReagentFormWidget():
reagent, _ = widget.parse_form()
reagents.append(reagent)
case ImportReagent():
reagent = dict(name=widget.objectName().replace("lot_", ""), lot=widget.currentText(), type=None, expiry=None)
reagents.append(PydReagent(ctx=self.parent.parent.ctx, **reagent))
case QLineEdit():
info[widget.objectName()] = dict(value=widget.text())
case QComboBox():
info[widget.objectName()] = dict(value=widget.currentText())
case QDateEdit():
info[widget.objectName()] = dict(value=widget.date().toPyDate())
case self.InfoItem():
field, value = widget.parse_form()
if field != None:
info[field] = value
# case ImportReagent():
# reagent = dict(name=widget.objectName().replace("lot_", ""), lot=widget.currentText(), type=None, expiry=None)
# # ctx: self.SubmissionContinerWidget.AddSubForm
# reagents.append(PydReagent(ctx=self.parent.parent.ctx, **reagent))
# case QLineEdit():
# info[widget.objectName()] = dict(value=widget.text())
# case QComboBox():
# info[widget.objectName()] = dict(value=widget.currentText())
# case QDateEdit():
# info[widget.objectName()] = dict(value=widget.date().toPyDate())
logger.debug(f"Info: {pformat(info)}")
logger.debug(f"Reagents: {pformat(reagents)}")
# sys.exit("Hi Landon. Check the reagents! frontend.__init__ line 442")
submission = PydSubmission(ctx=self.parent.parent.ctx, filepath=self.parent.parent.current_file, reagents=reagents, samples=samples, **info)
app = self.parent().parent().parent().parent().parent().parent().parent().parent
submission = PydSubmission(ctx=app.ctx, filepath=self.filepath, reagents=reagents, samples=self.samples, **info)
return submission
class InfoItem(QWidget):
def __init__(self, parent: QWidget, key:str, value:dict, submission_type:str|None=None) -> None:
super().__init__(parent)
layout = QVBoxLayout()
self.label = self.ParsedQLabel(key=key, value=value)
self.input: QWidget = self.set_widget(parent=self, key=key, value=value, submission_type=submission_type['value'])
self.setObjectName(key)
try:
self.missing:bool = value['missing']
except (TypeError, KeyError):
self.missing:bool = False
if self.input != None:
layout.addWidget(self.label)
layout.addWidget(self.input)
layout.setContentsMargins(0,0,0,0)
self.setLayout(layout)
match self.input:
case QComboBox():
self.input.currentTextChanged.connect(self.update_missing)
case QDateEdit():
self.input.dateChanged.connect(self.update_missing)
case QLineEdit():
self.input.textChanged.connect(self.update_missing)
def parse_form(self):
match self.input:
case QLineEdit():
value = self.input.text()
case QComboBox():
value = self.input.currentText()
case QDateEdit():
value = self.input.date().toPyDate()
case _:
return None, None
return self.input.objectName(), dict(value=value, missing=self.missing)
def set_widget(self, parent: QWidget, key:str, value:dict, submission_type:str|None=None) -> QWidget:
try:
value = value['value']
except (TypeError, KeyError):
pass
obj = parent.parent().parent()
logger.debug(f"Creating widget for: {key}")
match key:
case 'submitting_lab':
add_widget = QComboBox()
# lookup organizations suitable for submitting_lab (ctx: self.InfoItem.SubmissionFormWidget.SubmissionFormContainer.AddSubForm )
labs = [item.__str__() for item in lookup_organizations(ctx=obj.ctx)]
# try to set closest match to top of list
try:
labs = difflib.get_close_matches(value, labs, len(labs), 0)
except (TypeError, ValueError):
pass
# set combobox values to lookedup values
add_widget.addItems(labs)
case 'extraction_kit':
# if extraction kit not available, all other values fail
if not check_not_nan(value):
msg = AlertPop(message="Make sure to check your extraction kit in the excel sheet!", status="warning")
msg.exec()
# create combobox to hold looked up kits
add_widget = QComboBox()
# lookup existing kits by 'submission_type' decided on by sheetparser
logger.debug(f"Looking up kits used for {submission_type}")
uses = [item.__str__() for item in lookup_kit_types(ctx=obj.ctx, used_for=submission_type)]
obj.uses = uses
logger.debug(f"Kits received for {submission_type}: {uses}")
if check_not_nan(value):
logger.debug(f"The extraction kit in parser was: {value}")
uses.insert(0, uses.pop(uses.index(value)))
obj.ext_kit = value
else:
logger.error(f"Couldn't find {obj.prsr.sub['extraction_kit']}")
obj.ext_kit = uses[0]
add_widget.addItems(uses)
# Run reagent scraper whenever extraction kit is changed.
# add_widget.currentTextChanged.connect(obj.scrape_reagents)
case 'submitted_date':
# uses base calendar
add_widget = QDateEdit(calendarPopup=True)
# sets submitted date based on date found in excel sheet
try:
add_widget.setDate(value)
# if not found, use today
except:
add_widget.setDate(date.today())
case 'submission_category':
add_widget = QComboBox()
cats = ['Diagnostic', "Surveillance", "Research"]
cats += [item.name for item in lookup_submission_type(ctx=obj.ctx)]
try:
cats.insert(0, cats.pop(cats.index(value)))
except ValueError:
cats.insert(0, cats.pop(cats.index(submission_type)))
add_widget.addItems(cats)
case _:
# anything else gets added in as a line edit
add_widget = QLineEdit()
logger.debug(f"Setting widget text to {str(value).replace('_', ' ')}")
add_widget.setText(str(value).replace("_", " "))
if add_widget != None:
add_widget.setObjectName(key)
add_widget.setParent(parent)
return add_widget
def update_missing(self):
self.missing = True
self.label.updated(self.objectName())
class ParsedQLabel(QLabel):
def __init__(self, key:str, value:dict, title:bool=True, label_name:str|None=None):
super().__init__()
try:
check = not value['missing']
except:
check = True
if label_name != None:
self.setObjectName(label_name)
else:
self.setObjectName(f"{key}_label")
if title:
output = key.replace('_', ' ').title()
else:
output = key.replace('_', ' ')
if check:
self.setText(f"Parsed {output}")
else:
self.setText(f"MISSING {output}")
def updated(self, key:str, title:bool=True):
if title:
output = key.replace('_', ' ').title()
else:
output = key.replace('_', ' ')
self.setText(f"UPDATED {output}")

356
src/submissions/frontend/main_window_functions.py
View File

@@ -7,6 +7,7 @@ from getpass import getuser
import inspect
import pprint
import re
import sys
import yaml
import json
from typing import Tuple, List
@@ -35,7 +36,6 @@ from backend.validators import PydSubmission, PydSample, PydReagent
from tools import check_not_nan, convert_well_to_row_column
from .custom_widgets.pop_ups import AlertPop, QuestionAsker
from .custom_widgets import ReportDatePicker
from .custom_widgets.misc import ImportReagent, ParsedQLabel
from .visualizations.control_charts import create_charts, construct_html
from pathlib import Path
from frontend.custom_widgets.misc import FirstStrandSalvage, FirstStrandPlateList, ReagentFormWidget
@@ -54,8 +54,12 @@ def import_submission_function(obj:QMainWindow, fname:Path|None=None) -> Tuple[Q
"""
logger.debug(f"\n\nStarting Import...\n\n")
result = None
logger.debug(obj.ctx)
# logger.debug(obj.ctx)
# initialize samples
try:
obj.form.setParent(None)
except AttributeError:
pass
obj.samples = []
obj.missing_info = []
# set file dialog
@@ -73,106 +77,114 @@ def import_submission_function(obj:QMainWindow, fname:Path|None=None) -> Tuple[Q
return obj, result
try:
logger.debug(f"Submission dictionary:\n{pprint.pformat(obj.prsr.sub)}")
pyd = obj.prsr.to_pydantic()
logger.debug(f"Pydantic result: \n\n{pprint.pformat(pyd)}\n\n")
obj.pyd = obj.prsr.to_pydantic()
logger.debug(f"Pydantic result: \n\n{pprint.pformat(obj.pyd)}\n\n")
except Exception as e:
return obj, dict(message= f"Problem creating pydantic model:\n\n{e}", status="critical")
# destroy any widgets from previous imports
obj.table_widget.formwidget.clear_form()
obj.current_submission_type = pyd.submission_type['value']
obj.current_file = pyd.filepath
# obj.table_widget.formwidget.set_parent(None)
# obj.current_submission_type = pyd.submission_type['value']
# obj.current_file = pyd.filepath
# Get list of fields from pydantic model.
fields = list(pyd.model_fields.keys()) + list(pyd.model_extra.keys())
fields.remove('filepath')
logger.debug(f"pydantic fields: {fields}")
for field in fields:
value = getattr(pyd, field)
logger.debug(f"Checking: {field}: {value}")
# Get from pydantic model whether field was completed in the form
if isinstance(value, dict) and field != 'ctx':
logger.debug(f"The field {field} is a dictionary: {value}")
if not value['parsed']:
obj.missing_info.append(field)
label = ParsedQLabel(value, field)
match field:
case 'submitting_lab':
logger.debug(f"{field}: {value['value']}")
# create combobox to hold looked up submitting labs
add_widget = QComboBox()
labs = [item.__str__() for item in lookup_organizations(ctx=obj.ctx)]
# try to set closest match to top of list
try:
labs = difflib.get_close_matches(value['value'], labs, len(labs), 0)
except (TypeError, ValueError):
pass
# set combobox values to lookedup values
add_widget.addItems(labs)
case 'extraction_kit':
# if extraction kit not available, all other values fail
if not check_not_nan(value['value']):
msg = AlertPop(message="Make sure to check your extraction kit in the excel sheet!", status="warning")
msg.exec()
# create combobox to hold looked up kits
add_widget = QComboBox()
# lookup existing kits by 'submission_type' decided on by sheetparser
logger.debug(f"Looking up kits used for {pyd.submission_type['value']}")
uses = [item.__str__() for item in lookup_kit_types(ctx=obj.ctx, used_for=pyd.submission_type['value'])]
logger.debug(f"Kits received for {pyd.submission_type['value']}: {uses}")
if check_not_nan(value['value']):
logger.debug(f"The extraction kit in parser was: {value['value']}")
uses.insert(0, uses.pop(uses.index(value['value'])))
obj.ext_kit = value['value']
else:
logger.error(f"Couldn't find {obj.prsr.sub['extraction_kit']}")
obj.ext_kit = uses[0]
# Run reagent scraper whenever extraction kit is changed.
add_widget.currentTextChanged.connect(obj.scrape_reagents)
case 'submitted_date':
# uses base calendar
add_widget = QDateEdit(calendarPopup=True)
# sets submitted date based on date found in excel sheet
try:
add_widget.setDate(value['value'])
# if not found, use today
except:
add_widget.setDate(date.today())
case 'samples':
# hold samples in 'obj' until form submitted
logger.debug(f"{field}:\n\t{value}")
obj.samples = value
continue
case 'submission_category':
add_widget = QComboBox()
cats = ['Diagnostic', "Surveillance", "Research"]
cats += [item.name for item in lookup_submission_type(ctx=obj.ctx)]
try:
cats.insert(0, cats.pop(cats.index(value['value'])))
except ValueError:
cats.insert(0, cats.pop(cats.index(pyd.submission_type['value'])))
add_widget.addItems(cats)
case "ctx" | 'reagents' | 'csv' | 'filepath':
continue
case _:
# anything else gets added in as a line edit
add_widget = QLineEdit()
logger.debug(f"Setting widget text to {str(value['value']).replace('_', ' ')}")
add_widget.setText(str(value['value']).replace("_", " "))
try:
add_widget.setObjectName(field)
logger.debug(f"Widget name set to: {add_widget.objectName()}")
obj.table_widget.formlayout.addWidget(label)
obj.table_widget.formlayout.addWidget(add_widget)
except AttributeError as e:
logger.error(e)
kit_widget = obj.table_widget.formlayout.parentWidget().findChild(QComboBox, 'extraction_kit')
kit_widget.addItems(uses)
# fields = list(pyd.model_fields.keys()) + list(pyd.model_extra.keys())
# fields.remove('filepath')
# logger.debug(f"pydantic fields: {fields}")
# for field in fields:
# value = getattr(pyd, field)
# logger.debug(f"Checking: {field}: {value}")
# # Get from pydantic model whether field was completed in the form
# if isinstance(value, dict) and field != 'ctx':
# logger.debug(f"The field {field} is a dictionary: {value}")
# if not value['parsed']:
# obj.missing_info.append(field)
# label = ParsedQLabel(value, field)
# match field:
# case 'submitting_lab':
# logger.debug(f"{field}: {value['value']}")
# # create combobox to hold looked up submitting labs
# add_widget = QComboBox()
# labs = [item.__str__() for item in lookup_organizations(ctx=obj.ctx)]
# # try to set closest match to top of list
# try:
# labs = difflib.get_close_matches(value['value'], labs, len(labs), 0)
# except (TypeError, ValueError):
# pass
# # set combobox values to lookedup values
# add_widget.addItems(labs)
# case 'extraction_kit':
# # if extraction kit not available, all other values fail
# if not check_not_nan(value['value']):
# msg = AlertPop(message="Make sure to check your extraction kit in the excel sheet!", status="warning")
# msg.exec()
# # create combobox to hold looked up kits
# add_widget = QComboBox()
# # lookup existing kits by 'submission_type' decided on by sheetparser
# logger.debug(f"Looking up kits used for {pyd.submission_type['value']}")
# uses = [item.__str__() for item in lookup_kit_types(ctx=obj.ctx, used_for=pyd.submission_type['value'])]
# logger.debug(f"Kits received for {pyd.submission_type['value']}: {uses}")
# if check_not_nan(value['value']):
# logger.debug(f"The extraction kit in parser was: {value['value']}")
# uses.insert(0, uses.pop(uses.index(value['value'])))
# obj.ext_kit = value['value']
# else:
# logger.error(f"Couldn't find {obj.prsr.sub['extraction_kit']}")
# obj.ext_kit = uses[0]
# # Run reagent scraper whenever extraction kit is changed.
# add_widget.currentTextChanged.connect(obj.scrape_reagents)
# case 'submitted_date':
# # uses base calendar
# add_widget = QDateEdit(calendarPopup=True)
# # sets submitted date based on date found in excel sheet
# try:
# add_widget.setDate(value['value'])
# # if not found, use today
# except:
# add_widget.setDate(date.today())
# case 'samples':
# # hold samples in 'obj' until form submitted
# logger.debug(f"{field}:\n\t{value}")
# obj.samples = value
# continue
# case 'submission_category':
# add_widget = QComboBox()
# cats = ['Diagnostic', "Surveillance", "Research"]
# cats += [item.name for item in lookup_submission_type(ctx=obj.ctx)]
# try:
# cats.insert(0, cats.pop(cats.index(value['value'])))
# except ValueError:
# cats.insert(0, cats.pop(cats.index(pyd.submission_type['value'])))
# add_widget.addItems(cats)
# case "ctx" | 'reagents' | 'csv' | 'filepath':
# continue
# case _:
# # anything else gets added in as a line edit
# add_widget = QLineEdit()
# logger.debug(f"Setting widget text to {str(value['value']).replace('_', ' ')}")
# add_widget.setText(str(value['value']).replace("_", " "))
# try:
# add_widget.setObjectName(field)
# logger.debug(f"Widget name set to: {add_widget.objectName()}")
# obj.table_widget.formlayout.addWidget(label)
# obj.table_widget.formlayout.addWidget(add_widget)
# except AttributeError as e:
# logger.error(e)
obj.form = obj.pyd.toForm(parent=obj)
obj.table_widget.formlayout.addWidget(obj.form)
# kit_widget = obj.table_widget.formlayout.parentWidget().findChild(QComboBox, 'extraction_kit')
kit_widget = obj.form.find_widgets(object_name="extraction_kit")[0].input
logger.debug(f"Kitwidget {kit_widget}")
# block
# with QSignalBlocker(kit_widget) as blocker:
# kit_widget.addItems(obj.uses)
obj.scrape_reagents(kit_widget.currentText())
kit_widget.currentTextChanged.connect(obj.scrape_reagents)
# compare obj.reagents with expected reagents in kit
if obj.prsr.sample_result != None:
msg = AlertPop(message=obj.prsr.sample_result, status="WARNING")
msg.exec()
logger.debug(f"Pydantic extra fields: {pyd.model_extra}")
if "csv" in pyd.model_extra:
obj.csv = pyd.model_extra['csv']
# logger.debug(f"Pydantic extra fields: {obj.pyd.model_extra}")
# if "csv" in pyd.model_extra:
# obj.csv = pyd.model_extra['csv']
logger.debug(f"All attributes of obj:\n{pprint.pformat(obj.__dict__)}")
return obj, result
@@ -187,15 +199,19 @@ def kit_reload_function(obj:QMainWindow) -> Tuple[QMainWindow, dict]:
Tuple[QMainWindow, dict]: Collection of new main app window and result dict
"""
result = None
for item in obj.table_widget.formlayout.parentWidget().findChildren(QWidget):
if isinstance(item, QLabel):
if item.text().startswith("Lot"):
item.setParent(None)
else:
logger.debug(f"Type of {item.objectName()} is {type(item)}")
if item.objectName().startswith("lot_"):
item.setParent(None)
obj.kit_integrity_completion_function()
# for item in obj.table_widget.formlayout.parentWidget().findChildren(QWidget):
logger.debug(f"Attempting to clear {obj.form.find_widgets()}")
for item in obj.form.find_widgets():
if isinstance(item, ReagentFormWidget):
item.setParent(None)
# if item.text().startswith("Lot"):
# item.setParent(None)
# else:
# logger.debug(f"Type of {item.objectName()} is {type(item)}")
# if item.objectName().startswith("lot_"):
# item.setParent(None)
kit_integrity_completion_function(obj)
return obj, result
def kit_integrity_completion_function(obj:QMainWindow) -> Tuple[QMainWindow, dict]:
@@ -209,23 +225,31 @@ def kit_integrity_completion_function(obj:QMainWindow) -> Tuple[QMainWindow, dic
Tuple[QMainWindow, dict]: Collection of new main app window and result dict
"""
result = None
missing_reagents = []
# kit_reload_function(obj=obj)
logger.debug(inspect.currentframe().f_back.f_code.co_name)
# find the widget that contains kit info
kit_widget = obj.table_widget.formlayout.parentWidget().findChild(QComboBox, 'extraction_kit')
kit_widget = obj.form.find_widgets(object_name="extraction_kit")[0].input
logger.debug(f"Kit selector: {kit_widget}")
# get current kit being used
obj.ext_kit = kit_widget.currentText()
for reagent in obj.reagents:
# for reagent in obj.pyd.reagents:
for reagent in obj.form.reagents:
# obj.table_widget.formlayout.addWidget(ParsedQLabel({'parsed':True}, item.type, title=False, label_name=f"lot_{item.type}"))
# reagent = dict(type=item.type, lot=item.lot, expiry=item.expiry, name=item.name)
# add_widget = ImportReagent(ctx=obj.ctx, reagent=reagent, extraction_kit=obj.ext_kit)
# obj.table_widget.formlayout.addWidget(add_widget)
add_widget = ReagentFormWidget(parent=obj.table_widget.formwidget, reagent=reagent, extraction_kit=obj.ext_kit)
obj.table_widget.formlayout.addWidget(add_widget)
add_widget.setParent(obj.form)
# obj.table_widget.formlayout.addWidget(add_widget)
obj.form.layout().addWidget(add_widget)
if reagent.missing:
missing_reagents.append(reagent)
logger.debug(f"Checking integrity of {obj.ext_kit}")
# TODO: put check_kit_integrity here instead of what's here?
# see if there are any missing reagents
if len(obj.missing_reagents) > 0:
result = dict(message=f"The submission you are importing is missing some reagents expected by the kit.\n\nIt looks like you are missing: {[item.type.upper() for item in obj.missing_reagents]}\n\nAlternatively, you may have set the wrong extraction kit.\n\nThe program will populate lists using existing reagents.\n\nPlease make sure you check the lots carefully!", status="Warning")
if len(missing_reagents) > 0:
result = dict(message=f"The submission you are importing is missing some reagents expected by the kit.\n\nIt looks like you are missing: {[item.type.upper() for item in missing_reagents]}\n\nAlternatively, you may have set the wrong extraction kit.\n\nThe program will populate lists using existing reagents.\n\nPlease make sure you check the lots carefully!", status="Warning")
# for item in obj.missing_reagents:
# # Add label that has parsed as False to show "MISSING" label.
# obj.table_widget.formlayout.addWidget(ParsedQLabel({'parsed':False}, item.type, title=False, label_name=f"missing_{item.type}"))
@@ -238,7 +262,7 @@ def kit_integrity_completion_function(obj:QMainWindow) -> Tuple[QMainWindow, dic
# Add submit button to the form.
submit_btn = QPushButton("Submit")
submit_btn.setObjectName("submit_btn")
obj.table_widget.formlayout.addWidget(submit_btn)
obj.form.layout().addWidget(submit_btn)
submit_btn.clicked.connect(obj.submit_new_sample)
return obj, result
@@ -254,61 +278,25 @@ def submit_new_sample_function(obj:QMainWindow) -> Tuple[QMainWindow, dict]:
"""
logger.debug(f"\n\nBeginning Submission\n\n")
result = None
# extract info from the form widgets
# info = extract_form_info(obj.table_widget.tab1)
# if isinstance(info, tuple):
# logger.warning(f"Got tuple for info for some reason.")
# info = info[0]
# # seperate out reagents
# reagents = {k.replace("lot_", ""):v for k,v in info.items() if k.startswith("lot_")}
# info = {k:v for k,v in info.items() if not k.startswith("lot_")}
# info, reagents = obj.table_widget.formwidget.parse_form()
submission: PydSubmission = obj.table_widget.formwidget.parse_form()
logger.debug(f"Submission: {pprint.pformat(submission)}")
# parsed_reagents = []
# compare reagents in form to reagent database
# for reagent in submission.reagents:
# # Lookup any existing reagent of this type with this lot number
# wanted_reagent = lookup_reagents(ctx=obj.ctx, lot_number=reagent.lot, reagent_type=reagent.name)
# logger.debug(f"Looked up reagent: {wanted_reagent}")
# # if reagent not found offer to add to database
# if wanted_reagent == None:
# # r_lot = reagent[reagent]
# dlg = QuestionAsker(title=f"Add {reagent.lot}?", message=f"Couldn't find reagent type {reagent.name.strip('Lot')}: {reagent.lot} in the database.\n\nWould you like to add it?")
# if dlg.exec():
# logger.debug(f"Looking through {pprint.pformat(obj.reagents)} for reagent {reagent.name}")
# try:
# picked_reagent = [item for item in obj.reagents if item.type == reagent.name][0]
# except IndexError:
# logger.error(f"Couldn't find {reagent.name} in obj.reagents. Checking missing reagents {pprint.pformat(obj.missing_reagents)}")
# picked_reagent = [item for item in obj.missing_reagents if item.type == reagent.name][0]
# logger.debug(f"checking reagent: {reagent.name} in obj.reagents. Result: {picked_reagent}")
# expiry_date = picked_reagent.expiry
# wanted_reagent = obj.add_reagent(reagent_lot=reagent.lot, reagent_type=reagent.name.replace("lot_", ""), expiry=expiry_date, name=picked_reagent.name)
# else:
# # In this case we will have an empty reagent and the submission will fail kit integrity check
# logger.debug("Will not add reagent.")
# return obj, dict(message="Failed integrity check", status="critical")
# # Append the PydReagent object o be added to the submission
# parsed_reagents.append(reagent)
# # move samples into preliminary submission dict
# submission.reagents = parsed_reagents
# submission.uploaded_by = getuser()
# construct submission object
# logger.debug(f"Here is the info_dict: {pprint.pformat(info)}")
# base_submission, result = construct_submission_info(ctx=obj.ctx, info_dict=info)
base_submission, result = submission.toSQL()
# delattr(base_submission, "ctx")
# raise ValueError(base_submission.__dict__)
obj.pyd: PydSubmission = obj.form.parse_form()
logger.debug(f"Submission: {pprint.pformat(obj.pyd)}")
logger.debug("Checking kit integrity...")
kit_integrity = check_kit_integrity(ctx=obj.ctx, sub=obj.pyd)
if kit_integrity != None:
return obj, dict(message=kit_integrity['message'], status="critical")
base_submission, result = obj.pyd.toSQL()
# check output message for issues
match result['code']:
# code 0: everything is fine.
case 0:
result = None
# code 1: ask for overwrite
case 1:
dlg = QuestionAsker(title=f"Review {base_submission.rsl_plate_num}?", message=result['message'])
if dlg.exec():
# Do not add duplicate reagents.
# base_submission.reagents = []
pass
result = None
else:
obj.ctx.database_session.rollback()
return obj, dict(message="Overwrite cancelled", status="Information")
@@ -322,30 +310,18 @@ def submit_new_sample_function(obj:QMainWindow) -> Tuple[QMainWindow, dict]:
update_last_used(ctx=obj.ctx, reagent=reagent, kit=base_submission.extraction_kit)
logger.debug(f"Here is the final submission: {pprint.pformat(base_submission.__dict__)}")
logger.debug(f"Parsed reagents: {pprint.pformat(base_submission.reagents)}")
logger.debug("Checking kit integrity...")
kit_integrity = check_kit_integrity(base_submission)
if kit_integrity != None:
return obj, dict(message=kit_integrity['message'], status="critical")
logger.debug(f"Sending submission: {base_submission.rsl_plate_num} to database.")
# result = store_object(ctx=obj.ctx, object=base_submission)
base_submission.save(ctx=obj.ctx)
# update summary sheet
obj.table_widget.sub_wid.setData()
# reset form
for item in obj.table_widget.formlayout.parentWidget().findChildren(QWidget):
item.setParent(None)
obj.form.setParent(None)
logger.debug(f"All attributes of obj: {pprint.pformat(obj.__dict__)}")
if len(obj.missing_reagents + obj.missing_info) > 0:
logger.debug(f"We have blank reagents in the excel sheet.\n\tLet's try to fill them in.")
extraction_kit = lookup_kit_types(ctx=obj.ctx, name=obj.ext_kit)
logger.debug(f"We have the extraction kit: {extraction_kit.name}")
excel_map = extraction_kit.construct_xl_map_for_use(obj.current_submission_type)
logger.debug(f"Extraction kit map:\n\n{pprint.pformat(excel_map)}")
input_reagents = [item.to_reagent_dict(extraction_kit=base_submission.extraction_kit) for item in base_submission.reagents]
logger.debug(f"Parsed reagents going into autofile: {pprint.pformat(input_reagents)}")
# autofill_excel(obj=obj, xl_map=excel_map, reagents=input_reagents, missing_reagents=obj.missing_reagents, info=info, missing_info=obj.missing_info)
autofill_excel(obj=obj, xl_map=excel_map, reagents=input_reagents, missing_reagents=obj.missing_reagents, info=base_submission.__dict__, missing_info=obj.missing_info)
if hasattr(obj, 'csv'):
wkb = obj.pyd.autofill_excel()
if wkb != None:
fname = select_save_file(obj=obj, default_name=obj.pyd.rsl_plate_num['value'], extension="xlsx")
wkb.save(filename=fname.__str__())
if hasattr(obj.pyd, 'csv'):
dlg = QuestionAsker("Export CSV?", "Would you like to export the csv file?")
if dlg.exec():
fname = select_save_file(obj, f"{base_submission.rsl_plate_num}.csv", extension="csv")
@@ -353,10 +329,6 @@ def submit_new_sample_function(obj:QMainWindow) -> Tuple[QMainWindow, dict]:
obj.csv.to_csv(fname.__str__(), index=False)
except PermissionError:
logger.debug(f"Could not get permissions to {fname}. Possibly the request was cancelled.")
try:
delattr(obj, "csv")
except AttributeError:
pass
return obj, result
def generate_report_function(obj:QMainWindow) -> Tuple[QMainWindow, dict]:
@@ -905,6 +877,8 @@ def autofill_excel(obj:QMainWindow, xl_map:dict, reagents:List[dict], missing_re
fname = select_save_file(obj=obj, default_name=info['rsl_plate_num'], extension="xlsx")
workbook.save(filename=fname.__str__())
def construct_first_strand_function(obj:QMainWindow) -> Tuple[QMainWindow, dict]:
"""
Generates a csv file from client submitted xlsx file.
@@ -1016,23 +990,29 @@ def scrape_reagents(obj:QMainWindow, extraction_kit:str) -> Tuple[QMainWindow, d
Returns:
Tuple[QMainWindow, dict]: Updated application and result
"""
logger.debug("\n\nHello from reagent scraper!!\n\n")
logger.debug(f"Extraction kit: {extraction_kit}")
obj.reagents = []
obj.missing_reagents = []
# obj.reagents = []
# obj.missing_reagents = []
# Remove previous reagent widgets
# [item.setParent(None) for item in obj.table_widget.formlayout.parentWidget().findChildren(QWidget) if item.objectName().startswith("lot_") or item.objectName().startswith("missing_")]
# [item.setParent(None) for item in obj.table_widget.formlayout.parentWidget().findChildren(QPushButton)]
reagents = obj.prsr.parse_reagents(extraction_kit=extraction_kit)
logger.debug(f"Got reagents: {reagents}")
try:
old_reagents = obj.form.find_widgets()
except AttributeError:
logger.error(f"Couldn't find old reagents.")
old_reagents = []
# logger.debug(f"\n\nAttempting to clear: {old_reagents}\n\n")
for reagent in old_reagents:
if isinstance(reagent, ReagentFormWidget) or isinstance(reagent, QPushButton):
reagent.setParent(None)
# reagents = obj.prsr.parse_reagents(extraction_kit=extraction_kit)
# logger.debug(f"Got reagents: {reagents}")
# for reagent in obj.prsr.sub['reagents']:
# # create label
# if reagent.parsed:
# obj.reagents.append(reagent)
# else:
# obj.missing_reagents.append(reagent)
obj.reagents = obj.prsr.sub['reagents']
logger.debug(f"Imported reagents: {obj.reagents}")
logger.debug(f"Missing reagents: {obj.missing_reagents}")
obj.form.reagents = obj.prsr.sub['reagents']
# logger.debug(f"Imported reagents: {obj.reagents}")
# logger.debug(f"Missing reagents: {obj.missing_reagents}")
return obj, None