landon/Submissions-App
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Large scale frontend refactor

Browse Source
This commit is contained in:
Landon Wark
2023-11-14 11:16:54 -06:00
parent bf4149b1b3
commit da714c355d
26 changed files with 2682 additions and 2447 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/submissions/backend/__init__.py
View File

@@ -1,3 +1,6 @@
'''
Contains database, validators and excel operations.
'''
'''
from .db import *
from .excel import *
from .validators import *

27
src/submissions/backend/db/functions.py
View File

@@ -199,7 +199,7 @@ def update_subsampassoc_with_pcr(submission:BasicSubmission, sample:BasicSample,
dict|None: result object
"""
# assoc = lookup_submission_sample_association(ctx, submission=submission, sample=sample)
assoc = SubmissionSampleAssociation.query(submission=submission, sample=sample)
assoc = SubmissionSampleAssociation.query(submission=submission, sample=sample, limit=1)
for k,v in input_dict.items():
try:
setattr(assoc, k, v)
@@ -209,28 +209,3 @@ def update_subsampassoc_with_pcr(submission:BasicSubmission, sample:BasicSample,
result = assoc.save()
return result
# def store_object(ctx:Settings, object) -> dict|None:
# """
# Store an object in the database
# Args:
# ctx (Settings): Settings object passed down from gui
# object (_type_): Object to be stored
# Returns:
# dict|None: Result of action
# """
# dbs = ctx.database_session
# dbs.merge(object)
# try:
# dbs.commit()
# except (SQLIntegrityError, AlcIntegrityError) as e:
# logger.debug(f"Hit an integrity error : {e}")
# dbs.rollback()
# return {"message":f"This object {object} already exists, so we can't add it.\n{e}", "status":"Critical"}
# except (SQLOperationalError, AlcOperationalError):
# logger.error(f"Hit an operational error: {e}")
# dbs.rollback()
# return {"message":"The database is locked for editing."}
# return None

41
src/submissions/backend/db/models/submissions.py
View File

@@ -310,9 +310,10 @@ class BasicSubmission(Base):
return input_excel
@classmethod
def enforce_name(cls, instr:str) -> str:
def enforce_name(cls, instr:str, data:dict|None=None) -> str:
logger.debug(f"Hello from {cls.__mapper_args__['polymorphic_identity']} Enforcer!")
logger.debug(f"Attempting enforcement on {instr}")
logger.debug(f"Attempting enforcement on {instr} using data: {pformat(data)}")
# sys.exit()
return instr
@classmethod
@@ -499,7 +500,7 @@ class BasicSubmission(Base):
cls: _description_
"""
code = 0
msg = None
msg = ""
disallowed = ["id"]
if kwargs == {}:
raise ValueError("Need to narrow down query or the first available instance will be returned.")
@@ -636,9 +637,9 @@ class BacterialCulture(BasicSubmission):
return input_excel
@classmethod
def enforce_name(cls, instr:str) -> str:
outstr = super().enforce_name(instr=instr)
def construct() -> str:
def enforce_name(cls, instr:str, data:dict|None=None) -> str:
outstr = super().enforce_name(instr=instr, data=data)
def construct(data:dict|None=None) -> str:
"""
DEPRECIATED due to slowness. Search for the largest rsl number and increment by 1
@@ -765,18 +766,28 @@ class Wastewater(BasicSubmission):
return samples
@classmethod
def enforce_name(cls, instr:str) -> str:
outstr = super().enforce_name(instr=instr)
def construct():
today = datetime.now()
def enforce_name(cls, instr:str, data:dict|None=None) -> str:
outstr = super().enforce_name(instr=instr, data=data)
def construct(data:dict|None=None):
if "submitted_date" in data.keys():
if data['submitted_date']['value'] != None:
today = data['submitted_date']['value']
else:
today = datetime.now()
else:
today = re.search(r"\d{4}(_|-)?\d{2}(_|-)?\d{2}", instr)
try:
today = parse(today.group())
except AttributeError:
today = datetime.now()
return f"RSL-WW-{today.year}{str(today.month).zfill(2)}{str(today.day).zfill(2)}"
if outstr == None:
outstr = construct()
outstr = construct(data)
try:
outstr = re.sub(r"PCR(-|_)", "", outstr)
except AttributeError as e:
logger.error(f"Problem using regex: {e}")
outstr = construct()
outstr = construct(data)
outstr = outstr.replace("RSLWW", "RSL-WW")
outstr = re.sub(r"WW(\d{4})", r"WW-\1", outstr, flags=re.IGNORECASE)
outstr = re.sub(r"(\d{4})-(\d{2})-(\d{2})", r"\1\2\3", outstr)
@@ -848,9 +859,9 @@ class WastewaterArtic(BasicSubmission):
return input_dict
@classmethod
def enforce_name(cls, instr:str) -> str:
outstr = super().enforce_name(instr=instr)
def construct():
def enforce_name(cls, instr:str, data:dict|None=None) -> str:
outstr = super().enforce_name(instr=instr, data=data)
def construct(data:dict|None=None):
today = datetime.now()
return f"RSL-AR-{today.year}{str(today.month).zfill(2)}{str(today.day).zfill(2)}"
try:

19
src/submissions/backend/excel/parser.py
View File

@@ -2,7 +2,7 @@
contains parser object for pulling values from client generated submission sheets.
'''
from getpass import getuser
import pprint
from pprint import pformat
from typing import List
import pandas as pd
import numpy as np
@@ -15,7 +15,7 @@ import re
from datetime import date
from dateutil.parser import parse, ParserError
from tools import check_not_nan, convert_nans_to_nones, Settings
from frontend.custom_widgets.pop_ups import KitSelector
logger = logging.getLogger(f"submissions.{__name__}")
@@ -70,7 +70,7 @@ class SheetParser(object):
pass
case _:
self.sub[k] = v
logger.debug(f"Parser.sub after info scrape: {pprint.pformat(self.sub)}")
logger.debug(f"Parser.sub after info scrape: {pformat(self.sub)}")
def parse_reagents(self, extraction_kit:str|None=None):
"""
@@ -100,6 +100,7 @@ class SheetParser(object):
Returns:
List[PydReagent]: List of reagents
"""
from frontend.widgets.pop_ups import KitSelector
if not check_not_nan(self.sub['extraction_kit']['value']):
dlg = KitSelector(title="Kit Needed", message="At minimum a kit is needed. Please select one.")
if dlg.exec():
@@ -117,7 +118,7 @@ class SheetParser(object):
# kit = lookup_kit_types(ctx=self.ctx, name=self.sub['extraction_kit']['value'])
kit = KitType.query(name=self.sub['extraction_kit']['value'])
allowed_reagents = [item.name for item in kit.get_reagents()]
logger.debug(f"List of reagents for comparison with allowed_reagents: {pprint.pformat(self.sub['reagents'])}")
logger.debug(f"List of reagents for comparison with allowed_reagents: {pformat(self.sub['reagents'])}")
# self.sub['reagents'] = [reagent for reagent in self.sub['reagents'] if reagent['value'].type in allowed_reagents]
self.sub['reagents'] = [reagent for reagent in self.sub['reagents'] if reagent.type in allowed_reagents]
@@ -133,7 +134,7 @@ class SheetParser(object):
Returns:
PydSubmission: output pydantic model
"""
logger.debug(f"Submission dictionary coming into 'to_pydantic':\n{pprint.pformat(self.sub)}")
logger.debug(f"Submission dictionary coming into 'to_pydantic':\n{pformat(self.sub)}")
psm = PydSubmission(filepath=self.filepath, **self.sub)
# delattr(psm, "filepath")
return psm
@@ -145,7 +146,7 @@ class InfoParser(object):
# self.ctx = ctx
self.map = self.fetch_submission_info_map(submission_type=submission_type)
self.xl = xl
logger.debug(f"Info map for InfoParser: {pprint.pformat(self.map)}")
logger.debug(f"Info map for InfoParser: {pformat(self.map)}")
def fetch_submission_info_map(self, submission_type:str|dict) -> dict:
@@ -186,7 +187,7 @@ class InfoParser(object):
continue
if sheet in self.map[k]['sheets']:
relevant[k] = v
logger.debug(f"relevant map for {sheet}: {pprint.pformat(relevant)}")
logger.debug(f"relevant map for {sheet}: {pformat(relevant)}")
if relevant == {}:
continue
for item in relevant:
@@ -236,7 +237,7 @@ class ReagentParser(object):
df = self.xl.parse(sheet, header=None, dtype=object)
df.replace({np.nan: None}, inplace = True)
relevant = {k.strip():v for k,v in self.map.items() if sheet in self.map[k]['sheet']}
logger.debug(f"relevant map for {sheet}: {pprint.pformat(relevant)}")
logger.debug(f"relevant map for {sheet}: {pformat(relevant)}")
if relevant == {}:
continue
for item in relevant:
@@ -302,7 +303,7 @@ class SampleParser(object):
logger.debug(f"Looking up submission type: {submission_type}")
# submission_type = lookup_submission_type(ctx=self.ctx, name=submission_type)
submission_type = SubmissionType.query(name=submission_type)
logger.debug(f"info_map: {pprint.pformat(submission_type.info_map)}")
logger.debug(f"info_map: {pformat(submission_type.info_map)}")
sample_info_map = submission_type.info_map['samples']
# self.custom_parser = get_polymorphic_subclass(models.BasicSubmission, submission_type.name).parse_samples
self.custom_sub_parser = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=submission_type.name).parse_samples

6
src/submissions/backend/excel/reports.py
View File

@@ -6,7 +6,7 @@ import logging
from datetime import date, timedelta
import re
from typing import Tuple
from tools import jinja_template_loading
from tools import jinja_template_loading, Settings
logger = logging.getLogger(f"submissions.{__name__}")
@@ -198,7 +198,7 @@ def get_unique_values_in_df_column(df: DataFrame, column_name: str) -> list:
return sorted(df[column_name].unique())
def drop_reruns_from_df(ctx:dict, df: DataFrame) -> DataFrame:
def drop_reruns_from_df(ctx:Settings, df: DataFrame) -> DataFrame:
"""
Removes semi-duplicates from dataframe after finding sequencing repeats.
@@ -211,7 +211,7 @@ def drop_reruns_from_df(ctx:dict, df: DataFrame) -> DataFrame:
"""
if 'rerun_regex' in ctx:
sample_names = get_unique_values_in_df_column(df, column_name="name")
rerun_regex = re.compile(fr"{ctx['rerun_regex']}")
rerun_regex = re.compile(fr"{ctx.rerun_regex}")
for sample in sample_names:
if rerun_regex.search(sample):
first_run = re.sub(rerun_regex, "", sample)

6
src/submissions/backend/validators/__init__.py
View File

@@ -10,7 +10,7 @@ class RSLNamer(object):
"""
Object that will enforce proper formatting on RSL plate names.
"""
def __init__(self, instr:str, sub_type:str|None=None):
def __init__(self, instr:str, sub_type:str|None=None, data:dict|None=None):
self.submission_type = sub_type
if self.submission_type == None:
@@ -19,7 +19,7 @@ class RSLNamer(object):
if self.submission_type != None:
enforcer = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=self.submission_type)
self.parsed_name = self.retrieve_rsl_number(instr=instr, regex=enforcer.get_regex())
self.parsed_name = enforcer.enforce_name(instr=self.parsed_name)
self.parsed_name = enforcer.enforce_name(instr=self.parsed_name, data=data)
@classmethod
def retrieve_submission_type(cls, instr:str|Path) -> str:
@@ -58,7 +58,7 @@ class RSLNamer(object):
except UnboundLocalError:
check = True
if check:
from frontend.custom_widgets import SubmissionTypeSelector
from frontend.widgets import SubmissionTypeSelector
dlg = SubmissionTypeSelector(title="Couldn't parse submission type.", message="Please select submission type from list below.")
if dlg.exec():
submission_type = dlg.parse_form()

28
src/submissions/backend/validators/pydant.py
View File

@@ -11,7 +11,7 @@ from . import RSLNamer
from pathlib import Path
import re
import logging
from tools import check_not_nan, convert_nans_to_nones, jinja_template_loading
from tools import check_not_nan, convert_nans_to_nones, jinja_template_loading, Report, Result
from backend.db.models import *
from sqlalchemy.exc import StatementError
from PyQt6.QtWidgets import QComboBox, QWidget
@@ -86,8 +86,8 @@ class PydReagent(BaseModel):
else:
return values.data['type']
def toSQL(self) -> Tuple[Reagent, dict]:
result = None
def toSQL(self) -> Tuple[Reagent, Report]:
report = Report()
logger.debug(f"Reagent SQL constructor is looking up type: {self.type}, lot: {self.lot}")
# reagent = lookup_reagents(ctx=self.ctx, lot_number=self.lot)
reagent = Reagent.query(lot_number=self.lot)
@@ -113,10 +113,10 @@ class PydReagent(BaseModel):
reagent.name = value
# add end-of-life extension from reagent type to expiry date
# NOTE: this will now be done only in the reporting phase to account for potential changes in end-of-life extensions
return reagent, result
return reagent, report
def toForm(self, parent:QWidget, extraction_kit:str) -> QComboBox:
from frontend.custom_widgets.misc import ReagentFormWidget
from frontend.widgets.submission_widget import ReagentFormWidget
return ReagentFormWidget(parent=parent, reagent=self, extraction_kit=extraction_kit)
class PydSample(BaseModel, extra='allow'):
@@ -180,8 +180,9 @@ class PydSubmission(BaseModel, extra='allow'):
submission_type: dict|None
# For defaults
submitter_plate_num: dict|None = Field(default=dict(value=None, missing=True), validate_default=True)
rsl_plate_num: dict|None = Field(default=dict(value=None, missing=True), validate_default=True)
submitted_date: dict|None
rsl_plate_num: dict|None = Field(default=dict(value=None, missing=True), validate_default=True)
# submitted_date: dict|None
submitting_lab: dict|None
sample_count: dict|None
extraction_kit: dict|None
@@ -243,7 +244,7 @@ class PydSubmission(BaseModel, extra='allow'):
@field_validator("rsl_plate_num")
@classmethod
def rsl_from_file(cls, value, values):
logger.debug(f"RSL-plate initial value: {value['value']}")
logger.debug(f"RSL-plate initial value: {value['value']} and other values: {values.data}")
sub_type = values.data['submission_type']['value']
if check_not_nan(value['value']):
# if lookup_submissions(ctx=values.data['ctx'], rsl_number=value['value']) == None:
@@ -256,7 +257,7 @@ class PydSubmission(BaseModel, extra='allow'):
# return dict(value=output, missing=True)
return value
else:
output = RSLNamer(instr=values.data['filepath'].__str__(), sub_type=sub_type).parsed_name
output = RSLNamer(instr=values.data['filepath'].__str__(), sub_type=sub_type, data=values.data).parsed_name
return dict(value=output, missing=True)
@field_validator("technician", mode="before")
@@ -346,12 +347,11 @@ class PydSubmission(BaseModel, extra='allow'):
missing_reagents = [reagent for reagent in self.reagents if reagent.missing]
return missing_info, missing_reagents
def toSQL(self):
code = 0
msg = None
status = None
def toSQL(self) -> Tuple[BasicSubmission, Result]:
self.__dict__.update(self.model_extra)
instance, code, msg = BasicSubmission.query_or_create(submission_type=self.submission_type['value'], rsl_plate_num=self.rsl_plate_num['value'])
result = Result(msg=msg, code=code)
self.handle_duplicate_samples()
logger.debug(f"Here's our list of duplicate removed samples: {self.samples}")
for key, value in self.__dict__.items():
@@ -389,10 +389,10 @@ class PydSubmission(BaseModel, extra='allow'):
except AttributeError as e:
logger.debug(f"Something went wrong constructing instance {self.rsl_plate_num}: {e}")
logger.debug(f"Constructed submissions message: {msg}")
return instance, {'code':code, 'message':msg, 'status':"Information"}
return instance, result
def toForm(self, parent:QWidget):
from frontend.custom_widgets.misc import SubmissionFormWidget
from frontend.widgets.submission_widget import SubmissionFormWidget
return SubmissionFormWidget(parent=parent, **self.improved_dict())
def autofill_excel(self, missing_only:bool=True):