diff --git a/CHANGELOG.md b/CHANGELOG.md index 12c914e..6450fba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## 202306.02 + +- Addition of bacterial plate maps to details export. +- Change in Artic cost calculation to reflect multiple output plates per submission. + ## 202306.01 - Large scale shake up of import and scraper functions. diff --git a/TODO.md b/TODO.md index 07caafa..7d477eb 100644 --- a/TODO.md +++ b/TODO.md @@ -1,4 +1,5 @@ -- [ ] Create a method for creation of hitpicking .csvs because of reasons that may or may not exist. +- [ ] Improve plate mapping by using layout in submission forms rather than PCR. +- [x] Create a method for creation of hitpicking .csvs because of reasons that may or may not exist. - [x] Create a method for commenting submissions. - [x] Create barcode generator, because of reasons that may or may not exist. - [x] Move bulk of functions from frontend.__init__ to frontend.functions as __init__ is getting bloated. \ No newline at end of file diff --git a/src/submissions/__init__.py b/src/submissions/__init__.py index 3b7d213..6bec4e6 100644 --- a/src/submissions/__init__.py +++ b/src/submissions/__init__.py @@ -4,7 +4,7 @@ from pathlib import Path # Version of the realpython-reader package __project__ = "submissions" -__version__ = "202306.1b" +__version__ = "202306.2b" __author__ = {"name":"Landon Wark", "email":"Landon.Wark@phac-aspc.gc.ca"} __copyright__ = "2022-2023, Government of Canada" diff --git a/src/submissions/backend/db/functions.py b/src/submissions/backend/db/functions.py index cc769c3..06db8c1 100644 --- a/src/submissions/backend/db/functions.py +++ b/src/submissions/backend/db/functions.py @@ -31,6 +31,7 @@ def set_sqlite_pragma(dbapi_connection, connection_record): cursor.execute("PRAGMA foreign_keys=ON") cursor.close() + def store_submission(ctx:dict, base_submission:models.BasicSubmission) -> None|dict: """ Upserts submissions into database @@ -799,9 +800,21 @@ def lookup_discounts_by_org_and_kit(ctx:dict, kit_id:int, lab_id:int): )).all() def hitpick_plate(submission:models.BasicSubmission, plate_number:int=0) -> list: + """ + Creates a list of sample positions and statuses to be used by plate mapping and csv output to biomek software. + + Args: + submission (models.BasicSubmission): Input submission + plate_number (int, optional): plate position in the series of selected plates. Defaults to 0. + + Returns: + list: list of sample dictionaries. + """ plate_dicto = [] for sample in submission.samples: # have sample report back its info if it's positive, otherwise, None + method_list = [func for func in dir(sample) if callable(getattr(sample, func))] + logger.debug(f"Method list of sample: {method_list}") samp = sample.to_hitpick() if samp == None: continue @@ -811,6 +824,43 @@ def hitpick_plate(submission:models.BasicSubmission, plate_number:int=0) -> list # if len(dicto) < 88: this_sample = dict( plate_number = plate_number, + sample_name = samp['name'], + column = samp['col'], + row = samp['row'], + positive = samp['positive'], + plate_name = submission.rsl_plate_num + ) + # append to plate samples + plate_dicto.append(this_sample) + # append to all samples + # image = make_plate_map(plate_dicto) + return plate_dicto + +def platemap_plate(submission:models.BasicSubmission) -> list: + """ + Depreciated. Replaced by new functionality in hitpick_plate + + Args: + submission (models.BasicSubmission): Input submission + + Returns: + list: list of sample dictionaries + """ + plate_dicto = [] + for sample in submission.samples: + # have sample report back its info if it's positive, otherwise, None + + try: + samp = sample.to_platemap() + except AttributeError: + continue + if samp == None: + continue + else: + logger.debug(f"Item name: {samp['name']}") + # plate can handle 88 samples to leave column for controls + # if len(dicto) < 88: + this_sample = dict( sample_name = samp['name'], column = samp['col'], row = samp['row'], diff --git a/src/submissions/backend/db/models/samples.py b/src/submissions/backend/db/models/samples.py index 9409c27..f2ab46d 100644 --- a/src/submissions/backend/db/models/samples.py +++ b/src/submissions/backend/db/models/samples.py @@ -62,8 +62,10 @@ class WWSample(Base): # if well_col > 4: # well if self.ct_n1 != None and self.ct_n2 != None: + # logger.debug(f"Using well info in name.") name = f"{self.ww_sample_full_id}\n\t- ct N1: {'{:.2f}'.format(self.ct_n1)} ({self.n1_status})\n\t- ct N2: {'{:.2f}'.format(self.ct_n2)} ({self.n2_status})" else: + # logger.debug(f"NOT using well info in name for: {self.ww_sample_full_id}") name = self.ww_sample_full_id return { "well": self.well_number, @@ -85,18 +87,23 @@ class WWSample(Base): except TypeError as e: logger.error(f"Couldn't check positives for {self.rsl_number}. Looks like there isn't PCR data.") return None - if positive: - try: - # The first character of the elution well is the row - well_row = row_dict[self.elution_well[0]] - # The remaining charagers are the columns - well_col = self.elution_well[1:] - except TypeError as e: - logger.error(f"This sample doesn't have elution plate info.") - return None - return dict(name=self.ww_sample_full_id, row=well_row, col=well_col) - else: - return None + well_row = row_dict[self.elution_well[0]] + well_col = self.elution_well[1:] + # if positive: + # try: + # # The first character of the elution well is the row + # well_row = row_dict[self.elution_well[0]] + # # The remaining charagers are the columns + # well_col = self.elution_well[1:] + # except TypeError as e: + # logger.error(f"This sample doesn't have elution plate info.") + # return None + return dict(name=self.ww_sample_full_id, + row=well_row, + col=well_col, + positive=positive) + # else: + # return None class BCSample(Base): @@ -134,7 +141,24 @@ class BCSample(Base): "name": f"{self.sample_id} - ({self.organism})", } + def to_hitpick(self) -> dict|None: + """ + Outputs a dictionary of locations + Returns: + dict: dictionary of sample id, row and column in elution plate + """ + # dictionary to translate row letters into numbers + row_dict = dict(A=1, B=2, C=3, D=4, E=5, F=6, G=7, H=8) + # if either n1 or n2 is positive, include this sample + well_row = row_dict[self.well_number[0]] + # The remaining charagers are the columns + well_col = self.well_number[1:] + return dict(name=self.sample_id, + row=well_row, + col=well_col, + positive=False) + # class ArticSample(Base): # """ # base of artic sample diff --git a/src/submissions/backend/db/models/submissions.py b/src/submissions/backend/db/models/submissions.py index fd81312..dbe2a04 100644 --- a/src/submissions/backend/db/models/submissions.py +++ b/src/submissions/backend/db/models/submissions.py @@ -1,6 +1,7 @@ ''' Models for the main submission types. ''' +import math from . import Base from sqlalchemy import Column, String, TIMESTAMP, INTEGER, ForeignKey, Table, JSON, FLOAT from sqlalchemy.orm import relationship @@ -246,5 +247,24 @@ class WastewaterArtic(BasicSubmission): derivative submission type for artic wastewater """ samples = relationship("WWSample", back_populates="artic_rsl_plate", uselist=True) - # Can in use the pcr_info from the wastewater? Cause I can't define pcr_info here due to conflicts with that - __mapper_args__ = {"polymorphic_identity": "wastewater_artic", "polymorphic_load": "inline"} \ No newline at end of file + # Can it use the pcr_info from the wastewater? Cause I can't define pcr_info here due to conflicts with that + # Not necessary because we don't get any results for this procedure. + __mapper_args__ = {"polymorphic_identity": "wastewater_artic", "polymorphic_load": "inline"} + + def calculate_base_cost(self): + """ + This method overrides parent method due to multiple output plates from a single submission + """ + logger.debug(f"Hello from calculate base cost in WWArtic") + try: + cols_count_96 = ceil(int(self.sample_count) / 8) + except Exception as e: + logger.error(f"Column count error: {e}") + # Since we have multiple output plates per submission form, the constant cost will have to reflect this. + output_plate_count = math.ceil(int(self.sample_count) / 16) + logger.debug(f"Looks like we have {output_plate_count} output plates.") + const_cost = self.extraction_kit.constant_cost * output_plate_count + try: + self.run_cost = const_cost + (self.extraction_kit.mutable_cost_column * cols_count_96) + (self.extraction_kit.mutable_cost_sample * int(self.sample_count)) + except Exception as e: + logger.error(f"Calculation error: {e}") diff --git a/src/submissions/backend/excel/parser.py b/src/submissions/backend/excel/parser.py index 28b868e..8fdd4d5 100644 --- a/src/submissions/backend/excel/parser.py +++ b/src/submissions/backend/excel/parser.py @@ -3,6 +3,7 @@ contains parser object for pulling values from client generated submission sheet ''' from getpass import getuser import math +import pprint from typing import Tuple import pandas as pd from pathlib import Path @@ -160,14 +161,19 @@ class SheetParser(object): sample_parser = SampleParser(self.ctx, submission_info.iloc[16:112]) sample_parse = getattr(sample_parser, f"parse_{self.sub['submission_type'].lower()}_samples") logger.debug(f"Parser result: {self.sub}") - self.sub['samples'] = sample_parse() + self.sample_result, self.sub['samples'] = sample_parse() def parse_wastewater(self) -> None: """ pulls info specific to wastewater sample type """ - + def retrieve_elution_map(): + full = self.xl.parse("Extraction Worksheet") + elu_map = full.iloc[9:18, 5:] + elu_map.set_index(elu_map.columns[0], inplace=True) + elu_map.columns = elu_map.iloc[0] + return elu_map def parse_reagents(df:pd.DataFrame) -> None: """ Pulls reagents from the bacterial sub-dataframe @@ -216,9 +222,9 @@ class SheetParser(object): parse_reagents(ext_reagent_range) parse_reagents(pcr_reagent_range) # parse samples - sample_parser = SampleParser(self.ctx, submission_info.iloc[16:]) + sample_parser = SampleParser(self.ctx, submission_info.iloc[16:], elution_map=retrieve_elution_map()) sample_parse = getattr(sample_parser, f"parse_{self.sub['submission_type'].lower()}_samples") - self.sub['samples'] = sample_parse() + self.sample_result, self.sub['samples'] = sample_parse() self.sub['csv'] = self.xl.parse("Copy to import file", dtype=object) @@ -272,7 +278,7 @@ class SheetParser(object): return_list.append(dict(sample_name=re.sub(r"\s?\(.*\)", "", df.loc[ii.name, int(c)]), \ well=f"{ii.name}{c}", artic_plate=self.sub['rsl_plate_num'])) - logger.debug(f"massaged sample list for {self.sub['rsl_plate_num']}: {return_list}") + logger.debug(f"massaged sample list for {self.sub['rsl_plate_num']}: {pprint.pprint(return_list)}") return return_list submission_info = self.xl.parse("First Strand", dtype=object) biomek_info = self.xl.parse("ArticV4 Biomek", dtype=object) @@ -280,7 +286,7 @@ class SheetParser(object): biomek_reagent_range = biomek_info.iloc[60:, 0:3].dropna(how='all') self.sub['submitter_plate_num'] = "" self.sub['rsl_plate_num'] = RSLNamer(self.filepath.__str__()).parsed_name - self.sub['submitted_date'] = submission_info.iloc[0][2] + self.sub['submitted_date'] = biomek_info.iloc[1][1] self.sub['submitting_lab'] = "Enterics Wastewater Genomics" self.sub['sample_count'] = submission_info.iloc[4][6] self.sub['extraction_kit'] = "ArticV4.1" @@ -290,7 +296,7 @@ class SheetParser(object): samples = massage_samples(biomek_info.iloc[22:31, 0:]) sample_parser = SampleParser(self.ctx, pd.DataFrame.from_records(samples)) sample_parse = getattr(sample_parser, f"parse_{self.sub['submission_type'].lower()}_samples") - self.sub['samples'] = sample_parse() + self.sample_result, self.sub['samples'] = sample_parse() @@ -299,18 +305,21 @@ class SampleParser(object): object to pull data for samples in excel sheet and construct individual sample objects """ - def __init__(self, ctx:dict, df:pd.DataFrame) -> None: + def __init__(self, ctx:dict, df:pd.DataFrame, elution_map:pd.DataFrame|None=None) -> None: """ convert sample sub-dataframe to dictionary of records Args: + ctx (dict): setting passed down from gui df (pd.DataFrame): input sample dataframe + elution_map (pd.DataFrame | None, optional): optional map of elution plate. Defaults to None. """ self.ctx = ctx self.samples = df.to_dict("records") + self.elution_map = elution_map - def parse_bacterial_culture_samples(self) -> list[BCSample]: + def parse_bacterial_culture_samples(self) -> Tuple[str|None, list[BCSample]]: """ construct bacterial culture specific sample objects @@ -334,16 +343,28 @@ class SampleParser(object): not_a_nan = True if not_a_nan: new_list.append(new) - return new_list + return None, new_list - def parse_wastewater_samples(self) -> list[WWSample]: + def parse_wastewater_samples(self) -> Tuple[str|None, list[WWSample]]: """ construct wastewater specific sample objects Returns: list[WWSample]: list of sample objects """ + def search_df_for_sample(sample_rsl:str): + logger.debug(f"Attempting to find sample {sample_rsl} in \n {self.elution_map}") + print(f"Attempting to find sample {sample_rsl} in \n {self.elution_map}") + well = self.elution_map.where(self.elution_map==sample_rsl).dropna(how='all').dropna(axis=1) + self.elution_map.at[well.index[0], well.columns[0]] = np.nan + try: + col = str(int(well.columns[0])) + except ValueError: + col = str(well.columns[0]) + except TypeError as e: + logger.error(f"Problem parsing out column number for {well}:\n {e}") + return f"{well.index[0]}{col}" new_list = [] for sample in self.samples: new = WWSample() @@ -368,10 +389,11 @@ class SampleParser(object): # new.site_status = sample['Unnamed: 7'] new.notes = str(sample['Unnamed: 6']) # previously Unnamed: 8 new.well_number = sample['Unnamed: 1'] + new.elution_well = search_df_for_sample(new.rsl_number) new_list.append(new) - return new_list + return None, new_list - def parse_wastewater_artic_samples(self) -> list[WWSample]: + def parse_wastewater_artic_samples(self) -> Tuple[str|None, list[WWSample]]: """ The artic samples are the wastewater samples that are to be sequenced So we will need to lookup existing ww samples and append Artic well # and plate relation @@ -380,17 +402,20 @@ class SampleParser(object): list[WWSample]: list of wastewater samples to be updated """ new_list = [] + missed_samples = [] for sample in self.samples: with self.ctx['database_session'].no_autoflush: instance = lookup_ww_sample_by_ww_sample_num(ctx=self.ctx, sample_number=sample['sample_name']) logger.debug(f"Checking: {sample['sample_name']}") if instance == None: logger.error(f"Unable to find match for: {sample['sample_name']}") + missed_samples.append(sample['sample_name']) continue logger.debug(f"Got instance: {instance.ww_sample_full_id}") instance.artic_well_number = sample['well'] new_list.append(instance) - return new_list + missed_str = "\n\t".join(missed_samples) + return f"Could not find matches for the following samples:\n\t {missed_str}", new_list @@ -472,6 +497,7 @@ class PCRParser(object): df = self.parse_general(sheet_name="Results") column_names = ["Well", "Well Position", "Omit","Sample","Target","Task"," Reporter","Quencher","Amp Status","Amp Score","Curve Quality","Result Quality Issues","Cq","Cq Confidence","Cq Mean","Cq SD","Auto Threshold","Threshold", "Auto Baseline", "Baseline Start", "Baseline End"] self.samples_df = df.iloc[23:][0:] + logger.debug(f"Dataframe of PCR results:\n\t{self.samples_df}") self.samples_df.columns = column_names logger.debug(f"Samples columns: {self.samples_df.columns}") well_call_df = self.xl.parse(sheet_name="Well Call").iloc[24:][0:].iloc[:,-1:] @@ -488,7 +514,7 @@ class PCRParser(object): sample_obj = dict( sample = row['Sample'], plate_rsl = self.plate_num, - elution_well = row['Well Position'] + # elution_well = row['Well Position'] ) logger.debug(f"Got sample obj: {sample_obj}") # logger.debug(f"row: {row}") diff --git a/src/submissions/frontend/custom_widgets/sub_details.py b/src/submissions/frontend/custom_widgets/sub_details.py index cfb43e3..04a36af 100644 --- a/src/submissions/frontend/custom_widgets/sub_details.py +++ b/src/submissions/frontend/custom_widgets/sub_details.py @@ -114,6 +114,10 @@ class SubmissionsSheet(QTableView): del self.data['reagents'] except KeyError: pass + try: + del self.data['comments'] + except KeyError: + pass proxyModel = QSortFilterProxyModel() proxyModel.setSourceModel(pandasModel(self.data)) self.setModel(proxyModel) @@ -226,11 +230,12 @@ class SubmissionsSheet(QTableView): else: logger.error(f"We had to truncate the number of samples to 94.") logger.debug(f"We found {len(dicto)} to hitpick") - msg = AlertPop(message=f"We found {len(dicto)} samples to hitpick", status="INFORMATION") - msg.exec() # convert all samples to dataframe df = make_hitpicks(dicto) - logger.debug(f"Size of the dataframe: {df.size}") + df = df[df.positive != False] + logger.debug(f"Size of the dataframe: {df.shape[0]}") + msg = AlertPop(message=f"We found {df.shape[0]} samples to hitpick", status="INFORMATION") + msg.exec() if df.size == 0: return date = datetime.strftime(datetime.today(), "%Y-%m-%d") @@ -264,6 +269,7 @@ class SubmissionDetails(QDialog): interior.setParent(self) # get submision from db data = lookup_submission_by_id(ctx=ctx, id=id) + logger.debug(f"Submission details data:\n{data.to_dict()}") self.base_dict = data.to_dict() # don't want id del self.base_dict['id'] @@ -308,8 +314,11 @@ class SubmissionDetails(QDialog): platemap = make_plate_map(plate_dicto) logger.debug(f"platemap: {platemap}") image_io = BytesIO() - platemap.save(image_io, 'JPEG') - platemap.save("test.jpg", 'JPEG') + try: + platemap.save(image_io, 'JPEG') + except AttributeError: + logger.error(f"No plate map found for {sub.rsl_plate_num}") + # platemap.save("test.jpg", 'JPEG') self.base_dict['platemap'] = base64.b64encode(image_io.getvalue()).decode('utf-8') logger.debug(self.base_dict) html = template.render(sub=self.base_dict) diff --git a/src/submissions/frontend/main_window_functions.py b/src/submissions/frontend/main_window_functions.py index 108fe0e..da00665 100644 --- a/src/submissions/frontend/main_window_functions.py +++ b/src/submissions/frontend/main_window_functions.py @@ -26,7 +26,7 @@ from backend.db.functions import ( lookup_all_orgs, lookup_kittype_by_use, lookup_kittype_by_name, construct_submission_info, lookup_reagent, store_submission, lookup_submissions_by_date_range, create_kit_from_yaml, create_org_from_yaml, get_control_subtypes, get_all_controls_by_type, - lookup_all_submissions_by_type, get_all_controls, lookup_submission_by_rsl_num, update_ww_sample + lookup_all_submissions_by_type, get_all_controls, lookup_submission_by_rsl_num, update_ww_sample, hitpick_plate ) from backend.excel.parser import SheetParser, PCRParser from backend.excel.reports import make_report_html, make_report_xlsx, convert_data_list_to_df @@ -35,6 +35,7 @@ from .custom_widgets.pop_ups import AlertPop, QuestionAsker from .custom_widgets import ReportDatePicker, ReagentTypeForm from .custom_widgets.misc import ImportReagent from .visualizations.control_charts import create_charts, construct_html +from .visualizations import make_plate_map logger = logging.getLogger(f"submissions.{__name__}") @@ -60,6 +61,7 @@ def import_submission_function(obj:QMainWindow) -> Tuple[QMainWindow, dict|None] if prsr.sub['rsl_plate_num'] == None: prsr.sub['rsl_plate_num'] = RSLNamer(fname.__str__()).parsed_name logger.debug(f"prsr.sub = {prsr.sub}") + obj.current_submission_type = prsr.sub['submission_type'] # destroy any widgets from previous imports for item in obj.table_widget.formlayout.parentWidget().findChildren(QWidget): item.setParent(None) @@ -111,7 +113,7 @@ def import_submission_function(obj:QMainWindow) -> Tuple[QMainWindow, dict|None] uses.insert(0, uses.pop(uses.index(prsr.sub[item]))) obj.ext_kit = prsr.sub[item] else: - logger.error(f"Couldn't find prsr.sub[extraction_kit]") + logger.error(f"Couldn't find {prsr.sub['extraction_kit']}") obj.ext_kit = uses[0] add_widget.addItems(uses) case 'submitted_date': @@ -156,6 +158,9 @@ def import_submission_function(obj:QMainWindow) -> Tuple[QMainWindow, dict|None] if hasattr(obj, 'ext_kit'): obj.kit_integrity_completion() logger.debug(f"Imported reagents: {obj.reagents}") + if prsr.sample_result != None: + msg = AlertPop(message=prsr.sample_result, status="WARNING") + msg.exec() return obj, result def kit_reload_function(obj:QMainWindow) -> QMainWindow: @@ -263,6 +268,7 @@ def submit_new_sample_function(obj:QMainWindow) -> QMainWindow: # reset form for item in obj.table_widget.formlayout.parentWidget().findChildren(QWidget): item.setParent(None) + logger.debug(f"All attributes of obj: {pprint.pprint(obj.__dict__)}") if hasattr(obj, 'csv'): dlg = QuestionAsker("Export CSV?", "Would you like to export the csv file?") if dlg.exec(): @@ -271,6 +277,14 @@ def submit_new_sample_function(obj:QMainWindow) -> QMainWindow: obj.csv.to_csv(fname.__str__(), index=False) except PermissionError: logger.debug(f"Could not get permissions to {fname}. Possibly the request was cancelled.") + try: + delattr(obj, "csv") + except AttributeError: + pass + # if obj.current_submission_type == "Bacterial_Culture": + # hitpick = hitpick_plate(base_submission) + # image = make_plate_map(hitpick) + # image.show() return obj, result def generate_report_function(obj:QMainWindow) -> QMainWindow: diff --git a/src/submissions/frontend/visualizations/plate_map.py b/src/submissions/frontend/visualizations/plate_map.py index 20e0af6..19352c4 100644 --- a/src/submissions/frontend/visualizations/plate_map.py +++ b/src/submissions/frontend/visualizations/plate_map.py @@ -12,7 +12,7 @@ def make_plate_map(sample_list:list) -> Image: Makes a pillow image of a plate from hitpicks Args: - sample_list (list): list of positive sample dictionaries from the hitpicks + sample_list (list): list of sample dictionaries from the hitpicks Returns: Image: Image of the 96 well plate with positive samples in red. @@ -26,12 +26,17 @@ def make_plate_map(sample_list:list) -> Image: except TypeError as e: logger.error(f"No samples for this plate. Nothing to do.") return None - # Make a 8 row, 12 column, 3 color ints array, filled with white by default + # Make an 8 row, 12 column, 3 color ints array, filled with white by default grid = np.full((8,12,3),255, dtype=np.uint8) - # Go through samples and change its row/column to red + # Go through samples and change its row/column to red if positive, else blue for sample in sample_list: - grid[int(sample['row'])-1][int(sample['column'])-1] = [255,0,0] - # Create image from the grid + logger.debug(f"sample keys: {list(sample.keys())}") + if sample['positive']: + colour = [255,0,0] + else: + colour = [0,0,255] + grid[int(sample['row'])-1][int(sample['column'])-1] = colour + # Create pixel image from the grid and enlarge img = Image.fromarray(grid).resize((1200, 800), resample=Image.NEAREST) # create a drawer over the image draw = ImageDraw.Draw(img) @@ -58,7 +63,6 @@ def make_plate_map(sample_list:list) -> Image: new_img.paste(img, box) # create drawer over the new image draw = ImageDraw.Draw(new_img) - # font = ImageFont.truetype("sans-serif.ttf", 16) if check_if_app(): font_path = Path(sys._MEIPASS).joinpath("files", "resources") else: diff --git a/src/submissions/tools/__init__.py b/src/submissions/tools/__init__.py index 420a125..f013ee8 100644 --- a/src/submissions/tools/__init__.py +++ b/src/submissions/tools/__init__.py @@ -260,4 +260,5 @@ def massage_common_reagents(reagent_name:str): reagent_name = "molecular_grade_water" reagent_name = reagent_name.replace("ยต", "u") return reagent_name - \ No newline at end of file + +