landon/Submissions-App
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Missing sample message after Artic parsing.

Browse Source
This commit is contained in:
Landon Wark
2023-06-16 13:58:28 -05:00
parent a7132cd1b4
commit 0bdcad0eee
11 changed files with 199 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGELOG.md
View File

@@ -1,3 +1,8 @@
## 202306.02
- Addition of bacterial plate maps to details export.
- Change in Artic cost calculation to reflect multiple output plates per submission.
## 202306.01
- Large scale shake up of import and scraper functions.

3
TODO.md
View File

@@ -1,4 +1,5 @@
- [ ] Create a method for creation of hitpicking .csvs because of reasons that may or may not exist.
- [ ] Improve plate mapping by using layout in submission forms rather than PCR.
- [x] Create a method for creation of hitpicking .csvs because of reasons that may or may not exist.
- [x] Create a method for commenting submissions.
- [x] Create barcode generator, because of reasons that may or may not exist.
- [x] Move bulk of functions from frontend.__init__ to frontend.functions as __init__ is getting bloated.

2
src/submissions/__init__.py
View File

@@ -4,7 +4,7 @@ from pathlib import Path
# Version of the realpython-reader package
__project__ = "submissions"
__version__ = "202306.1b"
__version__ = "202306.2b"
__author__ = {"name":"Landon Wark", "email":"Landon.Wark@phac-aspc.gc.ca"}
__copyright__ = "2022-2023, Government of Canada"

50
src/submissions/backend/db/functions.py
View File

@@ -31,6 +31,7 @@ def set_sqlite_pragma(dbapi_connection, connection_record):
cursor.execute("PRAGMA foreign_keys=ON")
cursor.close()
def store_submission(ctx:dict, base_submission:models.BasicSubmission) -> None|dict:
"""
Upserts submissions into database
@@ -799,9 +800,21 @@ def lookup_discounts_by_org_and_kit(ctx:dict, kit_id:int, lab_id:int):
)).all()
def hitpick_plate(submission:models.BasicSubmission, plate_number:int=0) -> list:
"""
Creates a list of sample positions and statuses to be used by plate mapping and csv output to biomek software.
Args:
submission (models.BasicSubmission): Input submission
plate_number (int, optional): plate position in the series of selected plates. Defaults to 0.
Returns:
list: list of sample dictionaries.
"""
plate_dicto = []
for sample in submission.samples:
# have sample report back its info if it's positive, otherwise, None
method_list = [func for func in dir(sample) if callable(getattr(sample, func))]
logger.debug(f"Method list of sample: {method_list}")
samp = sample.to_hitpick()
if samp == None:
continue
@@ -811,6 +824,43 @@ def hitpick_plate(submission:models.BasicSubmission, plate_number:int=0) -> list
# if len(dicto) < 88:
this_sample = dict(
plate_number = plate_number,
sample_name = samp['name'],
column = samp['col'],
row = samp['row'],
positive = samp['positive'],
plate_name = submission.rsl_plate_num
)
# append to plate samples
plate_dicto.append(this_sample)
# append to all samples
# image = make_plate_map(plate_dicto)
return plate_dicto
def platemap_plate(submission:models.BasicSubmission) -> list:
"""
Depreciated. Replaced by new functionality in hitpick_plate
Args:
submission (models.BasicSubmission): Input submission
Returns:
list: list of sample dictionaries
"""
plate_dicto = []
for sample in submission.samples:
# have sample report back its info if it's positive, otherwise, None
try:
samp = sample.to_platemap()
except AttributeError:
continue
if samp == None:
continue
else:
logger.debug(f"Item name: {samp['name']}")
# plate can handle 88 samples to leave column for controls
# if len(dicto) < 88:
this_sample = dict(
sample_name = samp['name'],
column = samp['col'],
row = samp['row'],

48
src/submissions/backend/db/models/samples.py
View File

@@ -62,8 +62,10 @@ class WWSample(Base):
# if well_col > 4:
# well
if self.ct_n1 != None and self.ct_n2 != None:
# logger.debug(f"Using well info in name.")
name = f"{self.ww_sample_full_id}\n\t- ct N1: {'{:.2f}'.format(self.ct_n1)} ({self.n1_status})\n\t- ct N2: {'{:.2f}'.format(self.ct_n2)} ({self.n2_status})"
else:
# logger.debug(f"NOT using well info in name for: {self.ww_sample_full_id}")
name = self.ww_sample_full_id
return {
"well": self.well_number,
@@ -85,18 +87,23 @@ class WWSample(Base):
except TypeError as e:
logger.error(f"Couldn't check positives for {self.rsl_number}. Looks like there isn't PCR data.")
return None
if positive:
try:
# The first character of the elution well is the row
well_row = row_dict[self.elution_well[0]]
# The remaining charagers are the columns
well_col = self.elution_well[1:]
except TypeError as e:
logger.error(f"This sample doesn't have elution plate info.")
return None
return dict(name=self.ww_sample_full_id, row=well_row, col=well_col)
else:
return None
well_row = row_dict[self.elution_well[0]]
well_col = self.elution_well[1:]
# if positive:
# try:
# # The first character of the elution well is the row
# well_row = row_dict[self.elution_well[0]]
# # The remaining charagers are the columns
# well_col = self.elution_well[1:]
# except TypeError as e:
# logger.error(f"This sample doesn't have elution plate info.")
# return None
return dict(name=self.ww_sample_full_id,
row=well_row,
col=well_col,
positive=positive)
# else:
# return None
class BCSample(Base):
@@ -134,7 +141,24 @@ class BCSample(Base):
"name": f"{self.sample_id} - ({self.organism})",
}
def to_hitpick(self) -> dict|None:
"""
Outputs a dictionary of locations
Returns:
dict: dictionary of sample id, row and column in elution plate
"""
# dictionary to translate row letters into numbers
row_dict = dict(A=1, B=2, C=3, D=4, E=5, F=6, G=7, H=8)
# if either n1 or n2 is positive, include this sample
well_row = row_dict[self.well_number[0]]
# The remaining charagers are the columns
well_col = self.well_number[1:]
return dict(name=self.sample_id,
row=well_row,
col=well_col,
positive=False)
# class ArticSample(Base):
# """
# base of artic sample

24
src/submissions/backend/db/models/submissions.py
View File

@@ -1,6 +1,7 @@
'''
Models for the main submission types.
'''
import math
from . import Base
from sqlalchemy import Column, String, TIMESTAMP, INTEGER, ForeignKey, Table, JSON, FLOAT
from sqlalchemy.orm import relationship
@@ -246,5 +247,24 @@ class WastewaterArtic(BasicSubmission):
derivative submission type for artic wastewater
"""
samples = relationship("WWSample", back_populates="artic_rsl_plate", uselist=True)
# Can in use the pcr_info from the wastewater? Cause I can't define pcr_info here due to conflicts with that
__mapper_args__ = {"polymorphic_identity": "wastewater_artic", "polymorphic_load": "inline"}
# Can it use the pcr_info from the wastewater? Cause I can't define pcr_info here due to conflicts with that
# Not necessary because we don't get any results for this procedure.
__mapper_args__ = {"polymorphic_identity": "wastewater_artic", "polymorphic_load": "inline"}
def calculate_base_cost(self):
"""
This method overrides parent method due to multiple output plates from a single submission
"""
logger.debug(f"Hello from calculate base cost in WWArtic")
try:
cols_count_96 = ceil(int(self.sample_count) / 8)
except Exception as e:
logger.error(f"Column count error: {e}")
# Since we have multiple output plates per submission form, the constant cost will have to reflect this.
output_plate_count = math.ceil(int(self.sample_count) / 16)
logger.debug(f"Looks like we have {output_plate_count} output plates.")
const_cost = self.extraction_kit.constant_cost * output_plate_count
try:
self.run_cost = const_cost + (self.extraction_kit.mutable_cost_column * cols_count_96) + (self.extraction_kit.mutable_cost_sample * int(self.sample_count))
except Exception as e:
logger.error(f"Calculation error: {e}")

56
src/submissions/backend/excel/parser.py
View File

@@ -3,6 +3,7 @@ contains parser object for pulling values from client generated submission sheet
'''
from getpass import getuser
import math
import pprint
from typing import Tuple
import pandas as pd
from pathlib import Path
@@ -160,14 +161,19 @@ class SheetParser(object):
sample_parser = SampleParser(self.ctx, submission_info.iloc[16:112])
sample_parse = getattr(sample_parser, f"parse_{self.sub['submission_type'].lower()}_samples")
logger.debug(f"Parser result: {self.sub}")
self.sub['samples'] = sample_parse()
self.sample_result, self.sub['samples'] = sample_parse()
def parse_wastewater(self) -> None:
"""
pulls info specific to wastewater sample type
"""
def retrieve_elution_map():
full = self.xl.parse("Extraction Worksheet")
elu_map = full.iloc[9:18, 5:]
elu_map.set_index(elu_map.columns[0], inplace=True)
elu_map.columns = elu_map.iloc[0]
return elu_map
def parse_reagents(df:pd.DataFrame) -> None:
"""
Pulls reagents from the bacterial sub-dataframe
@@ -216,9 +222,9 @@ class SheetParser(object):
parse_reagents(ext_reagent_range)
parse_reagents(pcr_reagent_range)
# parse samples
sample_parser = SampleParser(self.ctx, submission_info.iloc[16:])
sample_parser = SampleParser(self.ctx, submission_info.iloc[16:], elution_map=retrieve_elution_map())
sample_parse = getattr(sample_parser, f"parse_{self.sub['submission_type'].lower()}_samples")
self.sub['samples'] = sample_parse()
self.sample_result, self.sub['samples'] = sample_parse()
self.sub['csv'] = self.xl.parse("Copy to import file", dtype=object)
@@ -272,7 +278,7 @@ class SheetParser(object):
return_list.append(dict(sample_name=re.sub(r"\s?\(.*\)", "", df.loc[ii.name, int(c)]), \
well=f"{ii.name}{c}",
artic_plate=self.sub['rsl_plate_num']))
logger.debug(f"massaged sample list for {self.sub['rsl_plate_num']}: {return_list}")
logger.debug(f"massaged sample list for {self.sub['rsl_plate_num']}: {pprint.pprint(return_list)}")
return return_list
submission_info = self.xl.parse("First Strand", dtype=object)
biomek_info = self.xl.parse("ArticV4 Biomek", dtype=object)
@@ -280,7 +286,7 @@ class SheetParser(object):
biomek_reagent_range = biomek_info.iloc[60:, 0:3].dropna(how='all')
self.sub['submitter_plate_num'] = ""
self.sub['rsl_plate_num'] = RSLNamer(self.filepath.__str__()).parsed_name
self.sub['submitted_date'] = submission_info.iloc[0][2]
self.sub['submitted_date'] = biomek_info.iloc[1][1]
self.sub['submitting_lab'] = "Enterics Wastewater Genomics"
self.sub['sample_count'] = submission_info.iloc[4][6]
self.sub['extraction_kit'] = "ArticV4.1"
@@ -290,7 +296,7 @@ class SheetParser(object):
samples = massage_samples(biomek_info.iloc[22:31, 0:])
sample_parser = SampleParser(self.ctx, pd.DataFrame.from_records(samples))
sample_parse = getattr(sample_parser, f"parse_{self.sub['submission_type'].lower()}_samples")
self.sub['samples'] = sample_parse()
self.sample_result, self.sub['samples'] = sample_parse()
@@ -299,18 +305,21 @@ class SampleParser(object):
object to pull data for samples in excel sheet and construct individual sample objects
"""
def __init__(self, ctx:dict, df:pd.DataFrame) -> None:
def __init__(self, ctx:dict, df:pd.DataFrame, elution_map:pd.DataFrame|None=None) -> None:
"""
convert sample sub-dataframe to dictionary of records
Args:
ctx (dict): setting passed down from gui
df (pd.DataFrame): input sample dataframe
elution_map (pd.DataFrame | None, optional): optional map of elution plate. Defaults to None.
"""
self.ctx = ctx
self.samples = df.to_dict("records")
self.elution_map = elution_map
def parse_bacterial_culture_samples(self) -> list[BCSample]:
def parse_bacterial_culture_samples(self) -> Tuple[str|None, list[BCSample]]:
"""
construct bacterial culture specific sample objects
@@ -334,16 +343,28 @@ class SampleParser(object):
not_a_nan = True
if not_a_nan:
new_list.append(new)
return new_list
return None, new_list
def parse_wastewater_samples(self) -> list[WWSample]:
def parse_wastewater_samples(self) -> Tuple[str|None, list[WWSample]]:
"""
construct wastewater specific sample objects
Returns:
list[WWSample]: list of sample objects
"""
def search_df_for_sample(sample_rsl:str):
logger.debug(f"Attempting to find sample {sample_rsl} in \n {self.elution_map}")
print(f"Attempting to find sample {sample_rsl} in \n {self.elution_map}")
well = self.elution_map.where(self.elution_map==sample_rsl).dropna(how='all').dropna(axis=1)
self.elution_map.at[well.index[0], well.columns[0]] = np.nan
try:
col = str(int(well.columns[0]))
except ValueError:
col = str(well.columns[0])
except TypeError as e:
logger.error(f"Problem parsing out column number for {well}:\n {e}")
return f"{well.index[0]}{col}"
new_list = []
for sample in self.samples:
new = WWSample()
@@ -368,10 +389,11 @@ class SampleParser(object):
# new.site_status = sample['Unnamed: 7']
new.notes = str(sample['Unnamed: 6']) # previously Unnamed: 8
new.well_number = sample['Unnamed: 1']
new.elution_well = search_df_for_sample(new.rsl_number)
new_list.append(new)
return new_list
return None, new_list
def parse_wastewater_artic_samples(self) -> list[WWSample]:
def parse_wastewater_artic_samples(self) -> Tuple[str|None, list[WWSample]]:
"""
The artic samples are the wastewater samples that are to be sequenced
So we will need to lookup existing ww samples and append Artic well # and plate relation
@@ -380,17 +402,20 @@ class SampleParser(object):
list[WWSample]: list of wastewater samples to be updated
"""
new_list = []
missed_samples = []
for sample in self.samples:
with self.ctx['database_session'].no_autoflush:
instance = lookup_ww_sample_by_ww_sample_num(ctx=self.ctx, sample_number=sample['sample_name'])
logger.debug(f"Checking: {sample['sample_name']}")
if instance == None:
logger.error(f"Unable to find match for: {sample['sample_name']}")
missed_samples.append(sample['sample_name'])
continue
logger.debug(f"Got instance: {instance.ww_sample_full_id}")
instance.artic_well_number = sample['well']
new_list.append(instance)
return new_list
missed_str = "\n\t".join(missed_samples)
return f"Could not find matches for the following samples:\n\t {missed_str}", new_list
@@ -472,6 +497,7 @@ class PCRParser(object):
df = self.parse_general(sheet_name="Results")
column_names = ["Well", "Well Position", "Omit","Sample","Target","Task"," Reporter","Quencher","Amp Status","Amp Score","Curve Quality","Result Quality Issues","Cq","Cq Confidence","Cq Mean","Cq SD","Auto Threshold","Threshold", "Auto Baseline", "Baseline Start", "Baseline End"]
self.samples_df = df.iloc[23:][0:]
logger.debug(f"Dataframe of PCR results:\n\t{self.samples_df}")
self.samples_df.columns = column_names
logger.debug(f"Samples columns: {self.samples_df.columns}")
well_call_df = self.xl.parse(sheet_name="Well Call").iloc[24:][0:].iloc[:,-1:]
@@ -488,7 +514,7 @@ class PCRParser(object):
sample_obj = dict(
sample = row['Sample'],
plate_rsl = self.plate_num,
elution_well = row['Well Position']
# elution_well = row['Well Position']
)
logger.debug(f"Got sample obj: {sample_obj}")
# logger.debug(f"row: {row}")

19
src/submissions/frontend/custom_widgets/sub_details.py
View File

@@ -114,6 +114,10 @@ class SubmissionsSheet(QTableView):
del self.data['reagents']
except KeyError:
pass
try:
del self.data['comments']
except KeyError:
pass
proxyModel = QSortFilterProxyModel()
proxyModel.setSourceModel(pandasModel(self.data))
self.setModel(proxyModel)
@@ -226,11 +230,12 @@ class SubmissionsSheet(QTableView):
else:
logger.error(f"We had to truncate the number of samples to 94.")
logger.debug(f"We found {len(dicto)} to hitpick")
msg = AlertPop(message=f"We found {len(dicto)} samples to hitpick", status="INFORMATION")
msg.exec()
# convert all samples to dataframe
df = make_hitpicks(dicto)
logger.debug(f"Size of the dataframe: {df.size}")
df = df[df.positive != False]
logger.debug(f"Size of the dataframe: {df.shape[0]}")
msg = AlertPop(message=f"We found {df.shape[0]} samples to hitpick", status="INFORMATION")
msg.exec()
if df.size == 0:
return
date = datetime.strftime(datetime.today(), "%Y-%m-%d")
@@ -264,6 +269,7 @@ class SubmissionDetails(QDialog):
interior.setParent(self)
# get submision from db
data = lookup_submission_by_id(ctx=ctx, id=id)
logger.debug(f"Submission details data:\n{data.to_dict()}")
self.base_dict = data.to_dict()
# don't want id
del self.base_dict['id']
@@ -308,8 +314,11 @@ class SubmissionDetails(QDialog):
platemap = make_plate_map(plate_dicto)
logger.debug(f"platemap: {platemap}")
image_io = BytesIO()
platemap.save(image_io, 'JPEG')
platemap.save("test.jpg", 'JPEG')
try:
platemap.save(image_io, 'JPEG')
except AttributeError:
logger.error(f"No plate map found for {sub.rsl_plate_num}")
# platemap.save("test.jpg", 'JPEG')
self.base_dict['platemap'] = base64.b64encode(image_io.getvalue()).decode('utf-8')
logger.debug(self.base_dict)
html = template.render(sub=self.base_dict)

18
src/submissions/frontend/main_window_functions.py
View File

@@ -26,7 +26,7 @@ from backend.db.functions import (
lookup_all_orgs, lookup_kittype_by_use, lookup_kittype_by_name,
construct_submission_info, lookup_reagent, store_submission, lookup_submissions_by_date_range,
create_kit_from_yaml, create_org_from_yaml, get_control_subtypes, get_all_controls_by_type,
lookup_all_submissions_by_type, get_all_controls, lookup_submission_by_rsl_num, update_ww_sample
lookup_all_submissions_by_type, get_all_controls, lookup_submission_by_rsl_num, update_ww_sample, hitpick_plate
)
from backend.excel.parser import SheetParser, PCRParser
from backend.excel.reports import make_report_html, make_report_xlsx, convert_data_list_to_df
@@ -35,6 +35,7 @@ from .custom_widgets.pop_ups import AlertPop, QuestionAsker
from .custom_widgets import ReportDatePicker, ReagentTypeForm
from .custom_widgets.misc import ImportReagent
from .visualizations.control_charts import create_charts, construct_html
from .visualizations import make_plate_map
logger = logging.getLogger(f"submissions.{__name__}")
@@ -60,6 +61,7 @@ def import_submission_function(obj:QMainWindow) -> Tuple[QMainWindow, dict|None]
if prsr.sub['rsl_plate_num'] == None:
prsr.sub['rsl_plate_num'] = RSLNamer(fname.__str__()).parsed_name
logger.debug(f"prsr.sub = {prsr.sub}")
obj.current_submission_type = prsr.sub['submission_type']
# destroy any widgets from previous imports
for item in obj.table_widget.formlayout.parentWidget().findChildren(QWidget):
item.setParent(None)
@@ -111,7 +113,7 @@ def import_submission_function(obj:QMainWindow) -> Tuple[QMainWindow, dict|None]
uses.insert(0, uses.pop(uses.index(prsr.sub[item])))
obj.ext_kit = prsr.sub[item]
else:
logger.error(f"Couldn't find prsr.sub[extraction_kit]")
logger.error(f"Couldn't find {prsr.sub['extraction_kit']}")
obj.ext_kit = uses[0]
add_widget.addItems(uses)
case 'submitted_date':
@@ -156,6 +158,9 @@ def import_submission_function(obj:QMainWindow) -> Tuple[QMainWindow, dict|None]
if hasattr(obj, 'ext_kit'):
obj.kit_integrity_completion()
logger.debug(f"Imported reagents: {obj.reagents}")
if prsr.sample_result != None:
msg = AlertPop(message=prsr.sample_result, status="WARNING")
msg.exec()
return obj, result
def kit_reload_function(obj:QMainWindow) -> QMainWindow:
@@ -263,6 +268,7 @@ def submit_new_sample_function(obj:QMainWindow) -> QMainWindow:
# reset form
for item in obj.table_widget.formlayout.parentWidget().findChildren(QWidget):
item.setParent(None)
logger.debug(f"All attributes of obj: {pprint.pprint(obj.__dict__)}")
if hasattr(obj, 'csv'):
dlg = QuestionAsker("Export CSV?", "Would you like to export the csv file?")
if dlg.exec():
@@ -271,6 +277,14 @@ def submit_new_sample_function(obj:QMainWindow) -> QMainWindow:
obj.csv.to_csv(fname.__str__(), index=False)
except PermissionError:
logger.debug(f"Could not get permissions to {fname}. Possibly the request was cancelled.")
try:
delattr(obj, "csv")
except AttributeError:
pass
# if obj.current_submission_type == "Bacterial_Culture":
# hitpick = hitpick_plate(base_submission)
# image = make_plate_map(hitpick)
# image.show()
return obj, result
def generate_report_function(obj:QMainWindow) -> QMainWindow:

16
src/submissions/frontend/visualizations/plate_map.py
View File

@@ -12,7 +12,7 @@ def make_plate_map(sample_list:list) -> Image:
Makes a pillow image of a plate from hitpicks
Args:
sample_list (list): list of positive sample dictionaries from the hitpicks
sample_list (list): list of sample dictionaries from the hitpicks
Returns:
Image: Image of the 96 well plate with positive samples in red.
@@ -26,12 +26,17 @@ def make_plate_map(sample_list:list) -> Image:
except TypeError as e:
logger.error(f"No samples for this plate. Nothing to do.")
return None
# Make a 8 row, 12 column, 3 color ints array, filled with white by default
# Make an 8 row, 12 column, 3 color ints array, filled with white by default
grid = np.full((8,12,3),255, dtype=np.uint8)
# Go through samples and change its row/column to red
# Go through samples and change its row/column to red if positive, else blue
for sample in sample_list:
grid[int(sample['row'])-1][int(sample['column'])-1] = [255,0,0]
# Create image from the grid
logger.debug(f"sample keys: {list(sample.keys())}")
if sample['positive']:
colour = [255,0,0]
else:
colour = [0,0,255]
grid[int(sample['row'])-1][int(sample['column'])-1] = colour
# Create pixel image from the grid and enlarge
img = Image.fromarray(grid).resize((1200, 800), resample=Image.NEAREST)
# create a drawer over the image
draw = ImageDraw.Draw(img)
@@ -58,7 +63,6 @@ def make_plate_map(sample_list:list) -> Image:
new_img.paste(img, box)
# create drawer over the new image
draw = ImageDraw.Draw(new_img)
# font = ImageFont.truetype("sans-serif.ttf", 16)
if check_if_app():
font_path = Path(sys._MEIPASS).joinpath("files", "resources")
else:

3
src/submissions/tools/__init__.py
View File

@@ -260,4 +260,5 @@ def massage_common_reagents(reagent_name:str):
reagent_name = "molecular_grade_water"
reagent_name = reagent_name.replace("µ", "u")
return reagent_name