diff --git a/CHANGELOG.md b/CHANGELOG.md index 944bfc3..19cf46d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# 202504.01 + +- Added in checkbox to use all samples in Concentrations tab (very slow). + # 202503.05 - Created Sample verification before import. diff --git a/src/submissions/backend/db/models/__init__.py b/src/submissions/backend/db/models/__init__.py index a7ad4ec..78749bc 100644 --- a/src/submissions/backend/db/models/__init__.py +++ b/src/submissions/backend/db/models/__init__.py @@ -60,7 +60,7 @@ class BaseClass(Base): try: return f"<{self.__class__.__name__}({self.name})>" except AttributeError: - return f"<{self.__class__.__name__}({self.__name__})>" + return f"<{self.__class__.__name__}(Unknown)>" # @classproperty # def skip_on_edit(cls): @@ -411,13 +411,13 @@ class BaseClass(Base): except AttributeError: return super().__setattr__(key, value) if isinstance(field_type, InstrumentedAttribute): - logger.debug(f"{key} is an InstrumentedAttribute.") + # logger.debug(f"{key} is an InstrumentedAttribute.") match field_type.property: case ColumnProperty(): # logger.debug(f"Setting ColumnProperty to {value}") return super().__setattr__(key, value) case _RelationshipDeclared(): - logger.debug(f"{self.__class__.__name__} Setting _RelationshipDeclared for {key} to {value}") + # logger.debug(f"{self.__class__.__name__} Setting _RelationshipDeclared for {key} to {value}") if field_type.property.uselist: logger.debug(f"Setting with uselist") existing = self.__getattribute__(key) diff --git a/src/submissions/backend/db/models/submissions.py b/src/submissions/backend/db/models/submissions.py index aa6f91c..af8fe65 100644 --- a/src/submissions/backend/db/models/submissions.py +++ b/src/submissions/backend/db/models/submissions.py @@ -2,6 +2,8 @@ Models for the main submission and sample types. """ from __future__ import annotations + +import pickle from copy import deepcopy from getpass import getuser import logging, uuid, tempfile, re, base64, numpy as np, pandas as pd, types, sys @@ -588,7 +590,39 @@ class BasicSubmission(BaseClass, LogMixin): except AttributeError as e: logger.error(f"Could not set {self} attribute {key} to {value} due to \n{e}") - def update_subsampassoc(self, sample: BasicSample, input_dict: dict) -> SubmissionSampleAssociation: + # def update_subsampassoc(self, sample: BasicSample, input_dict: dict) -> SubmissionSampleAssociation: + # """ + # Update a joined submission sample association. + # + # Args: + # sample (BasicSample): Associated sample. + # input_dict (dict): values to be updated + # + # Returns: + # SubmissionSampleAssociation: Updated association + # """ + # try: + # logger.debug(f"Searching for sample {sample} at column {input_dict['column']} and row {input_dict['row']}") + # assoc = next((item for item in self.submission_sample_associations + # if item.sample == sample and + # item.row == input_dict['row'] and + # item.column == input_dict['column'])) + # logger.debug(f"Found assoc {pformat(assoc.__dict__)}") + # except StopIteration: + # report = Report() + # report.add_result( + # Result(msg=f"Couldn't find submission sample association for {sample.submitter_id}", status="Warning")) + # return report + # for k, v in input_dict.items(): + # try: + # # logger.debug(f"Setting assoc {assoc} with key {k} to value {v}") + # setattr(assoc, k, v) + # # NOTE: for some reason I don't think assoc.__setattr__(k, v) works here. + # except AttributeError: + # logger.error(f"Can't set {k} to {v}") + # return assoc + + def update_subsampassoc(self, assoc: SubmissionSampleAssociation, input_dict: dict) -> SubmissionSampleAssociation: """ Update a joined submission sample association. @@ -599,19 +633,26 @@ class BasicSubmission(BaseClass, LogMixin): Returns: SubmissionSampleAssociation: Updated association """ - try: - assoc = next(item for item in self.submission_sample_associations if item.sample == sample) - except StopIteration: - report = Report() - report.add_result( - Result(msg=f"Couldn't find submission sample association for {sample.submitter_id}", status="Warning")) - return report + # try: + # logger.debug(f"Searching for sample {sample} at column {input_dict['column']} and row {input_dict['row']}") + # assoc = next((item for item in self.submission_sample_associations + # if item.sample == sample and + # item.row == input_dict['row'] and + # item.column == input_dict['column'])) + # logger.debug(f"Found assoc {pformat(assoc.__dict__)}") + # except StopIteration: + # report = Report() + # report.add_result( + # Result(msg=f"Couldn't find submission sample association for {sample.submitter_id}", status="Warning")) + # return report for k, v in input_dict.items(): try: + # logger.debug(f"Setting assoc {assoc} with key {k} to value {v}") setattr(assoc, k, v) # NOTE: for some reason I don't think assoc.__setattr__(k, v) works here. except AttributeError: - logger.error(f"Can't set {k} to {v}") + # logger.error(f"Can't set {k} to {v}") + pass return assoc def update_reagentassoc(self, reagent: Reagent, role: str): @@ -949,10 +990,12 @@ class BasicSubmission(BaseClass, LogMixin): pcr_sample_map = cls.get_submission_type().sample_map['pcr_samples'] main_sheet = xl[pcr_sample_map['main_sheet']] fields = {k: v for k, v in pcr_sample_map.items() if k not in ['main_sheet', 'start_row']} + logger.debug(f"Fields: {fields}") for row in main_sheet.iter_rows(min_row=pcr_sample_map['start_row']): idx = row[0].row sample = {} for k, v in fields.items(): + # logger.debug(f"Checking key: {k} with value {v}") sheet = xl[v['sheet']] sample[k] = sheet.cell(row=idx, column=v['column']).value yield sample @@ -1535,12 +1578,15 @@ class BacterialCulture(BasicSubmission): column=lookup_table['sample_columns']['concentration']).value yield sample - def get_provisional_controls(self): - if self.controls: - provs = (control.sample for control in self.controls) + def get_provisional_controls(self, controls_only: bool = True): + if controls_only: + if self.controls: + provs = (control.sample for control in self.controls) + else: + regex = re.compile(r"^(ATCC)|(MCS)|(EN)") + provs = (sample for sample in self.samples if bool(regex.match(sample.submitter_id))) else: - regex = re.compile(r"^(ATCC)|(MCS)|(EN)") - provs = (sample for sample in self.samples if bool(regex.match(sample.submitter_id))) + provs = self.samples for prov in provs: prov.submission = self.rsl_plate_num prov.submitted_date = self.submitted_date @@ -1668,7 +1714,7 @@ class Wastewater(BasicSubmission): # NOTE: Also, you can't change the size of a list while iterating it, so don't even think about it. output = [] for sample in samples: - # logger.debug(sample) + logger.debug(sample) # NOTE: remove '-{target}' from controls sample['sample'] = re.sub('-N\\d*$', '', sample['sample']) # NOTE: if sample is already in output skip @@ -1679,7 +1725,7 @@ class Wastewater(BasicSubmission): # logger.debug(f"Sample ct: {sample['ct']}") sample[f"ct_{sample['target'].lower()}"] = sample['ct'] if isinstance(sample['ct'], float) else 0.0 # NOTE: Set assessment - logger.debug(f"Sample assessemnt: {sample['assessment']}") + # logger.debug(f"Sample assessemnt: {sample['assessment']}") # sample[f"{sample['target'].lower()}_status"] = sample['assessment'] # NOTE: Get sample having other target other_targets = [s for s in samples if re.sub('-N\\d*$', '', s['sample']) == sample['sample']] @@ -1694,6 +1740,11 @@ class Wastewater(BasicSubmission): del sample['assessment'] except KeyError: pass + # logger.debug(sample) + row = int(row_keys[sample['well'][:1]]) + column = int(sample['well'][1:]) + sample['row'] = row + sample['column'] = column output.append(sample) # NOTE: And then convert back to list to keep fidelity with parent method. for sample in output: @@ -1779,18 +1830,43 @@ class Wastewater(BasicSubmission): parser = PCRParser(filepath=fname, submission=self) self.set_attribute("pcr_info", parser.pcr_info) # NOTE: These are generators here, need to expand. - pcr_samples = [sample for sample in parser.samples] + pcr_samples = sorted([sample for sample in parser.samples], key=itemgetter('column')) + logger.debug(f"Samples from parser: {pformat(pcr_samples)}") + # NOTE: Samples from parser check out. pcr_controls = [control for control in parser.controls] self.save(original=False) - for sample in self.samples: + for assoc in self.submission_sample_associations: + logger.debug(f"Checking pcr_samples for {assoc.sample.rsl_number}, {assoc.sample.ww_full_sample_id} at " + f"column {assoc.column} and row {assoc.row}") + # NOTE: Associations of interest do exist in the submission, are not being found below + # Okay, I've found the problem, at last, the problem is that only one RSL number is saved for each sample, + # Even though each instance of say "25-YUL13-PU3-0320" has multiple RSL numbers in the excel sheet. + # so, yeah, the submitters need to make sure that there are unique values for each one. try: - # NOTE: Fix for ENs which have no rsl_number... - sample_dict = next(item for item in pcr_samples if item['sample'] == sample.rsl_number) + sample_dict = next(item for item in pcr_samples if item['sample'] == assoc.sample.rsl_number + and item['row'] == assoc.row and item['column'] == assoc.column) + logger.debug( + f"Found sample {sample_dict} at index {pcr_samples.index(sample_dict)}: {pcr_samples[pcr_samples.index(sample_dict)]}") except StopIteration: + logger.error(f"Couldn't find {assoc} in the Parser samples") continue - assoc = self.update_subsampassoc(sample=sample, input_dict=sample_dict) + logger.debug(f"Length of pcr_samples: {len(pcr_samples)}") + assoc = self.update_subsampassoc(assoc=assoc, input_dict=sample_dict) result = assoc.save() - report.add_result(result) + if result: + report.add_result(result) + # for sample in self.samples: + # logger.debug(f"Checking pcr_samples for {sample.rsl_number}, {sample.ww_full_sample_id}") + # try: + # # NOTE: Fix for ENs which have no rsl_number... + # sample_dict = next(item for item in pcr_samples if item['sample'] == sample.rsl_number) + # logger.debug(f"Found sample {sample_dict} at index {pcr_samples.index(sample_dict)}: {pcr_samples[pcr_samples.index(sample_dict)]}") + # except StopIteration: + # logger.error(f"Couldn't find {sample} in the Parser samples") + # continue + # assoc = self.update_subsampassoc(sample=sample, input_dict=sample_dict) + # result = assoc.save() + # report.add_result(result) controltype = ControlType.query(name="PCR Control") submitted_date = datetime.strptime(" ".join(parser.pcr_info['run_start_date/time'].split(" ")[:-1]), "%Y-%m-%d %I:%M:%S %p") @@ -1804,7 +1880,7 @@ class Wastewater(BasicSubmission): new_control.save() return report - def update_subsampassoc(self, sample: BasicSample, input_dict: dict) -> SubmissionSampleAssociation: + def update_subsampassoc(self, assoc: SubmissionSampleAssociation, input_dict: dict) -> SubmissionSampleAssociation: """ Updates a joined submission sample association by assigning ct values to n1 or n2 based on alphabetical sorting. @@ -1815,13 +1891,22 @@ class Wastewater(BasicSubmission): Returns: SubmissionSampleAssociation: Updated association """ - assoc = super().update_subsampassoc(sample=sample, input_dict=input_dict) - targets = {k: input_dict[k] for k in sorted(input_dict.keys()) if k.startswith("ct_")} - assert 0 < len(targets) <= 2 - for i, v in enumerate(targets.values(), start=1): - update_key = f"ct_n{i}" - if getattr(assoc, update_key) is None: - setattr(assoc, update_key, v) + # logger.debug(f"Input dict: {pformat(input_dict)}") + # + assoc = super().update_subsampassoc(assoc=assoc, input_dict=input_dict) + # targets = {k: input_dict[k] for k in sorted(input_dict.keys()) if k.startswith("ct_")} + # assert 0 < len(targets) <= 2 + # for k, v in targets.items(): + # # logger.debug(f"Setting sample {sample} with key {k} to value {v}") + # # update_key = f"ct_n{i}" + # current_value = getattr(assoc, k) + # logger.debug(f"Current value came back as: {current_value}") + # if current_value is None: + # setattr(assoc, k, v) + # else: + # logger.debug(f"Have a value already, {current_value}... skipping.") + if assoc.column == 3: + logger.debug(f"Final association for association {assoc}:\n{pformat(assoc.__dict__)}") return assoc @@ -2105,14 +2190,20 @@ class WastewaterArtic(BasicSubmission): Returns: str: output name """ + logger.debug(f"PBS adapter on {input_str}") # NOTE: Remove letters. processed = input_str.replace("RSL", "") + # logger.debug(processed) # NOTE: Remove brackets at end processed = re.sub(r"\(.*\)$", "", processed).strip() + logger.debug(processed) + processed = re.sub(r"-RPT", "", processed, flags=re.IGNORECASE) # NOTE: Remove any non-R letters at end. processed = re.sub(r"[A-QS-Z]+\d*", "", processed) + logger.debug(processed) # NOTE: Remove trailing '-' if any processed = processed.strip("-") + logger.debug(processed) try: plate_num = re.search(r"\-\d{1}R?\d?$", processed).group() processed = rreplace(processed, plate_num, "") @@ -2126,16 +2217,24 @@ class WastewaterArtic(BasicSubmission): repeat_num = None if repeat_num is None and "R" in plate_num: repeat_num = "1" - plate_num = re.sub(r"R", rf"R{repeat_num}", plate_num) + try: + plate_num = re.sub(r"R", rf"R{repeat_num}", plate_num) + except AttributeError: + logger.error(f"Problem re-evaluating plate number for {processed}") + logger.debug(processed) # NOTE: Remove any redundant -digits processed = re.sub(r"-\d$", "", processed) + logger.debug(processed) day = re.search(r"\d{2}$", processed).group() processed = rreplace(processed, day, "") + logger.debug(processed) month = re.search(r"\d{2}$", processed).group() processed = rreplace(processed, month, "") processed = processed.replace("--", "") + logger.debug(processed) year = re.search(r'^(?:\d{2})?\d{2}', processed).group() year = f"20{year}" + logger.debug(processed) final_en_name = f"PBS{year}{month}{day}-{plate_num}" return final_en_name diff --git a/src/submissions/backend/excel/reports.py b/src/submissions/backend/excel/reports.py index 43170b7..dca98de 100644 --- a/src/submissions/backend/excel/reports.py +++ b/src/submissions/backend/excel/reports.py @@ -196,20 +196,22 @@ class TurnaroundMaker(ReportArchetype): class ConcentrationMaker(ReportArchetype): - def __init__(self, start_date: date, end_date: date, submission_type: str = "Bacterial Culture"): + def __init__(self, start_date: date, end_date: date, submission_type: str = "Bacterial Culture", + controls_only: bool = True): self.start_date = start_date self.end_date = end_date # NOTE: Set page size to zero to override limiting query size. self.subs = BasicSubmission.query(start_date=start_date, end_date=end_date, submission_type_name=submission_type, page_size=0) # self.known_controls = list(itertools.chain.from_iterable([sub.controls for sub in self.subs])) - self.controls = list(itertools.chain.from_iterable([sub.get_provisional_controls() for sub in self.subs])) - self.records = [self.build_record(control) for control in self.controls] + self.samples = list(itertools.chain.from_iterable([sub.get_provisional_controls(controls_only=controls_only) for sub in self.subs])) + self.records = [self.build_record(sample) for sample in self.samples] self.df = DataFrame.from_records(self.records) self.sheet_name = "Concentration" @classmethod def build_record(cls, control) -> dict: + positive = not control.submitter_id.lower().startswith("en") try: concentration = float(control.concentration) diff --git a/src/submissions/backend/validators/__init__.py b/src/submissions/backend/validators/__init__.py index 7d310b0..6efd21d 100644 --- a/src/submissions/backend/validators/__init__.py +++ b/src/submissions/backend/validators/__init__.py @@ -1,6 +1,6 @@ -''' +""" Contains all validators -''' +""" import logging, re import sys from pathlib import Path diff --git a/src/submissions/backend/validators/omni_gui_objects.py b/src/submissions/backend/validators/omni_gui_objects.py index 59096e7..c681c40 100644 --- a/src/submissions/backend/validators/omni_gui_objects.py +++ b/src/submissions/backend/validators/omni_gui_objects.py @@ -366,7 +366,10 @@ class OmniKitTypeReagentRoleAssociation(BaseOmni): logger.debug(f"KitTypeReagentRoleAssociation coming out of query_or_create: {instance.__dict__}\nnew: {new}") if new: logger.warning(f"This is a new instance: {instance.__dict__}") - reagent_role = self.reagent_role.to_sql() + try: + reagent_role = self.reagent_role.to_sql() + except AttributeError: + reagent_role = ReagentRole.query(name=self.reagent_role) instance.reagent_role = reagent_role logger.debug(f"KTRRAssoc uses: {self.uses}") instance.uses = self.uses @@ -509,15 +512,24 @@ class OmniProcess(BaseOmni): def to_sql(self): instance, new = self.class_object.query_or_create(name=self.name) for st in self.submission_types: - new_assoc = st.to_sql() + try: + new_assoc = st.to_sql() + except AttributeError: + new_assoc = SubmissionType.query(name=st) if new_assoc not in instance.submission_types: instance.submission_types.append(new_assoc) for er in self.equipment_roles: - new_assoc = er.to_sql() + try: + new_assoc = er.to_sql() + except AttributeError: + new_assoc = EquipmentRole.query(name=er) if new_assoc not in instance.equipment_roles: instance.equipment_roles.append(new_assoc) for tr in self.tip_roles: - new_assoc = tr.to_sql() + try: + new_assoc = tr.to_sql() + except AttributeError: + new_assoc = TipRole.query(name=tr) if new_assoc not in instance.tip_roles: instance.tip_roles.append(new_assoc) return instance diff --git a/src/submissions/frontend/widgets/concentrations.py b/src/submissions/frontend/widgets/concentrations.py index f7facb0..95eaa84 100644 --- a/src/submissions/frontend/widgets/concentrations.py +++ b/src/submissions/frontend/widgets/concentrations.py @@ -1,7 +1,7 @@ """ Pane showing BC control concentrations summary. """ -from PyQt6.QtWidgets import QWidget, QPushButton +from PyQt6.QtWidgets import QWidget, QPushButton, QCheckBox, QLabel from .info_tab import InfoPane from backend.excel.reports import ConcentrationMaker from frontend.visualizations.concentrations_chart import ConcentrationsChart @@ -20,6 +20,12 @@ class Concentrations(InfoPane): self.export_button = QPushButton("Save Data", parent=self) self.export_button.pressed.connect(self.save_excel) self.layout.addWidget(self.export_button, 0, 3, 1, 1) + check_label = QLabel("Controls Only") + self.all_box = QCheckBox() + self.all_box.setChecked(True) + self.all_box.checkStateChanged.connect(self.update_data) + self.layout.addWidget(check_label, 1, 0, 1, 1) + self.layout.addWidget(self.all_box, 1, 1, 1, 1) self.fig = None self.report_object = None self.update_data() @@ -33,7 +39,8 @@ class Concentrations(InfoPane): """ super().update_data() months = self.diff_month(self.start_date, self.end_date) - chart_settings = dict(start_date=self.start_date, end_date=self.end_date) + # logger.debug(f"Box checked: {self.all_box.isChecked()}") + chart_settings = dict(start_date=self.start_date, end_date=self.end_date, controls_only=self.all_box.isChecked()) self.report_obj = ConcentrationMaker(**chart_settings) self.fig = ConcentrationsChart(df=self.report_obj.df, settings=chart_settings, modes=[], months=months) self.webview.setHtml(self.fig.html) diff --git a/src/submissions/frontend/widgets/sample_checker.py b/src/submissions/frontend/widgets/sample_checker.py index c369ee3..102daff 100644 --- a/src/submissions/frontend/widgets/sample_checker.py +++ b/src/submissions/frontend/widgets/sample_checker.py @@ -1,7 +1,7 @@ import logging from pathlib import Path -from typing import List +from typing import List, Generator from PyQt6.QtCore import Qt, pyqtSlot from PyQt6.QtWebChannel import QWebChannel @@ -37,7 +37,7 @@ class SampleChecker(QDialog): template_path = Path(template.environment.loader.__getattribute__("searchpath")[0]) with open(template_path.joinpath("css", "styles.css"), "r") as f: css = f.read() - html = template.render(samples=pyd.sample_list, css=css) + html = template.render(samples=self.formatted_list, css=css) self.webview.setHtml(html) QBtn = QDialogButtonBox.StandardButton.Ok | QDialogButtonBox.StandardButton.Cancel self.buttonBox = QDialogButtonBox(QBtn) @@ -59,8 +59,19 @@ class SampleChecker(QDialog): item = next((sample for sample in self.pyd.samples if int(submission_rank) in sample.submission_rank)) except StopIteration: logger.error(f"Unable to find sample {submission_rank}") + return item.__setattr__(key, value) + @property + def formatted_list(self) -> List[dict]: + output = [] + for sample in self.pyd.sample_list: + if sample['submitter_id'] in [item['submitter_id'] for item in output]: + sample['color'] = "red" + else: + sample['color'] = "black" + output.append(sample) + return output diff --git a/src/submissions/frontend/widgets/submission_widget.py b/src/submissions/frontend/widgets/submission_widget.py index d6f8035..4a3d500 100644 --- a/src/submissions/frontend/widgets/submission_widget.py +++ b/src/submissions/frontend/widgets/submission_widget.py @@ -140,7 +140,7 @@ class SubmissionFormContainer(QWidget): # logger.debug(f"Samples: {pformat(self.pyd.samples)}") checker = SampleChecker(self, "Sample Checker", self.pyd) if checker.exec(): - logger.debug(pformat(self.pyd.samples)) + # logger.debug(pformat(self.pyd.samples)) self.form = self.pyd.to_form(parent=self) self.layout().addWidget(self.form) else: diff --git a/src/submissions/templates/sample_checker.html b/src/submissions/templates/sample_checker.html index 0adac07..12e887e 100644 --- a/src/submissions/templates/sample_checker.html +++ b/src/submissions/templates/sample_checker.html @@ -15,7 +15,7 @@   Submitter ID              Row           Column
{% for sample in samples %} {{ '%02d' % sample['submission_rank'] }} - + >