From cc53b894b289a46cacaceb6e1168e782f48f9911 Mon Sep 17 00:00:00 2001 From: lwark Date: Wed, 4 Dec 2024 12:11:10 -0600 Subject: [PATCH] Addition of turnaround time tracking. --- CHANGELOG.md | 4 ++ src/submissions/backend/db/__init__.py | 6 +-- src/submissions/backend/db/models/controls.py | 17 ++++--- src/submissions/backend/db/models/kits.py | 17 +++---- .../backend/db/models/submissions.py | 23 +++++---- src/submissions/backend/excel/parser.py | 14 +++-- src/submissions/backend/excel/reports.py | 34 +++++++++++++ .../frontend/visualizations/__init__.py | 11 +++- .../frontend/visualizations/irida_charts.py | 16 +++--- .../frontend/visualizations/pcr_charts.py | 8 ++- src/submissions/frontend/widgets/__init__.py | 2 + src/submissions/frontend/widgets/app.py | 7 +++ .../frontend/widgets/submission_details.py | 4 +- src/submissions/frontend/widgets/summary.py | 51 +++++++------------ 14 files changed, 136 insertions(+), 78 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7356506..2fc8140 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 202412.02 + +- Addition of turnaround time tracking + ## 202411.05 - Can now calculate turnaround time including holidays. diff --git a/src/submissions/backend/db/__init__.py b/src/submissions/backend/db/__init__.py index 7f3ae7c..582ee86 100644 --- a/src/submissions/backend/db/__init__.py +++ b/src/submissions/backend/db/__init__.py @@ -69,6 +69,6 @@ def update_log(mapper, connection, target): else: logger.info(f"No changes detected, not updating logs.") -if ctx.database_schema == "sqlite": - event.listen(LogMixin, 'after_update', update_log, propagate=True) - event.listen(LogMixin, 'after_insert', update_log, propagate=True) +# if ctx.database_schema == "sqlite": +event.listen(LogMixin, 'after_update', update_log, propagate=True) +event.listen(LogMixin, 'after_insert', update_log, propagate=True) diff --git a/src/submissions/backend/db/models/controls.py b/src/submissions/backend/db/models/controls.py index 547269f..8fdb753 100644 --- a/src/submissions/backend/db/models/controls.py +++ b/src/submissions/backend/db/models/controls.py @@ -279,7 +279,7 @@ class Control(BaseClass): @classmethod def make_parent_buttons(cls, parent: QWidget) -> None: """ - Super that will make buttons in a CustomFigure. Made to be overrided. + Super that will make buttons in a CustomFigure. Made to be overridden. Args: parent (QWidget): chart holding widget to add buttons to. @@ -299,6 +299,10 @@ class Control(BaseClass): class PCRControl(Control): + """ + Class made to hold info from Design & Analysis software. + """ + id = Column(INTEGER, ForeignKey('_control.id'), primary_key=True) subtype = Column(String(16)) #: PC or NC target = Column(String(16)) #: N1, N2, etc. @@ -348,7 +352,7 @@ class PCRControl(Control): df = df[df.ct > 0.0] except AttributeError: df = df - fig = PCRFigure(df=df, modes=[]) + fig = PCRFigure(df=df, modes=[], settings=chart_settings) return report, fig def to_pydantic(self): @@ -433,12 +437,12 @@ class IridaControl(Control): def convert_by_mode(self, control_sub_type: str, mode: Literal['kraken', 'matches', 'contains'], consolidate: bool = False) -> Generator[dict, None, None]: """ - split this instance into analysis types for controls graphs + split this instance into analysis types ('kraken', 'matches', 'contains') for controls graphs Args: consolidate (bool): whether to merge all off-target genera. Defaults to False control_sub_type (str): control subtype, 'MCS-NOS', etc. - mode (str): analysis type, 'contains', etc. + mode (Literal['kraken', 'matches', 'contains']): analysis type, 'contains', etc. Returns: List[dict]: list of records @@ -562,7 +566,7 @@ class IridaControl(Control): df, modes = cls.prep_df(ctx=ctx, df=df) # logger.debug(f"prepped df: \n {df}") fig = IridaFigure(df=df, ytitle=title, modes=modes, parent=parent, - months=chart_settings['months']) + settings=chart_settings) return report, fig @classmethod @@ -571,9 +575,8 @@ class IridaControl(Control): Convert list of control records to dataframe Args: - ctx (dict): settings passed from gui input_df (list[dict]): list of dictionaries containing records - sub_type (str | None, optional): sub_type of submission type. Defaults to None. + sub_mode (str | None, optional): sub_type of submission type. Defaults to None. Returns: DataFrame: dataframe of controls diff --git a/src/submissions/backend/db/models/kits.py b/src/submissions/backend/db/models/kits.py index e081903..5539b39 100644 --- a/src/submissions/backend/db/models/kits.py +++ b/src/submissions/backend/db/models/kits.py @@ -168,7 +168,6 @@ class KitType(BaseClass): else: return (item.reagent_role for item in relevant_associations) - # TODO: Move to BasicSubmission? def construct_xl_map_for_use(self, submission_type: str | SubmissionType) -> Generator[(str, str), None, None]: """ Creates map of locations in Excel workbook for a SubmissionType @@ -274,8 +273,7 @@ class KitType(BaseClass): for kk, vv in assoc.to_export_dict().items(): v[kk] = vv base_dict['reagent roles'].append(v) - # for k, v in submission_type.construct_equipment_map(): - for k, v in submission_type.contstruct_field_map("equipment"): + for k, v in submission_type.construct_field_map("equipment"): try: assoc = next(item for item in submission_type.submissiontype_equipmentrole_associations if item.equipment_role.name == k) @@ -428,7 +426,7 @@ class Reagent(BaseClass, LogMixin): submission=sub)) #: Association proxy to SubmissionSampleAssociation.samples def __repr__(self): - if self.name is not None: + if self.name: return f"" else: return f"" @@ -447,11 +445,12 @@ class Reagent(BaseClass, LogMixin): if extraction_kit is not None: # NOTE: Get the intersection of this reagent's ReagentType and all ReagentTypes in KitType - try: - reagent_role = list(set(self.role).intersection(extraction_kit.reagent_roles))[0] - # NOTE: Most will be able to fall back to first ReagentType in itself because most will only have 1. - except: - reagent_role = self.role[0] + reagent_role = next((item for item in set(self.role).intersection(extraction_kit.reagent_roles)), self.role[0]) + # try: + # reagent_role = list(set(self.role).intersection(extraction_kit.reagent_roles))[0] + # # NOTE: Most will be able to fall back to first ReagentType in itself because most will only have 1. + # except: + # reagent_role = self.role[0] else: try: reagent_role = self.role[0] diff --git a/src/submissions/backend/db/models/submissions.py b/src/submissions/backend/db/models/submissions.py index c676cb8..13f2904 100644 --- a/src/submissions/backend/db/models/submissions.py +++ b/src/submissions/backend/db/models/submissions.py @@ -24,7 +24,7 @@ from sqlite3 import OperationalError as SQLOperationalError, IntegrityError as S from openpyxl import Workbook from openpyxl.drawing.image import Image as OpenpyxlImage from tools import row_map, setup_lookup, jinja_template_loading, rreplace, row_keys, check_key_or_attr, Result, Report, \ - report_result, create_holidays_for_year + report_result, create_holidays_for_year, ctx from datetime import datetime, date, timedelta from typing import List, Any, Tuple, Literal, Generator from dateutil.parser import parse @@ -127,7 +127,7 @@ class BasicSubmission(BaseClass, LogMixin): def __repr__(self) -> str: submission_type = self.submission_type or "Basic" - return f"<{submission_type}Submission({self.rsl_plate_num})>" + return f"" @classmethod def jsons(cls) -> List[str]: @@ -1380,17 +1380,22 @@ class BasicSubmission(BaseClass, LogMixin): writer = pyd.to_writer() writer.xl.save(filename=fname.with_suffix(".xlsx")) - def get_turnaround_time(self): - completed = self.completed_date or datetime.now() - return self.calculate_turnaround(start_date=self.submitted_date.date(), end_date=completed.date()) + def get_turnaround_time(self) -> Tuple[int|None, bool|None]: + try: + completed = self.completed_date.date() + except AttributeError: + completed = None + return self.calculate_turnaround(start_date=self.submitted_date.date(), end_date=completed) @classmethod - def calculate_turnaround(cls, start_date:date|None=None, end_date:date|None=None) -> int|None: + def calculate_turnaround(cls, start_date:date|None=None, end_date:date|None=None) -> Tuple[int|None, bool|None]: + if not end_date: + return None, None try: - delta = np.busday_count(start_date, end_date, holidays=create_holidays_for_year(start_date.year)) + delta = np.busday_count(start_date, end_date, holidays=create_holidays_for_year(start_date.year)) + 1 except ValueError: - return None - return delta + 1 + return None, None + return delta, delta <= ctx.TaT_threshold # Below are the custom submission types diff --git a/src/submissions/backend/excel/parser.py b/src/submissions/backend/excel/parser.py index 338de12..1971400 100644 --- a/src/submissions/backend/excel/parser.py +++ b/src/submissions/backend/excel/parser.py @@ -293,20 +293,24 @@ class ReagentParser(object): report = Report() if isinstance(submission_type, dict): submission_type = submission_type['value'] + if isinstance(submission_type, str): + submission_type = SubmissionType.query(name=submission_type) reagent_map = {k: v for k, v in self.kit_object.construct_xl_map_for_use(submission_type)} try: del reagent_map['info'] except KeyError: pass - # logger.debug(f"Reagent map: {pformat(reagent_map)}") + logger.debug(f"Reagent map: {pformat(reagent_map)}") # NOTE: If reagent map is empty, maybe the wrong kit was given, check if there's only one kit for that submission type and use it if so. - if not reagent_map.keys(): + if not reagent_map: temp_kit_object = self.submission_type_obj.get_default_kit() + logger.debug(f"Temp kit: {temp_kit_object}") if temp_kit_object: self.kit_object = temp_kit_object - reagent_map = {k: v for k, v in self.kit_object.construct_xl_map_for_use(submission_type)} - logger.warning(f"Attempting to salvage {self.kit_object} with default kit map: {reagent_map}") - if not reagent_map.keys(): + # reagent_map = {k: v for k, v in self.kit_object.construct_xl_map_for_use(submission_type)} + logger.warning(f"Attempting to salvage with default kit {self.kit_object} and submission_type: {self.submission_type_obj}") + return self.fetch_kit_info_map(submission_type=self.submission_type_obj) + else: logger.error(f"Still no reagent map, displaying error.") try: ext_kit_loc = self.submission_type_obj.info_map['extraction_kit']['read'][0] diff --git a/src/submissions/backend/excel/reports.py b/src/submissions/backend/excel/reports.py index c7505ca..70ae2b2 100644 --- a/src/submissions/backend/excel/reports.py +++ b/src/submissions/backend/excel/reports.py @@ -1,6 +1,8 @@ ''' Contains functions for generating summary reports ''' +from pprint import pformat + from pandas import DataFrame, ExcelWriter import logging from pathlib import Path @@ -137,3 +139,35 @@ class ReportMaker(object): for cell in worksheet['D']: if cell.row > 1: cell.style = 'Currency' + +class TurnaroundMaker(object): + + def __init__(self, start_date: date, end_date: date): + self.start_date = start_date + self.end_date = end_date + # NOTE: Set page size to zero to override limiting query size. + self.subs = BasicSubmission.query(start_date=start_date, end_date=end_date, page_size=0) + records = [self.build_record(sub) for sub in self.subs] + self.df = DataFrame.from_records(records) + + @classmethod + def build_record(cls, sub): + days, tat_ok = sub.get_turnaround_time() + return dict(name=sub.rsl_plate_num, days=days, submitted_date=sub.submitted_date, + completed_date=sub.completed_date, acceptable=tat_ok) + + def write_report(self, filename: Path | str, obj: QWidget | None = None): + """ + Writes info to files. + + Args: + filename (Path | str): Basename of output file + obj (QWidget | None, optional): Parent object. Defaults to None. + """ + if isinstance(filename, str): + filename = Path(filename) + filename = filename.absolute() + self.writer = ExcelWriter(filename.with_suffix(".xlsx"), engine='openpyxl') + self.df.to_excel(self.writer, sheet_name="Turnaround") + # logger.debug(f"Writing report to: {filename}") + self.writer.close() \ No newline at end of file diff --git a/src/submissions/frontend/visualizations/__init__.py b/src/submissions/frontend/visualizations/__init__.py index c4af2e9..25994b7 100644 --- a/src/submissions/frontend/visualizations/__init__.py +++ b/src/submissions/frontend/visualizations/__init__.py @@ -1,6 +1,8 @@ ''' Contains all operations for creating charts, graphs and visual effects. ''' +from datetime import timedelta + from PyQt6.QtWidgets import QWidget import plotly, logging from plotly.graph_objects import Figure @@ -14,10 +16,15 @@ class CustomFigure(Figure): df = None - def __init__(self, df: pd.DataFrame, modes: list, ytitle: str | None = None, parent: QWidget | None = None, - months: int = 6): + def __init__(self, df: pd.DataFrame, settings: dict, modes: list, ytitle: str | None = None, parent: QWidget | None = None): super().__init__() + # self.settings = settings + try: + months = int(settings['months']) + except KeyError: + months = 6 self.df = df + self.update_xaxes(range=[settings['start_date'] - timedelta(days=1), settings['end_date']]) def save_figure(self, group_name: str = "plotly_output", parent: QWidget | None = None): """ diff --git a/src/submissions/frontend/visualizations/irida_charts.py b/src/submissions/frontend/visualizations/irida_charts.py index ab44919..eb8cd9d 100644 --- a/src/submissions/frontend/visualizations/irida_charts.py +++ b/src/submissions/frontend/visualizations/irida_charts.py @@ -16,19 +16,23 @@ logger = logging.getLogger(f"submissions.{__name__}") class IridaFigure(CustomFigure): - def __init__(self, df: pd.DataFrame, modes: list, ytitle: str | None = None, parent: QWidget | None = None, - months: int = 6): + def __init__(self, df: pd.DataFrame, modes: list, settings: dict, ytitle: str | None = None, parent: QWidget | None = None): - super().__init__(df=df, modes=modes) - - self.construct_chart(df=df, modes=modes) + super().__init__(df=df, modes=modes, settings=settings) + try: + months = int(settings['months']) + except KeyError: + months = 6 + self.construct_chart(df=df, modes=modes, start_date=settings['start_date'], end_date=settings['end_date']) self.generic_figure_markers(modes=modes, ytitle=ytitle, months=months) - def construct_chart(self, df: pd.DataFrame, modes: list): + def construct_chart(self, df: pd.DataFrame, modes: list, start_date: date, end_date:date): """ Creates a plotly chart for controls from a pandas dataframe Args: + end_date (): + start_date (): df (pd.DataFrame): input dataframe of controls modes (list): analysis modes to construct charts for ytitle (str | None, optional): title on the y-axis. Defaults to None. diff --git a/src/submissions/frontend/visualizations/pcr_charts.py b/src/submissions/frontend/visualizations/pcr_charts.py index 58041f8..0e0812a 100644 --- a/src/submissions/frontend/visualizations/pcr_charts.py +++ b/src/submissions/frontend/visualizations/pcr_charts.py @@ -13,9 +13,13 @@ logger = logging.getLogger(f"submissions.{__name__}") class PCRFigure(CustomFigure): - def __init__(self, df: pd.DataFrame, modes: list, ytitle: str | None = None, parent: QWidget | None = None, + def __init__(self, df: pd.DataFrame, modes: list, settings: dict, ytitle: str | None = None, parent: QWidget | None = None, months: int = 6): - super().__init__(df=df, modes=modes) + super().__init__(df=df, modes=modes, settings=settings) + try: + months = int(settings['months']) + except KeyError: + months = 6 # logger.debug(f"DF: {self.df}") self.construct_chart(df=df) diff --git a/src/submissions/frontend/widgets/__init__.py b/src/submissions/frontend/widgets/__init__.py index d4d67c5..afef2b5 100644 --- a/src/submissions/frontend/widgets/__init__.py +++ b/src/submissions/frontend/widgets/__init__.py @@ -11,4 +11,6 @@ from .controls_chart import * from .submission_details import * from .equipment_usage import * from .gel_checker import * +from .summary import Summary +from .turnaround import TurnaroundTime from .app import App diff --git a/src/submissions/frontend/widgets/app.py b/src/submissions/frontend/widgets/app.py index 60c856b..b0efa6b 100644 --- a/src/submissions/frontend/widgets/app.py +++ b/src/submissions/frontend/widgets/app.py @@ -25,6 +25,7 @@ from .submission_widget import SubmissionFormContainer from .controls_chart import ControlsViewer # from .sample_search import SampleSearchBox from .summary import Summary +from .turnaround import TurnaroundTime from .omni_search import SearchBox logger = logging.getLogger(f'submissions.{__name__}') @@ -269,12 +270,14 @@ class AddSubForm(QWidget): self.tab2 = QWidget() self.tab3 = QWidget() self.tab4 = QWidget() + self.tab5 = QWidget() self.tabs.resize(300, 200) # NOTE: Add tabs self.tabs.addTab(self.tab1, "Submissions") self.tabs.addTab(self.tab2, "Irida Controls") self.tabs.addTab(self.tab3, "PCR Controls") self.tabs.addTab(self.tab4, "Cost Report") + self.tabs.addTab(self.tab5, "Turnaround Times") # NOTE: Create submission adder form self.formwidget = SubmissionFormContainer(self) self.formlayout = QVBoxLayout(self) @@ -310,6 +313,10 @@ class AddSubForm(QWidget): self.tab4.layout = QVBoxLayout(self) self.tab4.layout.addWidget(summary_report) self.tab4.setLayout(self.tab4.layout) + turnaround = TurnaroundTime(self) + self.tab5.layout = QVBoxLayout(self) + self.tab5.layout.addWidget(turnaround) + self.tab5.setLayout(self.tab5.layout) # NOTE: add tabs to main widget self.layout.addWidget(self.tabs) self.setLayout(self.layout) diff --git a/src/submissions/frontend/widgets/submission_details.py b/src/submissions/frontend/widgets/submission_details.py index faebc85..b98d4fe 100644 --- a/src/submissions/frontend/widgets/submission_details.py +++ b/src/submissions/frontend/widgets/submission_details.py @@ -172,11 +172,11 @@ class SubmissionDetails(QDialog): @pyqtSlot(str) def sign_off(self, submission: str | BasicSubmission): - # logger.debug(f"Signing off on {submission} - ({getuser()})") + logger.debug(f"Signing off on {submission} - ({getuser()})") if isinstance(submission, str): submission = BasicSubmission.query(rsl_plate_num=submission) submission.signed_by = getuser() - submission.completed = datetime.now().date() + submission.completed_date = datetime.now().date() submission.save() self.submission_details(submission=self.rsl_plate_num) diff --git a/src/submissions/frontend/widgets/summary.py b/src/submissions/frontend/widgets/summary.py index 1f5c586..ad33f88 100644 --- a/src/submissions/frontend/widgets/summary.py +++ b/src/submissions/frontend/widgets/summary.py @@ -1,5 +1,6 @@ from PyQt6.QtCore import QSignalBlocker from PyQt6.QtWebEngineWidgets import QWebEngineView +from .info_tab import InfoPane from PyQt6.QtWidgets import QWidget, QGridLayout, QPushButton, QLabel from backend.db import Organization from backend.excel import ReportMaker @@ -11,38 +12,21 @@ import logging logger = logging.getLogger(f"submissions.{__name__}") -class Summary(QWidget): +class Summary(InfoPane): def __init__(self, parent: QWidget) -> None: super().__init__(parent) - self.app = self.parent().parent() - # logger.debug(f"\n\n{self.app}\n\n") - self.report = Report() - self.datepicker = StartEndDatePicker(default_start=-31) - self.webview = QWebEngineView() - self.datepicker.start_date.dateChanged.connect(self.get_report) - self.datepicker.end_date.dateChanged.connect(self.get_report) - self.layout = QGridLayout(self) - self.layout.addWidget(self.datepicker, 0, 0, 1, 2) - self.save_excel_button = QPushButton("Save Excel", parent=self) - self.save_excel_button.pressed.connect(self.save_excel) - self.save_pdf_button = QPushButton("Save PDF", parent=self) - self.save_pdf_button.pressed.connect(self.save_pdf) self.org_select = CheckableComboBox() self.org_select.setEditable(False) self.org_select.addItem("Select", header=True) for org in [org.name for org in Organization.query()]: self.org_select.addItem(org) - self.org_select.model().itemChanged.connect(self.get_report) - self.layout.addWidget(self.save_excel_button, 0, 2, 1, 1) - self.layout.addWidget(self.save_pdf_button, 0, 3, 1, 1) - self.layout.addWidget(self.webview, 2, 0, 1, 4) + self.org_select.model().itemChanged.connect(self.date_changed) self.layout.addWidget(QLabel("Client"), 1, 0, 1, 1) self.layout.addWidget(self.org_select, 1, 1, 1, 3) - self.setLayout(self.layout) - self.get_report() + self.date_changed() - def get_report(self): + def date_changed(self): orgs = [self.org_select.itemText(i) for i in range(self.org_select.count()) if self.org_select.itemChecked(i)] if self.datepicker.start_date.date() > self.datepicker.end_date.date(): logger.warning("Start date after end date is not allowed!") @@ -51,11 +35,12 @@ class Summary(QWidget): # Without triggering this function again with QSignalBlocker(self.datepicker.start_date) as blocker: self.datepicker.start_date.setDate(lastmonth) - self.get_report() + self.date_changed() return # NOTE: convert to python useable date objects - self.start_date = self.datepicker.start_date.date().toPyDate() - self.end_date = self.datepicker.end_date.date().toPyDate() + # self.start_date = self.datepicker.start_date.date().toPyDate() + # self.end_date = self.datepicker.end_date.date().toPyDate() + super().date_changed() logger.debug(f"Getting report from {self.start_date} to {self.end_date} using {orgs}") self.report_obj = ReportMaker(start_date=self.start_date, end_date=self.end_date, organizations=orgs) self.webview.setHtml(self.report_obj.html) @@ -66,12 +51,12 @@ class Summary(QWidget): self.save_pdf_button.setEnabled(False) self.save_excel_button.setEnabled(False) - def save_excel(self): - fname = select_save_file(self, default_name=f"Report {self.start_date.strftime('%Y%m%d')} - {self.end_date.strftime('%Y%m%d')}", extension="xlsx") - self.report_obj.write_report(fname, obj=self) - - def save_pdf(self): - fname = select_save_file(obj=self, - default_name=f"Report {self.start_date.strftime('%Y%m%d')} - {self.end_date.strftime('%Y%m%d')}", - extension="pdf") - save_pdf(obj=self.webview, filename=fname) + # def save_excel(self): + # fname = select_save_file(self, default_name=f"Report {self.start_date.strftime('%Y%m%d')} - {self.end_date.strftime('%Y%m%d')}", extension="xlsx") + # self.report_obj.write_report(fname, obj=self) + # + # def save_pdf(self): + # fname = select_save_file(obj=self, + # default_name=f"Report {self.start_date.strftime('%Y%m%d')} - {self.end_date.strftime('%Y%m%d')}", + # extension="pdf") + # save_pdf(obj=self.webview, filename=fname)