From ae5fb1b48f4abe041638fd8e722a67bd1d2c529c Mon Sep 17 00:00:00 2001 From: lwark Date: Tue, 8 Oct 2024 14:55:54 -0500 Subject: [PATCH] Increasing generator usage. --- CHANGELOG.md | 5 ++ src/submissions/backend/db/models/kits.py | 13 +-- .../backend/db/models/submissions.py | 11 ++- src/submissions/backend/excel/parser.py | 82 ++----------------- .../frontend/visualizations/control_charts.py | 46 +++++++---- src/submissions/frontend/widgets/app.py | 2 +- .../frontend/widgets/controls_chart.py | 11 ++- src/submissions/frontend/widgets/misc.py | 60 +++----------- src/submissions/tools/__init__.py | 2 +- 9 files changed, 81 insertions(+), 151 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 683b8b0..31905f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## 202410.02 + +- Trimmed down html timeline buttons for controls window. +- Improved paginator for submissions table. + ## 202410.01 - Reverted details exports from docx back to pdf. diff --git a/src/submissions/backend/db/models/kits.py b/src/submissions/backend/db/models/kits.py index 42da5e2..fc5e539 100644 --- a/src/submissions/backend/db/models/kits.py +++ b/src/submissions/backend/db/models/kits.py @@ -142,8 +142,8 @@ class KitType(BaseClass): """ return f"" - def get_reagents(self, required: bool = False, submission_type: str | SubmissionType | None = None) -> List[ - ReagentRole]: + def get_reagents(self, required: bool = False, submission_type: str | SubmissionType | None = None) -> Generator[ + ReagentRole, None, None]: """ Return ReagentTypes linked to kit through KitTypeReagentTypeAssociation. @@ -168,9 +168,9 @@ class KitType(BaseClass): relevant_associations = [item for item in self.kit_reagentrole_associations] if required: # logger.debug(f"Filtering by required.") - return [item.reagent_role for item in relevant_associations if item.required == 1] + return (item.reagent_role for item in relevant_associations if item.required == 1) else: - return [item.reagent_role for item in relevant_associations] + return (item.reagent_role for item in relevant_associations) # TODO: Move to BasicSubmission? def construct_xl_map_for_use(self, submission_type: str | SubmissionType) -> Generator[(str, str)]: @@ -198,6 +198,7 @@ class KitType(BaseClass): # logger.debug("Get all KitTypeReagentTypeAssociation for SubmissionType") for assoc in assocs: try: + logger.debug(f"Yielding: {assoc.reagent_role.name}, {assoc.uses}") yield assoc.reagent_role.name, assoc.uses except TypeError: continue @@ -764,14 +765,14 @@ class SubmissionType(BaseClass): tmap = {} yield item.tip_role.name, tmap - def get_equipment(self, extraction_kit: str | KitType | None = None) -> List['PydEquipmentRole']: + def get_equipment(self, extraction_kit: str | KitType | None = None) -> Generator['PydEquipmentRole', None, None]: """ Returns PydEquipmentRole of all equipment associated with this SubmissionType Returns: List[PydEquipmentRole]: List of equipment roles """ - return [item.to_pydantic(submission_type=self, extraction_kit=extraction_kit) for item in self.equipment] + return (item.to_pydantic(submission_type=self, extraction_kit=extraction_kit) for item in self.equipment) def get_processes_for_role(self, equipment_role: str | EquipmentRole, kit: str | KitType | None = None) -> list: """ diff --git a/src/submissions/backend/db/models/submissions.py b/src/submissions/backend/db/models/submissions.py index c32a178..9afd84c 100644 --- a/src/submissions/backend/db/models/submissions.py +++ b/src/submissions/backend/db/models/submissions.py @@ -219,11 +219,11 @@ class BasicSubmission(BaseClass): if len(args) == 1: try: return output[args[0]] - except KeyError: + except KeyError as e: if "pytest" in sys.modules and args[0] == "abbreviation": return "BS" else: - raise KeyError("args[0]") + raise KeyError(f"{args[0]} not found in {output}") return output @classmethod @@ -237,6 +237,13 @@ class BasicSubmission(BaseClass): Returns: SubmissionType: SubmissionType with name equal to this polymorphic identity """ + logger.debug(f"Running search for {sub_type}") + if isinstance(sub_type, dict): + try: + sub_type = sub_type['value'] + except KeyError as e: + logger.error(f"Couldn't extract value from {sub_type}") + raise e match sub_type: case str(): return SubmissionType.query(name=sub_type) diff --git a/src/submissions/backend/excel/parser.py b/src/submissions/backend/excel/parser.py index 5a1d87a..d8f39b7 100644 --- a/src/submissions/backend/excel/parser.py +++ b/src/submissions/backend/excel/parser.py @@ -80,14 +80,6 @@ class SheetParser(object): self.submission_type = RSLNamer.retrieve_submission_type(filename=self.filepath) self.parse_info() [self.sub.__setitem__(k, v) for k, v in info.items()] - # for k, v in info.items(): - # match k: - # # NOTE: exclude samples. - # case "sample": - # logger.debug(f"Sample found: {k}: {v}") - # continue - # case _: - # self.sub[k] = v def parse_reagents(self, extraction_kit: str | None = None): """ @@ -147,30 +139,8 @@ class SheetParser(object): Returns: PydSubmission: output pydantic model """ - # logger.debug(f"Submission dictionary coming into 'to_pydantic':\n{pformat(self.sub)}") - # pyd_dict = copy(self.sub) - # self.sub['samples'] = [PydSample(**sample) for sample in self.sub['samples']] - # logger.debug(f"Reagents: {pformat(self.sub['reagents'])}") - # self.sub['reagents'] = [PydReagent(**reagent) for reagent in self.sub['reagents']] - # logger.debug(f"Equipment: {self.sub['equipment']}") - # try: - # check = bool(self.sub['equipment']) - # except TypeError: - # check = False - # if check: - # self.sub['equipment'] = [PydEquipment(**equipment) for equipment in self.sub['equipment']] - # else: - # self.sub['equipment'] = None - # try: - # check = bool(self.sub['tips']) - # except TypeError: - # check = False - # if check: - # self.sub['tips'] = [PydTips(**tips) for tips in self.sub['tips']] - # else: - # self.sub['tips'] = None return PydSubmission(filepath=self.filepath, run_custom=True, **self.sub) - # return psm + class InfoParser(object): @@ -298,6 +268,7 @@ class ReagentParser(object): if isinstance(extraction_kit, dict): extraction_kit = extraction_kit['value'] self.kit_object = KitType.query(name=extraction_kit) + # logger.debug(f"Got extraction kit object: {self.kit_object}") self.map = self.fetch_kit_info_map(submission_type=submission_type) # logger.debug(f"Reagent Parser map: {self.map}") self.xl = xl @@ -329,15 +300,14 @@ class ReagentParser(object): Returns: List[PydReagent]: List of parsed reagents. """ - # listo = [] for sheet in self.xl.sheetnames: ws = self.xl[sheet] relevant = {k.strip(): v for k, v in self.map.items() if sheet in self.map[k]['sheet']} - # logger.debug(f"relevant map for {sheet}: {pformat(relevant)}") + logger.debug(f"relevant map for {sheet}: {pformat(relevant)}") if relevant == {}: continue for item in relevant: - # logger.debug(f"Attempting to scrape: {item}") + logger.debug(f"Attempting to scrape: {item}") try: reagent = relevant[item] name = ws.cell(row=reagent['name']['row'], column=reagent['name']['column']).value @@ -350,16 +320,15 @@ class ReagentParser(object): comment = "" except (KeyError, IndexError): yield dict(role=item.strip(), lot=None, expiry=None, name=None, comment="", missing=True) - # continue # NOTE: If the cell is blank tell the PydReagent if check_not_nan(lot): missing = False else: missing = True - # logger.debug(f"Got lot for {item}-{name}: {lot} as {type(lot)}") + logger.debug(f"Got lot for {item}-{name}: {lot} as {type(lot)}") lot = str(lot) - # logger.debug( - # f"Going into pydantic: name: {name}, lot: {lot}, expiry: {expiry}, type: {item.strip()}, comment: {comment}") + logger.debug( + f"Going into pydantic: name: {name}, lot: {lot}, expiry: {expiry}, type: {item.strip()}, comment: {comment}") try: check = name.lower() != "not applicable" except AttributeError: @@ -368,7 +337,6 @@ class ReagentParser(object): if check: yield dict(role=item.strip(), lot=lot, expiry=expiry, name=name, comment=comment, missing=missing) - # return listo class SampleParser(object): @@ -480,34 +448,6 @@ class SampleParser(object): lookup_samples.append(self.samp_object.parse_sample(row_dict)) return lookup_samples - # def parse_samples(self) -> Tuple[Report | None, List[dict] | List[PydSample]]: - # """ - # Parse merged platemap/lookup info into dicts/samples - # - # Returns: - # List[dict]|List[models.BasicSample]: List of samples - # """ - # result = None - # new_samples = [] - # # logger.debug(f"Starting samples: {pformat(self.samples)}") - # for sample in self.samples: - # translated_dict = {} - # for k, v in sample.items(): - # match v: - # case dict(): - # v = None - # case float(): - # v = convert_nans_to_nones(v) - # case _: - # v = v - # translated_dict[k] = convert_nans_to_nones(v) - # translated_dict['sample_type'] = f"{self.submission_type} Sample" - # translated_dict = self.sub_object.parse_samples(translated_dict) - # translated_dict = self.samp_object.parse_sample(translated_dict) - # # logger.debug(f"Here is the output of the custom parser:\n{translated_dict}") - # new_samples.append(PydSample(**translated_dict)) - # return result, new_samples - def parse_samples(self) -> Generator[dict, None, None]: """ Merges sample info from lookup table and plate map. @@ -522,9 +462,8 @@ class SampleParser(object): merge_on_id = self.sample_info_map['lookup_table']['merge_on_id'] plate_map_samples = sorted(copy(self.plate_map_samples), key=lambda d: d['id']) lookup_samples = sorted(copy(self.lookup_samples), key=lambda d: d[merge_on_id]) - # print(pformat(plate_map_samples)) - # print(pformat(lookup_samples)) for ii, psample in enumerate(plate_map_samples): + # NOTE: See if we can do this the easy way and just use the same list index. try: check = psample['id'] == lookup_samples[ii][merge_on_id] except (KeyError, IndexError): @@ -534,7 +473,7 @@ class SampleParser(object): new = lookup_samples[ii] | psample lookup_samples[ii] = {} else: - # logger.warning(f"Match for {psample['id']} not direct, running search.") + logger.warning(f"Match for {psample['id']} not direct, running search.") for jj, lsample in enumerate(lookup_samples): try: check = lsample[merge_on_id] == psample['id'] @@ -551,9 +490,6 @@ class SampleParser(object): new = self.sub_object.parse_samples(new) del new['id'] yield new - # samples.append(new) - # samples = remove_key_from_list_of_dicts(samples, "id") - # return sorted(samples, key=lambda k: (k['row'], k['column'])) class EquipmentParser(object): diff --git a/src/submissions/frontend/visualizations/control_charts.py b/src/submissions/frontend/visualizations/control_charts.py index dea7421..5d5084a 100644 --- a/src/submissions/frontend/visualizations/control_charts.py +++ b/src/submissions/frontend/visualizations/control_charts.py @@ -2,6 +2,7 @@ Functions for constructing controls graphs using plotly. """ from copy import deepcopy +from datetime import date from pprint import pformat import plotly @@ -18,10 +19,12 @@ logger = logging.getLogger(f"submissions.{__name__}") class CustomFigure(Figure): - def __init__(self, df: pd.DataFrame, modes: list, ytitle: str | None = None, parent: QWidget | None = None): + def __init__(self, df: pd.DataFrame, modes: list, ytitle: str | None = None, parent: QWidget | None = None, + months: int = 6): super().__init__() + self.construct_chart(df=df, modes=modes) - self.generic_figure_markers(modes=modes, ytitle=ytitle) + self.generic_figure_markers(modes=modes, ytitle=ytitle, months=months) def construct_chart(self, df: pd.DataFrame, modes: list): """ @@ -67,7 +70,7 @@ class CustomFigure(Figure): self.add_traces(bar.data) # return generic_figure_markers(modes=modes, ytitle=ytitle) - def generic_figure_markers(self, modes: list = [], ytitle: str | None = None): + def generic_figure_markers(self, modes: list = [], ytitle: str | None = None, months: int = 6): """ Adds standard layout to figure. @@ -94,27 +97,41 @@ class CustomFigure(Figure): x=0.7, y=1.2, showactive=True, - buttons=[button for button in self.make_buttons(modes=modes)], + buttons=[button for button in self.make_pyqt_buttons(modes=modes)], ) ] ) + self.update_xaxes( rangeslider_visible=True, rangeselector=dict( - buttons=list([ - dict(count=1, label="1m", step="month", stepmode="backward"), - dict(count=3, label="3m", step="month", stepmode="backward"), - dict(count=6, label="6m", step="month", stepmode="backward"), - dict(count=1, label="YTD", step="year", stepmode="todate"), - dict(count=1, label="1y", step="year", stepmode="backward"), - dict(step="all") - ]) + # buttons=list([ + # dict(count=1, label="1m", step="month", stepmode="backward"), + # dict(count=3, label="3m", step="month", stepmode="backward"), + # dict(count=6, label="6m", step="month", stepmode="backward"), + # dict(count=1, label="YTD", step="year", stepmode="todate"), + # dict(count=12, label="1y", step="month", stepmode="backward"), + # dict(step="all") + # ]) + buttons=[button for button in self.make_plotly_buttons(months=months)] ) ) assert isinstance(self, Figure) # return fig - def make_buttons(self, modes: list) -> list: + def make_plotly_buttons(self, months:int=6): + rng = [1] + if months > 2: + rng += [iii for iii in range(3, months, 3)] + logger.debug(f"Making buttons for months: {rng}") + buttons = [dict(count=iii, label=f"{iii}m", step="month", stepmode="backward") for iii in rng] + if months > date.today().month: + buttons += [dict(count=1, label="YTD", step="year", stepmode="todate")] + buttons += [dict(step="all")] + for button in buttons: + yield button + + def make_pyqt_buttons(self, modes: list) -> list: """ Creates list of buttons with one for each mode to be used in showing/hiding mode traces. @@ -144,7 +161,7 @@ class CustomFigure(Figure): {"yaxis.title.text": mode}, ]) - def save_figure(self, group_name: str = "plotly_output", parent:QWidget|None=None): + def save_figure(self, group_name: str = "plotly_output", parent: QWidget | None = None): """ Writes plotly figure to html file. @@ -158,7 +175,6 @@ class CustomFigure(Figure): output = select_save_file(obj=parent, default_name=group_name, extension="png") self.write_image(output.absolute().__str__(), engine="kaleido") - def to_html(self) -> str: """ Creates final html code from plotly diff --git a/src/submissions/frontend/widgets/app.py b/src/submissions/frontend/widgets/app.py index 0fceabd..fd4fd16 100644 --- a/src/submissions/frontend/widgets/app.py +++ b/src/submissions/frontend/widgets/app.py @@ -240,7 +240,7 @@ class App(QMainWindow): logger.warning("Save of submission type cancelled.") def update_data(self): - self.table_widget.sub_wid.setData(page=int(self.table_widget.pager.current_page.text()), page_size=page_size) + self.table_widget.sub_wid.setData(page=self.table_widget.pager.page_anchor, page_size=page_size) class AddSubForm(QWidget): diff --git a/src/submissions/frontend/widgets/controls_chart.py b/src/submissions/frontend/widgets/controls_chart.py index 09aa66c..1e220b6 100644 --- a/src/submissions/frontend/widgets/controls_chart.py +++ b/src/submissions/frontend/widgets/controls_chart.py @@ -3,7 +3,7 @@ Handles display of control charts """ import re import sys -from datetime import timedelta +from datetime import timedelta, date from typing import Tuple from PyQt6.QtWebEngineWidgets import QWebEngineView from PyQt6.QtWidgets import ( @@ -113,6 +113,9 @@ class ControlsViewer(QWidget): self.chart_maker() return report + def diff_month(self, d1:date, d2:date): + return abs((d1.year - d2.year) * 12 + d1.month - d2.month) + def chart_maker(self): """ Creates plotly charts for webview @@ -132,7 +135,8 @@ class ControlsViewer(QWidget): """ report = Report() # logger.debug(f"Control getter context: \n\tControl type: {self.con_type}\n\tMode: {self.mode}\n\tStart - # Date: {self.start_date}\n\tEnd Date: {self.end_date}") NOTE: set the subtype for kraken + # Date: {self.start_date}\n\tEnd Date: {self.end_date}") + # NOTE: set the subtype for kraken if self.sub_typer.currentText() == "": self.subtype = None else: @@ -161,7 +165,8 @@ class ControlsViewer(QWidget): title = f"{self.mode} - {self.subtype}" # NOTE: send dataframe to chart maker df, modes = self.prep_df(ctx=self.app.ctx, df=df) - fig = CustomFigure(df=df, ytitle=title, modes=modes, parent=self) + months = self.diff_month(self.start_date, self.end_date) + fig = CustomFigure(df=df, ytitle=title, modes=modes, parent=self, months=months) self.save_button.setEnabled(True) # logger.debug(f"Updating figure...") self.fig = fig diff --git a/src/submissions/frontend/widgets/misc.py b/src/submissions/frontend/widgets/misc.py index 1710bfd..813ac96 100644 --- a/src/submissions/frontend/widgets/misc.py +++ b/src/submissions/frontend/widgets/misc.py @@ -108,43 +108,6 @@ class AddReagentForm(QDialog): self.name_input.addItems(list(set([item.name for item in lookup]))) -# class ReportDatePicker(QDialog): -# """ -# custom dialog to ask for report start/stop dates -# """ -# def __init__(self) -> None: -# super().__init__() -# self.setWindowTitle("Select Report Date Range") -# # NOTE: make confirm/reject buttons -# QBtn = QDialogButtonBox.StandardButton.Ok | QDialogButtonBox.StandardButton.Cancel -# self.buttonBox = QDialogButtonBox(QBtn) -# self.buttonBox.accepted.connect(self.accept) -# self.buttonBox.rejected.connect(self.reject) -# # NOTE: widgets to ask for dates -# self.start_date = QDateEdit(calendarPopup=True) -# self.start_date.setObjectName("start_date") -# self.start_date.setDate(QDate.currentDate()) -# self.end_date = QDateEdit(calendarPopup=True) -# self.end_date.setObjectName("end_date") -# self.end_date.setDate(QDate.currentDate()) -# self.layout = QVBoxLayout() -# self.layout.addWidget(QLabel("Start Date")) -# self.layout.addWidget(self.start_date) -# self.layout.addWidget(QLabel("End Date")) -# self.layout.addWidget(self.end_date) -# self.layout.addWidget(self.buttonBox) -# self.setLayout(self.layout) -# -# def parse_form(self) -> dict: -# """ -# Converts information in this object to a dict -# -# Returns: -# dict: output dict. -# """ -# return dict(start_date=self.start_date.date().toPyDate(), end_date = self.end_date.date().toPyDate()) - - class LogParser(QDialog): def __init__(self, parent): @@ -253,35 +216,32 @@ class Pagifier(QWidget): def __init__(self, page_max:int): super().__init__() self.page_max = math.ceil(page_max) - + self.page_anchor = 1 next = QPushButton(parent=self, icon = QIcon.fromTheme(QIcon.ThemeIcon.GoNext)) next.pressed.connect(self.increment_page) previous = QPushButton(parent=self, icon=QIcon.fromTheme(QIcon.ThemeIcon.GoPrevious)) previous.pressed.connect(self.decrement_page) - label = QLabel(f"/ {self.page_max}") - label.setMinimumWidth(200) - label.setAlignment(Qt.AlignmentFlag.AlignCenter) self.current_page = QLineEdit(self) self.current_page.setEnabled(False) - # onlyInt = QIntValidator() - # onlyInt.setRange(1, 4) - # self.current_page.setValidator(onlyInt) - self.current_page.setText("1") + self.update_current_page() self.current_page.setAlignment(Qt.AlignmentFlag.AlignCenter) self.layout = QHBoxLayout() self.layout.addWidget(previous) self.layout.addWidget(self.current_page) - self.layout.addWidget(label) self.layout.addWidget(next) self.setLayout(self.layout) def increment_page(self): - new = int(self.current_page.text())+1 + new = self.page_anchor + 1 if new <= self.page_max: - self.current_page.setText(str(new)) + self.page_anchor = new + self.update_current_page() def decrement_page(self): - new = int(self.current_page.text())-1 + new = self.page_anchor - 1 if new >= 1: - self.current_page.setText(str(new)) + self.page_anchor = new + self.update_current_page() + def update_current_page(self): + self.current_page.setText(f"{self.page_anchor} of {self.page_max}") \ No newline at end of file diff --git a/src/submissions/tools/__init__.py b/src/submissions/tools/__init__.py index e3d92a4..48f52d5 100644 --- a/src/submissions/tools/__init__.py +++ b/src/submissions/tools/__init__.py @@ -19,7 +19,7 @@ from pydantic_settings import BaseSettings, SettingsConfigDict from typing import Any, Tuple, Literal, List from __init__ import project_path from configparser import ConfigParser -from tkinter import Tk # from tkinter import Tk for Python 3.x +from tkinter import Tk # NOTE: This is for choosing database path before app is created. from tkinter.filedialog import askdirectory from sqlalchemy.exc import IntegrityError as sqlalcIntegrityError