From c3a4aac68be4091b1947c564d072e58062ad6900 Mon Sep 17 00:00:00 2001 From: lwark Date: Wed, 16 Oct 2024 15:07:43 -0500 Subject: [PATCH] Updated controls to both Irida and PCR. --- CHANGELOG.md | 5 + src/submissions/backend/db/models/controls.py | 547 ++++++++++++++++-- .../backend/db/models/submissions.py | 114 ++-- src/submissions/backend/excel/parser.py | 2 +- .../frontend/visualizations/__init__.py | 50 +- .../{control_charts.py => irida_charts.py} | 58 +- .../frontend/visualizations/pcr_charts.py | 34 ++ src/submissions/frontend/widgets/app.py | 52 +- .../frontend/widgets/controls_chart.py | 184 +++--- .../frontend/widgets/submission_details.py | 15 +- src/submissions/tools/__init__.py | 3 +- 11 files changed, 750 insertions(+), 314 deletions(-) rename src/submissions/frontend/visualizations/{control_charts.py => irida_charts.py} (73%) create mode 100644 src/submissions/frontend/visualizations/pcr_charts.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 31905f1..d99510f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,12 @@ +## 202410.03 + +- Added code for cataloging of PCR controls. + ## 202410.02 - Trimmed down html timeline buttons for controls window. - Improved paginator for submissions table. +- Refactor of Controls to support multiple types. (Note: Irida parser not updated yet.) ## 202410.01 diff --git a/src/submissions/backend/db/models/controls.py b/src/submissions/backend/db/models/controls.py index 9eb7db2..a484be4 100644 --- a/src/submissions/backend/db/models/controls.py +++ b/src/submissions/backend/db/models/controls.py @@ -2,14 +2,19 @@ All control related models. """ from __future__ import annotations -from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey -from sqlalchemy.orm import relationship, Query +from pprint import pformat + +from PyQt6.QtWidgets import QWidget, QCheckBox, QLabel +from pandas import DataFrame +from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey, case, FLOAT +from sqlalchemy.orm import relationship, Query, validates import logging, re from operator import itemgetter + from . import BaseClass -from tools import setup_lookup -from datetime import date, datetime -from typing import List, Literal +from tools import setup_lookup, report_result, Result, Report, Settings, get_unique_values_in_df_column +from datetime import date, datetime, timedelta +from typing import List, Literal, Tuple, Generator from dateutil.parser import parse from re import Pattern @@ -21,7 +26,7 @@ class ControlType(BaseClass): Base class of a control archetype. """ id = Column(INTEGER, primary_key=True) #: primary key - name = Column(String(255), unique=True) #: controltype name (e.g. MCS) + name = Column(String(255), unique=True) #: controltype name (e.g. Irida Control) targets = Column(JSON) #: organisms checked for instances = relationship("Control", back_populates="controltype") #: control samples created of this type. @@ -53,7 +58,7 @@ class ControlType(BaseClass): pass return cls.execute_query(query=query, limit=limit) - def get_subtypes(self, mode: Literal['kraken', 'matches', 'contains']) -> List[str]: + def get_modes(self, mode: Literal['kraken', 'matches', 'contains']) -> List[str]: """ Get subtypes associated with this controltype (currently used only for Kraken) @@ -65,8 +70,10 @@ class ControlType(BaseClass): """ # NOTE: Get first instance since all should have same subtypes # NOTE: Get mode of instance + if not self.instances: + return jsoner = getattr(self.instances[0], mode) - # logger.debug(f"JSON out: {jsoner.keys()}") + # logger.debug(f"JSON retrieved: {jsoner.keys()}") try: # NOTE: Pick genera (all should have same subtypes) genera = list(jsoner.keys())[0] @@ -74,10 +81,14 @@ class ControlType(BaseClass): return [] # NOTE: remove items that don't have relevant data subtypes = [item for item in jsoner[genera] if "_hashes" not in item and "_ratio" not in item] + logger.debug(f"subtypes out: {pformat(subtypes)}") return subtypes + def get_instance_class(self): + return Control.find_polymorphic_subclass(polymorphic_identity=self.name) + @classmethod - def get_positive_control_types(cls) -> List[ControlType]: + def get_positive_control_types(cls) -> Generator[ControlType, None, None]: """ Gets list of Control types if they have targets @@ -104,35 +115,234 @@ class Control(BaseClass): """ id = Column(INTEGER, primary_key=True) #: primary key - parent_id = Column(String, - ForeignKey("_controltype.id", name="fk_control_parent_id")) #: primary key of control type + controltype_name = Column(String, ForeignKey("_controltype.name", ondelete="SET NULL", + name="fk_BC_subtype_name")) #: name of joined submission type controltype = relationship("ControlType", back_populates="instances", - foreign_keys=[parent_id]) #: reference to parent control type + foreign_keys=[controltype_name]) #: reference to parent control type name = Column(String(255), unique=True) #: Sample ID submitted_date = Column(TIMESTAMP) #: Date submitted to Robotics + submission_id = Column(INTEGER, ForeignKey("_basicsubmission.id")) #: parent submission id + submission = relationship("BasicSubmission", back_populates="controls", + foreign_keys=[submission_id]) #: parent submission + + __mapper_args__ = { + "polymorphic_identity": "Basic Control", + "polymorphic_on": case( + + (controltype_name == "PCR Control", "PCR Control"), + (controltype_name == "Irida Control", "Irida Control"), + + else_="Basic Control" + ), + "with_polymorphic": "*", + } + + def __repr__(self) -> str: + return f"<{self.controltype_name}({self.name})>" + + @classmethod + def find_polymorphic_subclass(cls, polymorphic_identity: str | ControlType | None = None, + attrs: dict | None = None): + """ + Find subclass based on polymorphic identity or relevant attributes. + + Args: + polymorphic_identity (str | None, optional): String representing polymorphic identity. Defaults to None. + attrs (str | SubmissionType | None, optional): Attributes of the relevant class. Defaults to None. + + Returns: + _type_: Subclass of interest. + """ + if isinstance(polymorphic_identity, dict): + # logger.debug(f"Controlling for dict value") + polymorphic_identity = polymorphic_identity['value'] + if isinstance(polymorphic_identity, ControlType): + polymorphic_identity = polymorphic_identity.name + model = cls + match polymorphic_identity: + case str(): + try: + model = cls.__mapper__.polymorphic_map[polymorphic_identity].class_ + except Exception as e: + logger.error( + f"Could not get polymorph {polymorphic_identity} of {cls} due to {e}, falling back to BasicSubmission") + case _: + pass + if attrs and any([not hasattr(cls, attr) for attr in attrs.keys()]): + # NOTE: looks for first model that has all included kwargs + try: + model = next(subclass for subclass in cls.__subclasses__() if + all([hasattr(subclass, attr) for attr in attrs.keys()])) + except StopIteration as e: + raise AttributeError( + f"Couldn't find existing class/subclass of {cls} with all attributes:\n{pformat(attrs.keys())}") + logger.info(f"Recruiting model: {model}") + return model + + @classmethod + def make_parent_buttons(cls, parent: QWidget) -> None: + """ + + Args: + parent (QWidget): chart holding widget to add buttons to. + + Returns: + + """ + pass + + @classmethod + def make_chart(cls, parent, chart_settings: dict, ctx): + """ + + Args: + chart_settings (dict): settings passed down from chart widget + ctx (Settings): settings passed down from gui + + Returns: + + """ + return None + + +class PCRControl(Control): + id = Column(INTEGER, ForeignKey('_control.id'), primary_key=True) + subtype = Column(String(16)) #: PC or NC + target = Column(String(16)) #: N1, N2, etc. + ct = Column(FLOAT) + reagent_lot = Column(String(64), ForeignKey("_reagent.name", ondelete="SET NULL", + name="fk_reagent_lot")) + reagent = relationship("Reagent", foreign_keys=reagent_lot) + + __mapper_args__ = dict(polymorphic_identity="PCR Control", + polymorphic_load="inline", + inherit_condition=(id == Control.id)) + + def to_sub_dict(self): + return dict(name=self.name, ct=self.ct, subtype=self.subtype, target=self.target, reagent_lot=self.reagent_lot, + submitted_date=self.submitted_date.date()) + + @classmethod + @setup_lookup + def query(cls, + sub_type: str | None = None, + start_date: date | str | int | None = None, + end_date: date | str | int | None = None, + control_name: str | None = None, + limit: int = 0 + ) -> Control | List[Control]: + """ + Lookup control objects in the database based on a number of parameters. + + Args: + sub_type (models.ControlType | str | None, optional): Control archetype. Defaults to None. + start_date (date | str | int | None, optional): Beginning date to search by. Defaults to 2023-01-01 if end_date not None. + end_date (date | str | int | None, optional): End date to search by. Defaults to today if start_date not None. + control_name (str | None, optional): Name of control. Defaults to None. + limit (int, optional): Maximum number of results to return (0 = all). Defaults to 0. + + Returns: + models.Control|List[models.Control]: Control object of interest. + """ + query: Query = cls.__database_session__.query(cls) + # NOTE: by date range + if start_date is not None and end_date is None: + logger.warning(f"Start date with no end date, using today.") + end_date = date.today() + if end_date is not None and start_date is None: + logger.warning(f"End date with no start date, using Jan 1, 2023") + start_date = date(2023, 1, 1) + if start_date is not None: + match start_date: + case date(): + # logger.debug(f"Lookup control by start date({start_date})") + start_date = start_date.strftime("%Y-%m-%d") + case int(): + # logger.debug(f"Lookup control by ordinal start date {start_date}") + start_date = datetime.fromordinal( + datetime(1900, 1, 1).toordinal() + start_date - 2).date().strftime("%Y-%m-%d") + case _: + # logger.debug(f"Lookup control with parsed start date {start_date}") + start_date = parse(start_date).strftime("%Y-%m-%d") + match end_date: + case date(): + # logger.debug(f"Lookup control by end date({end_date})") + end_date = end_date.strftime("%Y-%m-%d") + case int(): + # logger.debug(f"Lookup control by ordinal end date {end_date}") + end_date = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + end_date - 2).date().strftime( + "%Y-%m-%d") + case _: + # logger.debug(f"Lookup control with parsed end date {end_date}") + end_date = parse(end_date).strftime("%Y-%m-%d") + # logger.debug(f"Looking up BasicSubmissions from start date: {start_date} and end date: {end_date}") + query = query.filter(cls.submitted_date.between(start_date, end_date)) + match sub_type: + case str(): + from backend import BasicSubmission, SubmissionType + query = query.join(BasicSubmission).join(SubmissionType).filter(SubmissionType.name == sub_type) + case _: + pass + match control_name: + case str(): + # logger.debug(f"Lookup control by name {control_name}") + query = query.filter(cls.name.startswith(control_name)) + limit = 1 + case _: + pass + return cls.execute_query(query=query, limit=limit) + + @classmethod + def make_chart(cls, parent, chart_settings: dict, ctx): + from frontend.visualizations.pcr_charts import PCRFigure + parent.mode_typer.clear() + parent.mode_typer.setEnabled(False) + report = Report() + controls = cls.query(sub_type=chart_settings['sub_type'], start_date=chart_settings['start_date'], end_date=chart_settings['end_date']) + data = [control.to_sub_dict() for control in controls] + df = DataFrame.from_records(data) + try: + df = df[df.ct > 0.0] + except AttributeError: + df = df + fig = PCRFigure(df=df, modes=None) + return report, fig + + +class IridaControl(Control): + id = Column(INTEGER, ForeignKey('_control.id'), primary_key=True) contains = Column(JSON) #: unstructured hashes in contains.tsv for each organism matches = Column(JSON) #: unstructured hashes in matches.tsv for each organism kraken = Column(JSON) #: unstructured output from kraken_report - submission_id = Column(INTEGER, ForeignKey("_basicsubmission.id")) #: parent submission id - submission = relationship("BacterialCulture", back_populates="controls", - foreign_keys=[submission_id]) #: parent submission + sub_type = Column(String(16), nullable=False) #: EN-NOS, MCS-NOS, etc refseq_version = Column(String(16)) #: version of refseq used in fastq parsing kraken2_version = Column(String(16)) #: version of kraken2 used in fastq parsing kraken2_db_version = Column(String(32)) #: folder name of kraken2 db sample = relationship("BacterialCultureSample", back_populates="control") #: This control's submission sample sample_id = Column(INTEGER, ForeignKey("_basicsample.id", ondelete="SET NULL", name="cont_BCS_id")) #: sample id key + # submission_id = Column(INTEGER, ForeignKey("_basicsubmission.id")) #: parent submission id + # submission = relationship("BacterialCulture", back_populates="controls", + # foreign_keys=[submission_id]) #: parent submission - def __repr__(self) -> str: - return f"" + __mapper_args__ = dict(polymorphic_identity="Irida Control", + polymorphic_load="inline", + inherit_condition=(id == Control.id)) + + @validates("sub_type") + def enforce_subtype_literals(self, key: str, value: str): + acceptables = ['ATCC49226', 'ATCC49619', 'EN-NOS', "EN-SSTI", "MCS-NOS", "MCS-SSTI", "SN-NOS", "SN-SSTI"] + if value.upper() not in acceptables: + raise KeyError(f"Sub-type must be in {acceptables}") + return value def to_sub_dict(self) -> dict: """ - Converts object into convenient dictionary for use in submission summary + Converts object into convenient dictionary for use in submission summary - Returns: - dict: output dictionary containing: Name, Type, Targets, Top Kraken results - """ + Returns: + dict: output dictionary containing: Name, Type, Targets, Top Kraken results + """ # logger.debug("loading json string into dict") try: kraken = self.kraken @@ -153,25 +363,27 @@ class Control(BaseClass): else: targets = ["None"] # logger.debug("constructing output dictionary") - output = { - "name": self.name, - "type": self.controltype.name, - "targets": ", ".join(targets), - "kraken": new_kraken[0:10] - } + output = dict( + name=self.name, + type=self.controltype.name, + targets=", ".join(targets), + kraken=new_kraken[0:10] + ) return output - def convert_by_mode(self, mode: Literal['kraken', 'matches', 'contains']) -> List[dict]: + def convert_by_mode(self, control_sub_type: str, mode: Literal['kraken', 'matches', 'contains'], + consolidate: bool = False) -> Generator[dict, None, None]: """ split this instance into analysis types for controls graphs Args: - mode (str): analysis type, 'contains', etc + consolidate (bool): whether to merge all off-target genera. Defaults to False + control_sub_type (str): control subtype, 'MCS-NOS', etc. + mode (str): analysis type, 'contains', etc. Returns: List[dict]: list of records """ - output = [] # logger.debug("load json string for mode (i.e. contains, matches, kraken2)") try: data = self.__getattribute__(mode) @@ -179,6 +391,18 @@ class Control(BaseClass): data = {} if data is None: data = {} + # NOTE: Data truncation and consolidation. + if "kraken" in mode: + data = {k: v for k, v in sorted(data.items(), key=lambda d: d[1][f"{mode}_count"], reverse=True)[:50]} + else: + if consolidate: + on_tar = {k: v for k, v in data.items() if k.strip("*") in self.controltype.targets[control_sub_type]} + # logger.debug(f"Consolidating off-targets to: {self.controltype.targets[control_sub_type]}") + off_tar = sum(v[f'{mode}_ratio'] for k, v in data.items() if + k.strip("*") not in self.controltype.targets[control_sub_type]) + on_tar['Off-target'] = {f"{mode}_ratio": off_tar} + data = on_tar + # logger.debug(pformat(data)) # logger.debug(f"Length of data: {len(data)}") # logger.debug("dict keys are genera of bacteria, e.g. 'Streptococcus'") for genus in data: @@ -186,17 +410,13 @@ class Control(BaseClass): name=self.name, submitted_date=self.submitted_date, genus=genus, - target='Target' if genus.strip("*") in self.controltype.targets else "Off-target" + target='Target' if genus.strip("*") in self.controltype.targets[control_sub_type] else "Off-target" ) # logger.debug("get Target or Off-target of genus") # logger.debug("set 'contains_hashes', etc for genus") for key in data[genus]: _dict[key] = data[genus][key] - output.append(_dict) - # logger.debug("Have to triage kraken data to keep program from getting overwhelmed") - if "kraken" in mode: - output = sorted(output, key=lambda d: d[f"{mode}_count"], reverse=True)[:50] - return output + yield _dict @classmethod def get_modes(cls) -> List[str]: @@ -217,7 +437,7 @@ class Control(BaseClass): @classmethod @setup_lookup def query(cls, - control_type: ControlType | str | None = None, + sub_type: str | None = None, start_date: date | str | int | None = None, end_date: date | str | int | None = None, control_name: str | None = None, @@ -227,7 +447,7 @@ class Control(BaseClass): Lookup control objects in the database based on a number of parameters. Args: - control_type (models.ControlType | str | None, optional): Control archetype. Defaults to None. + sub_type (models.ControlType | str | None, optional): Control archetype. Defaults to None. start_date (date | str | int | None, optional): Beginning date to search by. Defaults to 2023-01-01 if end_date not None. end_date (date | str | int | None, optional): End date to search by. Defaults to today if start_date not None. control_name (str | None, optional): Name of control. Defaults to None. @@ -238,13 +458,14 @@ class Control(BaseClass): """ query: Query = cls.__database_session__.query(cls) # NOTE: by control type - match control_type: - case ControlType(): - # logger.debug(f"Looking up control by control type: {control_type}") - query = query.filter(cls.controltype == control_type) + match sub_type: + # case ControlType(): + # # logger.debug(f"Looking up control by control type: {sub_type}") + # query = query.filter(cls.controltype == sub_type) case str(): - # logger.debug(f"Looking up control by control type: {control_type}") - query = query.join(ControlType).filter(ControlType.name == control_type) + # logger.debug(f"Looking up control by control type: {sub_type}") + # query = query.join(ControlType).filter(ControlType.name == sub_type) + query = query.filter(cls.sub_type == sub_type) case _: pass # NOTE: by date range @@ -287,3 +508,241 @@ class Control(BaseClass): case _: pass return cls.execute_query(query=query, limit=limit) + + @classmethod + def make_parent_buttons(cls, parent: QWidget) -> None: + """ + + Args: + parent (QWidget): chart holding widget to add buttons to. + + Returns: + + """ + super().make_parent_buttons(parent=parent) + rows = parent.layout.rowCount() + logger.debug(f"Parent rows: {rows}") + checker = QCheckBox(parent) + checker.setChecked(True) + checker.setObjectName("irida_check") + checker.setToolTip("Pools off-target genera to save time.") + parent.layout.addWidget(QLabel("Consolidate Off-targets"), rows, 0, 1, 1) + parent.layout.addWidget(checker, rows, 1, 1, 2) + checker.checkStateChanged.connect(parent.controls_getter_function) + + @classmethod + @report_result + def make_chart(cls, chart_settings: dict, parent, ctx) -> Tuple[Report, "IridaFigure" | None]: + from frontend.visualizations import IridaFigure + try: + checker = parent.findChild(QCheckBox, name="irida_check") + if chart_settings['mode'] == "kraken": + checker.setEnabled(False) + checker.setChecked(False) + else: + checker.setEnabled(True) + consolidate = checker.isChecked() + except AttributeError: + consolidate = False + report = Report() + # logger.debug(f"settings: {pformat(chart_settings)}") + controls = cls.query(sub_type=chart_settings['sub_type'], start_date=chart_settings['start_date'], + end_date=chart_settings['end_date']) + # logger.debug(f"Controls found: {controls}") + if not controls: + report.add_result(Result(status="Critical", msg="No controls found in given date range.")) + return report, None + # NOTE: change each control to list of dictionaries + data = [control.convert_by_mode(control_sub_type=chart_settings['sub_type'], mode=chart_settings['mode'], + consolidate=consolidate) for + control in controls] + # NOTE: flatten data to one dimensional list + data = [item for sublist in data for item in sublist] + # logger.debug(f"Control objects going into df conversion: {pformat(data)}") + if not data: + report.add_result(Result(status="Critical", msg="No data found for controls in given date range.")) + return report, None + df = cls.convert_data_list_to_df(input_df=data, sub_mode=chart_settings['sub_mode']) + # logger.debug(f"Chart df: \n {df}") + if chart_settings['sub_mode'] is None: + title = chart_settings['sub_mode'] + else: + title = f"{chart_settings['mode']} - {chart_settings['sub_mode']}" + # NOTE: send dataframe to chart maker + df, modes = cls.prep_df(ctx=ctx, df=df) + # logger.debug(f"prepped df: \n {df}") + # assert modes + # logger.debug(f"modes: {modes}") + fig = IridaFigure(df=df, ytitle=title, modes=modes, parent=parent, + months=chart_settings['months']) + return report, fig + + @classmethod + def convert_data_list_to_df(cls, input_df: list[dict], sub_mode) -> DataFrame: + """ + Convert list of control records to dataframe + + Args: + ctx (dict): settings passed from gui + input_df (list[dict]): list of dictionaries containing records + sub_type (str | None, optional): sub_type of submission type. Defaults to None. + + Returns: + DataFrame: dataframe of controls + """ + # logger.debug(f"Subtype: {sub_mode}") + df = DataFrame.from_records(input_df) + # logger.debug(f"DF from records: {df}") + safe = ['name', 'submitted_date', 'genus', 'target'] + for column in df.columns: + if column not in safe: + if sub_mode is not None and column != sub_mode: + continue + else: + safe.append(column) + if "percent" in column: + # count_col = [item for item in df.columns if "count" in item][0] + try: + count_col = next(item for item in df.columns if "count" in item) + except StopIteration: + continue + # NOTE: The actual percentage from kraken was off due to exclusion of NaN, recalculating. + df[column] = 100 * df[count_col] / df.groupby('name')[count_col].transform('sum') + df = df[[c for c in df.columns if c in safe]] + # NOTE: move date of sample submitted on same date as previous ahead one. + df = cls.displace_date(df=df) + # NOTE: ad hoc method to make data labels more accurate. + df = cls.df_column_renamer(df=df) + return df + + @classmethod + def df_column_renamer(cls, df: DataFrame) -> DataFrame: + """ + Ad hoc function I created to clarify some fields + + Args: + df (DataFrame): input dataframe + + Returns: + DataFrame: dataframe with 'clarified' column names + """ + df = df[df.columns.drop(list(df.filter(regex='_hashes')))] + return df.rename(columns={ + "contains_ratio": "contains_shared_hashes_ratio", + "matches_ratio": "matches_shared_hashes_ratio", + "kraken_count": "kraken2_read_count_(top_50)", + "kraken_percent": "kraken2_read_percent_(top_50)" + }) + + @classmethod + def displace_date(cls, df: DataFrame) -> DataFrame: + """ + This function serves to split samples that were submitted on the same date by incrementing dates. + It will shift the date forward by one day if it is the same day as an existing date in a list. + + Args: + df (DataFrame): input dataframe composed of control records + + Returns: + DataFrame: output dataframe with dates incremented. + """ + # logger.debug(f"Unique items: {df['name'].unique()}") + # NOTE: get submitted dates for each control + dict_list = [dict(name=item, date=df[df.name == item].iloc[0]['submitted_date']) for item in + sorted(df['name'].unique())] + previous_dates = set() + for item in dict_list: + df, previous_dates = cls.check_date(df=df, item=item, previous_dates=previous_dates) + return df + + @classmethod + def check_date(cls, df: DataFrame, item: dict, previous_dates: set) -> Tuple[DataFrame, list]: + """ + Checks if an items date is already present in df and adjusts df accordingly + + Args: + df (DataFrame): input dataframe + item (dict): control for checking + previous_dates (list): list of dates found in previous controls + + Returns: + Tuple[DataFrame, list]: Output dataframe and appended list of previous dates + """ + try: + check = item['date'] in previous_dates + except IndexError: + check = False + previous_dates.add(item['date']) + if check: + # logger.debug(f"We found one! Increment date!\n\t{item['date']} to {item['date'] + timedelta(days=1)}") + # NOTE: get df locations where name == item name + mask = df['name'] == item['name'] + # NOTE: increment date in dataframe + df.loc[mask, 'submitted_date'] = df.loc[mask, 'submitted_date'].apply(lambda x: x + timedelta(days=1)) + item['date'] += timedelta(days=1) + passed = False + else: + passed = True + # logger.debug(f"\n\tCurrent date: {item['date']}\n\tPrevious dates:{previous_dates}") + # logger.debug(f"DF: {type(df)}, previous_dates: {type(previous_dates)}") + # NOTE: if run didn't lead to changed date, return values + if passed: + # logger.debug(f"Date check passed, returning.") + return df, previous_dates + # NOTE: if date was changed, rerun with new date + else: + logger.warning(f"Date check failed, running recursion") + df, previous_dates = cls.check_date(df, item, previous_dates) + return df, previous_dates + + @classmethod + def prep_df(cls, ctx: Settings, df: DataFrame) -> Tuple[DataFrame | None, list]: + """ + Constructs figures based on parsed pandas dataframe. + + Args: + ctx (Settings): settings passed down from gui + df (pd.DataFrame): input dataframe + ytitle (str | None, optional): title for the y-axis. Defaults to None. + + Returns: + Figure: Plotly figure + """ + # NOTE: converts starred genera to normal and splits off list of starred + if df.empty: + return None, [] + df['genus'] = df['genus'].replace({'\*': ''}, regex=True).replace({"NaN": "Unknown"}) + df['genera'] = [item[-1] if item and item[-1] == "*" else "" for item in df['genus'].to_list()] + # NOTE: remove original runs, using reruns if applicable + df = cls.drop_reruns_from_df(ctx=ctx, df=df) + # NOTE: sort by and exclude from + sorts = ['submitted_date', "target", "genus"] + exclude = ['name', 'genera'] + # logger.debug(df.columns) + modes = [item for item in df.columns if item not in sorts and item not in exclude] + # logger.debug(f"Modes coming out: {modes}") + # NOTE: Set descending for any columns that have "{mode}" in the header. + ascending = [False if item == "target" else True for item in sorts] + df = df.sort_values(by=sorts, ascending=ascending) + # logger.debug(df[df.isna().any(axis=1)]) + # NOTE: actual chart construction is done by + return df, modes + + @classmethod + def drop_reruns_from_df(cls, ctx: Settings, df: DataFrame) -> DataFrame: + """ + Removes semi-duplicates from dataframe after finding sequencing repeats. + + Args: + ctx (Settings): settings passed from gui + df (DataFrame): initial dataframe + + Returns: + DataFrame: dataframe with originals removed in favour of repeats. + """ + if 'rerun_regex' in ctx: + sample_names = get_unique_values_in_df_column(df, column_name="name") + rerun_regex = re.compile(fr"{ctx.rerun_regex}") + exclude = [re.sub(rerun_regex, "", sample) for sample in sample_names if rerun_regex.search(sample)] + df = df[df.name not in exclude] + return df diff --git a/src/submissions/backend/db/models/submissions.py b/src/submissions/backend/db/models/submissions.py index bf75453..e3cf596 100644 --- a/src/submissions/backend/db/models/submissions.py +++ b/src/submissions/backend/db/models/submissions.py @@ -13,7 +13,7 @@ from tempfile import TemporaryDirectory, TemporaryFile from operator import itemgetter from pprint import pformat from . import BaseClass, Reagent, SubmissionType, KitType, Organization, Contact -from sqlalchemy import Column, String, TIMESTAMP, INTEGER, ForeignKey, JSON, FLOAT, case, desc +from sqlalchemy import Column, String, TIMESTAMP, INTEGER, ForeignKey, JSON, FLOAT, case from sqlalchemy.orm import relationship, validates, Query from sqlalchemy.orm.attributes import flag_modified from sqlalchemy.ext.associationproxy import association_proxy @@ -22,7 +22,6 @@ from sqlalchemy.exc import OperationalError as AlcOperationalError, IntegrityErr from sqlite3 import OperationalError as SQLOperationalError, IntegrityError as SQLIntegrityError import pandas as pd from openpyxl import Workbook -from openpyxl.worksheet.worksheet import Worksheet from openpyxl.drawing.image import Image as OpenpyxlImage from tools import row_map, setup_lookup, jinja_template_loading, rreplace, row_keys, check_key_or_attr, Result, Report, \ report_result @@ -32,8 +31,6 @@ from dateutil.parser import parse from pathlib import Path from jinja2.exceptions import TemplateNotFound from jinja2 import Template -from docxtpl import InlineImage -from docx.shared import Inches from PIL import Image logger = logging.getLogger(f"submissions.{__name__}") @@ -74,6 +71,8 @@ class BasicSubmission(BaseClass): contact_id = Column(INTEGER, ForeignKey("_contact.id", ondelete="SET NULL", name="fk_BS_contact_id")) #: client lab id from _organizations custom = Column(JSON) + controls = relationship("Control", back_populates="submission", + uselist=True) #: A control sample added to submission submission_sample_associations = relationship( "SubmissionSampleAssociation", @@ -114,7 +113,6 @@ class BasicSubmission(BaseClass): # NOTE: Allows for subclassing into ex. BacterialCulture, Wastewater, etc. __mapper_args__ = { "polymorphic_identity": "Basic Submission", - # "polymorphic_on": submission_type_name, "polymorphic_on": case( (submission_type_name == "Wastewater", "Wastewater"), @@ -190,7 +188,7 @@ class BasicSubmission(BaseClass): # NOTE: Singles tells the query which fields to set limit to 1 dicto['singles'] = parent_defs['singles'] # logger.debug(dicto['singles']) - # NOTE: Grab subtype specific info. + # NOTE: Grab mode_sub_type specific info. output = {} for k, v in dicto.items(): if len(args) > 0 and k not in args: @@ -960,7 +958,6 @@ class BasicSubmission(BaseClass): pcr_sample_map = cls.get_submission_type().sample_map['pcr_samples'] # logger.debug(f'sample map: {pcr_sample_map}') main_sheet = xl[pcr_sample_map['main_sheet']] - # samples = [] fields = {k: v for k, v in pcr_sample_map.items() if k not in ['main_sheet', 'start_row']} for row in main_sheet.iter_rows(min_row=pcr_sample_map['start_row']): idx = row[0].row @@ -969,12 +966,11 @@ class BasicSubmission(BaseClass): sheet = xl[v['sheet']] sample[k] = sheet.cell(row=idx, column=v['column']).value yield sample - # samples.append(sample) - # return samples @classmethod - def parse_pcr_controls(cls, xl: Workbook) -> list: + def parse_pcr_controls(cls, xl: Workbook, rsl_plate_num: str) -> list: location_map = cls.get_submission_type().sample_map['pcr_controls'] + submission = cls.query(rsl_plate_num=rsl_plate_num) name_column = 1 for item in location_map: logger.debug(f"Looking for {item['name']}") @@ -983,7 +979,29 @@ class BasicSubmission(BaseClass): for cell in row: if cell.value == item['name']: logger.debug(f"Pulling from row {iii}, column {item['ct_column']}") - yield dict(name=item['name'], ct=worksheet.cell(row=iii, column=item['ct_column']).value) + subtype, target = item['name'].split("-") + ct = worksheet.cell(row=iii, column=item['ct_column']).value + if subtype == "PC": + ctrl = next((assoc.reagent for assoc in submission.submission_reagent_associations + if any(["positive control" in item.name.lower() for item in assoc.reagent.role])), None) + elif subtype == "NC": + ctrl = next((assoc.reagent for assoc in submission.submission_reagent_associations + if any(["molecular grade water" in item.name.lower() for item in assoc.reagent.role])), None) + try: + ct = float(ct) + except ValueError: + ct = 0.0 + if ctrl: + ctrl = ctrl.lot + else: + ctrl = None + yield dict( + name=f"{rsl_plate_num}<{item['name']}>", + ct=ct, + subtype=subtype, + target=target, + reagent_lot=ctrl + ) @classmethod def filename_template(cls) -> str: @@ -996,21 +1014,6 @@ class BasicSubmission(BaseClass): """ return "{{ rsl_plate_num }}" - # @classmethod - # def custom_sample_autofill_row(cls, sample, worksheet: Worksheet) -> int: - # """ - # Updates row information - # - # Args: - # sample (_type_): _description_ - # worksheet (Workbook): _description_ - # - # Returns: - # int: New row number - # """ - # logger.debug(f"Sample from args: {sample}") - # return None - @classmethod def adjust_autofill_samples(cls, samples: List[Any]) -> List[Any]: """ @@ -1025,19 +1028,6 @@ class BasicSubmission(BaseClass): logger.info(f"Hello from {cls.__mapper_args__['polymorphic_identity']} sampler") return samples - # def adjust_to_dict_samples(self, backup: bool = False) -> List[dict]: - # """ - # Updates sample dictionaries with custom values - # - # Args: - # backup (bool, optional): Whether to perform backup. Defaults to False. - # - # Returns: - # List[dict]: Updated dictionaries - # """ - # # logger.debug(f"Hello from {self.__class__.__name__} dictionary sample adjuster.") - # return [item.to_sub_dict() for item in self.submission_sample_associations] - @classmethod def get_details_template(cls, base_dict: dict) -> Template: """ @@ -1380,8 +1370,7 @@ class BacterialCulture(BasicSubmission): derivative submission type from BasicSubmission """ id = Column(INTEGER, ForeignKey('_basicsubmission.id'), primary_key=True) - controls = relationship("Control", back_populates="submission", - uselist=True) #: A control sample added to submission + __mapper_args__ = dict(polymorphic_identity="Bacterial Culture", polymorphic_load="inline", inherit_condition=(id == BasicSubmission.id)) @@ -1442,25 +1431,6 @@ class BacterialCulture(BasicSubmission): pos_control_reg.missing = False return pyd - # @classmethod - # def custom_sample_autofill_row(cls, sample, worksheet: Worksheet) -> int: - # """ - # Extends parent - # """ - # # logger.debug(f"Checking {sample.well}") - # # logger.debug(f"here's the worksheet: {worksheet}") - # row = super().custom_sample_autofill_row(sample, worksheet) - # df = pd.DataFrame(list(worksheet.values)) - # # logger.debug(f"Here's the dataframe: {df}") - # idx = df[df[0] == sample.well] - # if idx.empty: - # new = f"{sample.well[0]}{sample.well[1:].zfill(2)}" - # # logger.debug(f"Checking: {new}") - # idx = df[df[0] == new] - # # logger.debug(f"Here is the row: {idx}") - # row = idx.index.to_list()[0] - # return row + 1 - @classmethod def custom_info_parser(cls, input_dict: dict, xl: Workbook | None = None, custom_fields: dict = {}) -> dict: input_dict = super().custom_info_parser(input_dict=input_dict, xl=xl, custom_fields=custom_fields) @@ -1548,7 +1518,7 @@ class Wastewater(BasicSubmission): for sample in samples: # NOTE: remove '-{target}' from controls sample['sample'] = re.sub('-N\\d$', '', sample['sample']) - # # NOTE: if sample is already in output skip + # NOTE: if sample is already in output skip if sample['sample'] in [item['sample'] for item in output]: logger.warning(f"Already have {sample['sample']}") continue @@ -1577,8 +1547,6 @@ class Wastewater(BasicSubmission): # @classmethod # def parse_pcr_controls(cls, xl: Workbook, location_map: list) -> list: - - @classmethod def enforce_name(cls, instr: str, data: dict | None = {}) -> str: """ @@ -1681,15 +1649,17 @@ class Wastewater(BasicSubmission): obj (_type_): Parent widget """ from backend.excel import PCRParser + from backend.db import PCRControl, ControlType from frontend.widgets import select_open_file report = Report() fname = select_open_file(obj=obj, file_extension="xlsx") if not fname: report.add_result(Result(msg="No file selected, cancelling.", status="Warning")) return report - parser = PCRParser(filepath=fname) + parser = PCRParser(filepath=fname, submission=self) self.set_attribute("pcr_info", parser.pcr) pcr_samples = [sample for sample in parser.samples] + pcr_controls = [control for control in parser.controls] self.save(original=False) # logger.debug(f"Got {len(parser.samples)} samples to update!") # logger.debug(f"Parser samples: {parser.samples}") @@ -1700,6 +1670,16 @@ class Wastewater(BasicSubmission): except StopIteration: continue self.update_subsampassoc(sample=sample, input_dict=sample_dict) + controltype = ControlType.query(name="PCR Control") + logger.debug(parser.pcr) + submitted_date = datetime.strptime(" ".join(parser.pcr['run_start_date/time'].split(" ")[:-1]), + "%Y-%m-%d %I:%M:%S %p") + for control in pcr_controls: + new_control = PCRControl(**control) + new_control.submitted_date = submitted_date + new_control.controltype = controltype + new_control.submission = self + new_control.save() class WastewaterArtic(BasicSubmission): @@ -2207,7 +2187,7 @@ class BasicSample(BaseClass): id = Column(INTEGER, primary_key=True) #: primary key submitter_id = Column(String(64), nullable=False, unique=True) #: identification from submitter - sample_type = Column(String(32)) #: subtype of sample + sample_type = Column(String(32)) #: mode_sub_type of sample sample_submission_associations = relationship( "SubmissionSampleAssociation", @@ -2632,7 +2612,7 @@ class BacterialCultureSample(BasicSample): id = Column(INTEGER, ForeignKey('_basicsample.id'), primary_key=True) organism = Column(String(64)) #: bacterial specimen concentration = Column(String(16)) #: sample concentration - control = relationship("Control", back_populates="sample", uselist=False) + control = relationship("IridaControl", back_populates="sample", uselist=False) __mapper_args__ = dict(polymorphic_identity="Bacterial Culture Sample", polymorphic_load="inline", inherit_condition=(id == BasicSample.id)) @@ -2677,7 +2657,7 @@ class SubmissionSampleAssociation(BaseClass): # reference to the Sample object sample = relationship(BasicSample, back_populates="sample_submission_associations") #: associated sample - base_sub_type = Column(String) #: string of subtype name + base_sub_type = Column(String) #: string of mode_sub_type name # Refers to the type of parent. # Hooooooo boy, polymorphic association type, now we're getting into the weeds! diff --git a/src/submissions/backend/excel/parser.py b/src/submissions/backend/excel/parser.py index 5b21cb9..3ae8362 100644 --- a/src/submissions/backend/excel/parser.py +++ b/src/submissions/backend/excel/parser.py @@ -675,7 +675,7 @@ class PCRParser(object): rsl_plate_num = self.submission_obj.rsl_plate_num self.pcr = self.parse_general() self.samples = self.submission_obj.parse_pcr(xl=self.xl, rsl_plate_num=rsl_plate_num) - self.controls = self.submission_obj.parse_pcr_controls(xl=self.xl) + self.controls = self.submission_obj.parse_pcr_controls(xl=self.xl, rsl_plate_num=rsl_plate_num) def parse_general(self): """ diff --git a/src/submissions/frontend/visualizations/__init__.py b/src/submissions/frontend/visualizations/__init__.py index 157c0c6..56d54df 100644 --- a/src/submissions/frontend/visualizations/__init__.py +++ b/src/submissions/frontend/visualizations/__init__.py @@ -1,4 +1,52 @@ ''' Contains all operations for creating charts, graphs and visual effects. ''' -from .control_charts import * +from PyQt6.QtWidgets import QWidget +import plotly +from plotly.graph_objects import Figure +import pandas as pd +from frontend.widgets.functions import select_save_file + + +class CustomFigure(Figure): + + def __init__(self, df: pd.DataFrame, modes: list, ytitle: str | None = None, parent: QWidget | None = None, + months: int = 6): + super().__init__() + + def save_figure(self, group_name: str = "plotly_output", parent: QWidget | None = None): + """ + Writes plotly figure to html file. + + Args: + figs (): + settings (dict): settings passed down from click + fig (Figure): input figure object + group_name (str): controltype + """ + + output = select_save_file(obj=parent, default_name=group_name, extension="png") + self.write_image(output.absolute().__str__(), engine="kaleido") + + def to_html(self) -> str: + """ + Creates final html code from plotly + + Args: + figure (Figure): input figure + + Returns: + str: html string + """ + html = '' + if self is not None: + html += plotly.offline.plot(self, output_type='div', + include_plotlyjs='cdn') #, image = 'png', auto_open=True, image_filename='plot_image') + else: + html += "

No data was retrieved for the given parameters.

" + html += '' + return html + + +from .irida_charts import IridaFigure +from .pcr_charts import PCRFigure diff --git a/src/submissions/frontend/visualizations/control_charts.py b/src/submissions/frontend/visualizations/irida_charts.py similarity index 73% rename from src/submissions/frontend/visualizations/control_charts.py rename to src/submissions/frontend/visualizations/irida_charts.py index 5d5084a..16498d3 100644 --- a/src/submissions/frontend/visualizations/control_charts.py +++ b/src/submissions/frontend/visualizations/irida_charts.py @@ -1,15 +1,13 @@ """ -Functions for constructing controls graphs using plotly. +Functions for constructing irida controls graphs using plotly. """ -from copy import deepcopy from datetime import date from pprint import pformat - import plotly import plotly.express as px import pandas as pd from PyQt6.QtWidgets import QWidget -from plotly.graph_objects import Figure +from . import CustomFigure import logging from tools import get_unique_values_in_df_column, divide_chunks from frontend.widgets.functions import select_save_file @@ -17,11 +15,12 @@ from frontend.widgets.functions import select_save_file logger = logging.getLogger(f"submissions.{__name__}") -class CustomFigure(Figure): +class IridaFigure(CustomFigure): def __init__(self, df: pd.DataFrame, modes: list, ytitle: str | None = None, parent: QWidget | None = None, months: int = 6): - super().__init__() + + super().__init__(df=df, modes=modes) self.construct_chart(df=df, modes=modes) self.generic_figure_markers(modes=modes, ytitle=ytitle, months=months) @@ -105,25 +104,16 @@ class CustomFigure(Figure): self.update_xaxes( rangeslider_visible=True, rangeselector=dict( - # buttons=list([ - # dict(count=1, label="1m", step="month", stepmode="backward"), - # dict(count=3, label="3m", step="month", stepmode="backward"), - # dict(count=6, label="6m", step="month", stepmode="backward"), - # dict(count=1, label="YTD", step="year", stepmode="todate"), - # dict(count=12, label="1y", step="month", stepmode="backward"), - # dict(step="all") - # ]) buttons=[button for button in self.make_plotly_buttons(months=months)] ) ) - assert isinstance(self, Figure) - # return fig + assert isinstance(self, CustomFigure) - def make_plotly_buttons(self, months:int=6): + def make_plotly_buttons(self, months: int = 6): rng = [1] if months > 2: rng += [iii for iii in range(3, months, 3)] - logger.debug(f"Making buttons for months: {rng}") + # logger.debug(f"Making buttons for months: {rng}") buttons = [dict(count=iii, label=f"{iii}m", step="month", stepmode="backward") for iii in rng] if months > date.today().month: buttons += [dict(count=1, label="YTD", step="year", stepmode="todate")] @@ -161,35 +151,3 @@ class CustomFigure(Figure): {"yaxis.title.text": mode}, ]) - def save_figure(self, group_name: str = "plotly_output", parent: QWidget | None = None): - """ - Writes plotly figure to html file. - - Args: - figs (): - settings (dict): settings passed down from click - fig (Figure): input figure object - group_name (str): controltype - """ - - output = select_save_file(obj=parent, default_name=group_name, extension="png") - self.write_image(output.absolute().__str__(), engine="kaleido") - - def to_html(self) -> str: - """ - Creates final html code from plotly - - Args: - figure (Figure): input figure - - Returns: - str: html string - """ - html = '' - if self is not None: - html += plotly.offline.plot(self, output_type='div', - include_plotlyjs='cdn') #, image = 'png', auto_open=True, image_filename='plot_image') - else: - html += "

No data was retrieved for the given parameters.

" - html += '' - return html diff --git a/src/submissions/frontend/visualizations/pcr_charts.py b/src/submissions/frontend/visualizations/pcr_charts.py new file mode 100644 index 0000000..77bc024 --- /dev/null +++ b/src/submissions/frontend/visualizations/pcr_charts.py @@ -0,0 +1,34 @@ +""" +Functions for constructing irida controls graphs using plotly. +""" +from datetime import date +from pprint import pformat +import plotly +from . import CustomFigure +import plotly.express as px +import pandas as pd +from PyQt6.QtWidgets import QWidget +from plotly.graph_objects import Figure +import logging +from tools import get_unique_values_in_df_column, divide_chunks +from frontend.widgets.functions import select_save_file + +logger = logging.getLogger(f"submissions.{__name__}") + + +class PCRFigure(CustomFigure): + + def __init__(self, df: pd.DataFrame, modes: list, ytitle: str | None = None, parent: QWidget | None = None, + months: int = 6): + super().__init__(df=df, modes=modes) + self.construct_chart(df=df) + # self.generic_figure_markers(modes=modes, ytitle=ytitle, months=months) + + def construct_chart(self, df: pd.DataFrame): + logger.debug(f"PCR df: {df}") + try: + scatter = px.scatter(data_frame=df, x='submitted_date', y="ct", hover_data=["name", "target", "ct", "reagent_lot"], color='target') + except ValueError: + scatter = px.scatter() + self.add_traces(scatter.data) + diff --git a/src/submissions/frontend/widgets/app.py b/src/submissions/frontend/widgets/app.py index fd4fd16..0dcd30f 100644 --- a/src/submissions/frontend/widgets/app.py +++ b/src/submissions/frontend/widgets/app.py @@ -4,7 +4,7 @@ Constructs main application. from pprint import pformat from PyQt6.QtWidgets import ( QTabWidget, QWidget, QVBoxLayout, - QHBoxLayout, QScrollArea, QMainWindow, + QHBoxLayout, QScrollArea, QMainWindow, QToolBar ) from PyQt6.QtGui import QAction @@ -13,7 +13,7 @@ from pathlib import Path from markdown import markdown from __init__ import project_path from tools import check_if_app, Settings, Report, jinja_template_loading, check_authorization, page_size -from .functions import select_save_file,select_open_file +from .functions import select_save_file, select_open_file from datetime import date from .pop_ups import HTMLPop, AlertPop from .misc import LogParser, Pagifier @@ -29,6 +29,7 @@ from .summary import Summary logger = logging.getLogger(f'submissions.{__name__}') logger.info("Hello, I am a logger") + class App(QMainWindow): def __init__(self, ctx: Settings = None): @@ -61,11 +62,11 @@ class App(QMainWindow): self.show() self.statusBar().showMessage('Ready', 5000) self.backup_database() - + def _createMenuBar(self): """ adds items to menu bar - """ + """ # logger.debug(f"Creating menu bar...") menuBar = self.menuBar() fileMenu = menuBar.addMenu("&File") @@ -85,11 +86,11 @@ class App(QMainWindow): # reportMenu.addAction(self.generateReportAction) maintenanceMenu.addAction(self.joinExtractionAction) maintenanceMenu.addAction(self.joinPCRAction) - + def _createToolBar(self): """ adds items to toolbar - """ + """ # logger.debug(f"Creating toolbar...") toolbar = QToolBar("My main toolbar") self.addToolBar(toolbar) @@ -100,7 +101,7 @@ class App(QMainWindow): def _createActions(self): """ creates actions - """ + """ # logger.debug(f"Creating actions...") self.importAction = QAction("&Import Submission", self) self.addReagentAction = QAction("Add Reagent", self) @@ -139,7 +140,7 @@ class App(QMainWindow): def showAbout(self): """ Show the 'about' message - """ + """ j_env = jinja_template_loading() template = j_env.get_template("project.html") html = template.render(info=self.ctx.package.__dict__) @@ -150,7 +151,7 @@ class App(QMainWindow): def openDocs(self): """ Open the documentation html pages - """ + """ if check_if_app(): url = Path(sys._MEIPASS).joinpath("files", "docs", "index.html") else: @@ -182,14 +183,14 @@ class App(QMainWindow): def runSampleSearch(self): """ Create a search for samples. - """ + """ dlg = SearchBox(self) dlg.exec() def backup_database(self): """ Copies the database into the backup directory the first time it is opened every month. - """ + """ month = date.today().strftime("%Y-%m") current_month_bak = Path(self.ctx.backup_path).joinpath(f"submissions_backup-{month}").resolve() logger.info(f"Here is the db directory: {self.ctx.database_path}") @@ -244,8 +245,8 @@ class App(QMainWindow): class AddSubForm(QWidget): - - def __init__(self, parent:QWidget): + + def __init__(self, parent: QWidget): # logger.debug(f"Initializating subform...") super(QWidget, self).__init__(parent) self.layout = QVBoxLayout(self) @@ -255,11 +256,12 @@ class AddSubForm(QWidget): self.tab2 = QWidget() self.tab3 = QWidget() self.tab4 = QWidget() - self.tabs.resize(300,200) + self.tabs.resize(300, 200) # NOTE: Add tabs - self.tabs.addTab(self.tab1,"Submissions") - self.tabs.addTab(self.tab2,"Controls") - self.tabs.addTab(self.tab3, "Summary Report") + self.tabs.addTab(self.tab1, "Submissions") + self.tabs.addTab(self.tab2, "Irida Controls") + self.tabs.addTab(self.tab3, "PCR Controls") + self.tabs.addTab(self.tab4, "Cost Report") # self.tabs.addTab(self.tab4, "Add Kit") # NOTE: Create submission adder form self.formwidget = SubmissionFormContainer(self) @@ -276,7 +278,7 @@ class AddSubForm(QWidget): self.sheetlayout = QVBoxLayout(self) self.sheetwidget.setLayout(self.sheetlayout) self.sub_wid = SubmissionsSheet(parent=parent) - self.pager = Pagifier(page_max=self.sub_wid.total_count/page_size) + self.pager = Pagifier(page_max=self.sub_wid.total_count / page_size) self.sheetlayout.addWidget(self.sub_wid) self.sheetlayout.addWidget(self.pager) # NOTE: Create layout of first tab to hold form and sheet @@ -285,15 +287,19 @@ class AddSubForm(QWidget): self.tab1.layout.addWidget(self.interior) self.tab1.layout.addWidget(self.sheetwidget) self.tab2.layout = QVBoxLayout(self) - self.controls_viewer = ControlsViewer(self) - self.tab2.layout.addWidget(self.controls_viewer) + self.irida_viewer = ControlsViewer(self, archetype="Irida Control") + self.tab2.layout.addWidget(self.irida_viewer) self.tab2.setLayout(self.tab2.layout) + self.tab3.layout = QVBoxLayout(self) + self.pcr_viewer = ControlsViewer(self, archetype="PCR Control") + self.tab3.layout.addWidget(self.pcr_viewer) + self.tab3.setLayout(self.tab3.layout) # NOTE: create custom widget to add new tabs # ST_adder = SubmissionTypeAdder(self) summary_report = Summary(self) - self.tab3.layout = QVBoxLayout(self) - self.tab3.layout.addWidget(summary_report) - self.tab3.setLayout(self.tab3.layout) + self.tab4.layout = QVBoxLayout(self) + self.tab4.layout.addWidget(summary_report) + self.tab4.setLayout(self.tab4.layout) # kit_adder = KitAdder(self) # self.tab4.layout = QVBoxLayout(self) # self.tab4.layout.addWidget(kit_adder) diff --git a/src/submissions/frontend/widgets/controls_chart.py b/src/submissions/frontend/widgets/controls_chart.py index 1e220b6..a39f15f 100644 --- a/src/submissions/frontend/widgets/controls_chart.py +++ b/src/submissions/frontend/widgets/controls_chart.py @@ -4,6 +4,7 @@ Handles display of control charts import re import sys from datetime import timedelta, date +from pprint import pformat from typing import Tuple from PyQt6.QtWebEngineWidgets import QWebEngineView from PyQt6.QtWidgets import ( @@ -11,20 +12,26 @@ from PyQt6.QtWidgets import ( QDateEdit, QLabel, QSizePolicy, QPushButton, QGridLayout ) from PyQt6.QtCore import QSignalBlocker -from backend.db import ControlType, Control +from backend.db import ControlType, IridaControl from PyQt6.QtCore import QDate, QSize import logging from pandas import DataFrame from tools import Report, Result, get_unique_values_in_df_column, Settings, report_result -from frontend.visualizations.control_charts import CustomFigure +from frontend.visualizations import IridaFigure, PCRFigure from .misc import StartEndDatePicker + logger = logging.getLogger(f"submissions.{__name__}") class ControlsViewer(QWidget): - def __init__(self, parent: QWidget) -> None: + def __init__(self, parent: QWidget, archetype: str) -> None: super().__init__(parent) + logger.debug(f"Incoming Archetype: {archetype}") + self.archetype = ControlType.query(name=archetype) + if not self.archetype: + return + logger.debug(f"Archetype set as: {self.archetype}") self.app = self.parent().parent() # logger.debug(f"\n\n{self.app}\n\n") self.report = Report() @@ -32,51 +39,53 @@ class ControlsViewer(QWidget): self.webengineview = QWebEngineView() # NOTE: set tab2 layout self.layout = QGridLayout(self) - self.control_typer = QComboBox() + self.control_sub_typer = QComboBox() # NOTE: fetch types of controls - con_types = [item.name for item in ControlType.query()] - self.control_typer.addItems(con_types) + con_sub_types = [item for item in self.archetype.targets.keys()] + self.control_sub_typer.addItems(con_sub_types) # NOTE: create custom widget to get types of analysis self.mode_typer = QComboBox() - mode_types = Control.get_modes() + mode_types = IridaControl.get_modes() self.mode_typer.addItems(mode_types) # NOTE: create custom widget to get subtypes of analysis - self.sub_typer = QComboBox() - self.sub_typer.setEnabled(False) + self.mode_sub_typer = QComboBox() + self.mode_sub_typer.setEnabled(False) # NOTE: add widgets to tab2 layout - self.layout.addWidget(self.datepicker, 0,0,1,2) + self.layout.addWidget(self.datepicker, 0, 0, 1, 2) self.save_button = QPushButton("Save Chart", parent=self) - self.layout.addWidget(self.save_button, 0,2,1,1) - self.layout.addWidget(self.control_typer, 1,0,1,3) - self.layout.addWidget(self.mode_typer, 2,0,1,3) - self.layout.addWidget(self.sub_typer, 3,0,1,3) - self.layout.addWidget(self.webengineview, 4,0,1,3) + self.layout.addWidget(self.save_button, 0, 2, 1, 1) + self.layout.addWidget(self.control_sub_typer, 1, 0, 1, 3) + self.layout.addWidget(self.mode_typer, 2, 0, 1, 3) + self.layout.addWidget(self.mode_sub_typer, 3, 0, 1, 3) + self.archetype.get_instance_class().make_parent_buttons(parent=self) + self.layout.addWidget(self.webengineview, self.layout.rowCount(), 0, 1, 3) self.setLayout(self.layout) - self.controls_getter() - self.control_typer.currentIndexChanged.connect(self.controls_getter) - self.mode_typer.currentIndexChanged.connect(self.controls_getter) - self.datepicker.start_date.dateChanged.connect(self.controls_getter) - self.datepicker.end_date.dateChanged.connect(self.controls_getter) + self.controls_getter_function() + self.control_sub_typer.currentIndexChanged.connect(self.controls_getter_function) + self.mode_typer.currentIndexChanged.connect(self.controls_getter_function) + self.datepicker.start_date.dateChanged.connect(self.controls_getter_function) + self.datepicker.end_date.dateChanged.connect(self.controls_getter_function) self.save_button.pressed.connect(self.save_chart_function) + def save_chart_function(self): self.fig.save_figure(parent=self) - def controls_getter(self): - """ - Lookup controls from database and send to chartmaker - """ - self.controls_getter_function() + # def controls_getter(self): + # """ + # Lookup controls from database and send to chartmaker + # """ + # self.controls_getter_function() @report_result - def controls_getter_function(self): + def controls_getter_function(self, *args, **kwargs): """ Get controls based on start/end dates """ report = Report() - # NOTE: subtype defaults to disabled + # NOTE: mode_sub_type defaults to disabled try: - self.sub_typer.disconnect() + self.mode_sub_typer.disconnect() except TypeError: pass # NOTE: correct start date being more recent than end date and rerun @@ -93,37 +102,32 @@ class ControlsViewer(QWidget): # NOTE: convert to python useable date objects self.start_date = self.datepicker.start_date.date().toPyDate() self.end_date = self.datepicker.end_date.date().toPyDate() - self.con_type = self.control_typer.currentText() + self.con_sub_type = self.control_sub_typer.currentText() self.mode = self.mode_typer.currentText() - self.sub_typer.clear() + self.mode_sub_typer.clear() # NOTE: lookup subtypes try: - sub_types = ControlType.query(name=self.con_type).get_subtypes(mode=self.mode) + sub_types = self.archetype.get_modes(mode=self.mode) except AttributeError: sub_types = [] - if sub_types != []: - # NOTE: block signal that will rerun controls getter and update sub_typer - with QSignalBlocker(self.sub_typer) as blocker: - self.sub_typer.addItems(sub_types) - self.sub_typer.setEnabled(True) - self.sub_typer.currentTextChanged.connect(self.chart_maker) + if sub_types: + # NOTE: block signal that will rerun controls getter and update mode_sub_typer + with QSignalBlocker(self.mode_sub_typer) as blocker: + self.mode_sub_typer.addItems(sub_types) + self.mode_sub_typer.setEnabled(True) + self.mode_sub_typer.currentTextChanged.connect(self.chart_maker_function) else: - self.sub_typer.clear() - self.sub_typer.setEnabled(False) - self.chart_maker() + self.mode_sub_typer.clear() + self.mode_sub_typer.setEnabled(False) + self.chart_maker_function() return report - def diff_month(self, d1:date, d2:date): + def diff_month(self, d1: date, d2: date): return abs((d1.year - d2.year) * 12 + d1.month - d2.month) - def chart_maker(self): - """ - Creates plotly charts for webview - """ - self.chart_maker_function() - @report_result - def chart_maker_function(self): + def chart_maker_function(self, *args, **kwargs): + # TODO: Generalize this by moving as much code as possible to IridaControl """ Create html chart for controls reporting @@ -134,44 +138,26 @@ class ControlsViewer(QWidget): Tuple[QMainWindow, dict]: Collection of new main app window and result dict """ report = Report() - # logger.debug(f"Control getter context: \n\tControl type: {self.con_type}\n\tMode: {self.mode}\n\tStart - # Date: {self.start_date}\n\tEnd Date: {self.end_date}") - # NOTE: set the subtype for kraken - if self.sub_typer.currentText() == "": - self.subtype = None + # logger.debug(f"Control getter context: \n\tControl type: {self.con_sub_type}\n\tMode: {self.mode}\n\tStart \ + # Date: {self.start_date}\n\tEnd Date: {self.end_date}") + # NOTE: set the mode_sub_type for kraken + if self.mode_sub_typer.currentText() == "": + self.mode_sub_type = None else: - self.subtype = self.sub_typer.currentText() - # logger.debug(f"Subtype: {self.subtype}") + self.mode_sub_type = self.mode_sub_typer.currentText() + logger.debug(f"Subtype: {self.mode_sub_type}") + months = self.diff_month(self.start_date, self.end_date) # NOTE: query all controls using the type/start and end dates from the gui - controls = Control.query(control_type=self.con_type, start_date=self.start_date, end_date=self.end_date) - # NOTE: if no data found from query set fig to none for reporting in webview - if controls is None: - fig = None - self.save_button.setEnabled(False) - else: - # NOTE: change each control to list of dictionaries - data = [control.convert_by_mode(mode=self.mode) for control in controls] - # NOTE: flatten data to one dimensional list - data = [item for sublist in data for item in sublist] - # logger.debug(f"Control objects going into df conversion: {type(data)}") - if not data: - report.add_result(Result(status="Critical", msg="No data found for controls in given date range.")) - return - # NOTE send to dataframe creator - df = self.convert_data_list_to_df(input_df=data) - if self.subtype is None: - title = self.mode - else: - title = f"{self.mode} - {self.subtype}" - # NOTE: send dataframe to chart maker - df, modes = self.prep_df(ctx=self.app.ctx, df=df) - months = self.diff_month(self.start_date, self.end_date) - fig = CustomFigure(df=df, ytitle=title, modes=modes, parent=self, months=months) - self.save_button.setEnabled(True) + chart_settings = dict(sub_type=self.con_sub_type, start_date=self.start_date, end_date=self.end_date, + mode=self.mode, + sub_mode=self.mode_sub_type, parent=self, months=months) + _, self.fig = self.archetype.get_instance_class().make_chart(chart_settings=chart_settings, parent=self, ctx=self.app.ctx) + # if isinstance(self.fig, IridaFigure): + # self.save_button.setEnabled(True) # logger.debug(f"Updating figure...") - self.fig = fig + # self.fig = fig # NOTE: construct html for webview - html = fig.to_html() + html = self.fig.to_html() # logger.debug(f"The length of html code is: {len(html)}") self.webengineview.setHtml(html) self.webengineview.update() @@ -185,7 +171,7 @@ class ControlsViewer(QWidget): Args: ctx (dict): settings passed from gui input_df (list[dict]): list of dictionaries containing records - subtype (str | None, optional): sub_type of submission type. Defaults to None. + mode_sub_type (str | None, optional): sub_type of submission type. Defaults to None. Returns: DataFrame: dataframe of controls @@ -195,7 +181,7 @@ class ControlsViewer(QWidget): safe = ['name', 'submitted_date', 'genus', 'target'] for column in df.columns: if column not in safe: - if self.subtype is not None and column != self.subtype: + if self.mode_sub_type is not None and column != self.mode_sub_type: continue else: safe.append(column) @@ -338,34 +324,4 @@ class ControlsViewer(QWidget): rerun_regex = re.compile(fr"{ctx.rerun_regex}") exclude = [re.sub(rerun_regex, "", sample) for sample in sample_names if rerun_regex.search(sample)] df = df[df.name not in exclude] - # for sample in sample_names: - # if rerun_regex.search(sample): - # first_run = re.sub(rerun_regex, "", sample) - # df = df.drop(df[df.name == first_run].index) return df - - -# class ControlsDatePicker(QWidget): -# """ -# custom widget to pick start and end dates for controls graphs -# """ -# -# def __init__(self) -> None: -# super().__init__() -# self.start_date = QDateEdit(calendarPopup=True) -# # NOTE: start date is two months prior to end date by default -# sixmonthsago = QDate.currentDate().addDays(-180) -# self.start_date.setDate(sixmonthsago) -# self.end_date = QDateEdit(calendarPopup=True) -# self.end_date.setDate(QDate.currentDate()) -# self.layout = QHBoxLayout() -# self.layout.addWidget(QLabel("Start Date")) -# self.layout.addWidget(self.start_date) -# self.layout.addWidget(QLabel("End Date")) -# self.layout.addWidget(self.end_date) -# self.setLayout(self.layout) -# self.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed) -# -# -# def sizeHint(self) -> QSize: -# return QSize(80, 20) diff --git a/src/submissions/frontend/widgets/submission_details.py b/src/submissions/frontend/widgets/submission_details.py index 554866a..903f89f 100644 --- a/src/submissions/frontend/widgets/submission_details.py +++ b/src/submissions/frontend/widgets/submission_details.py @@ -7,7 +7,7 @@ from PyQt6.QtWidgets import (QDialog, QPushButton, QVBoxLayout, QDialogButtonBox, QTextEdit, QGridLayout) from PyQt6.QtWebEngineWidgets import QWebEngineView from PyQt6.QtWebChannel import QWebChannel -from PyQt6.QtCore import Qt, pyqtSlot, QMarginsF +from PyQt6.QtCore import Qt, pyqtSlot, QMarginsF, QSize from jinja2 import TemplateNotFound from backend.db.models import BasicSubmission, BasicSample, Reagent, KitType from tools import is_power_user, jinja_template_loading @@ -38,7 +38,7 @@ class SubmissionDetails(QDialog): self.app = None self.webview = QWebEngineView(parent=self) self.webview.setMinimumSize(900, 500) - self.webview.setMaximumSize(900, 700) + self.webview.setMaximumWidth(900) self.webview.loadFinished.connect(self.activate_export) self.layout = QGridLayout() # self.setFixedSize(900, 500) @@ -98,9 +98,6 @@ class SubmissionDetails(QDialog): sample = BasicSample.query(submitter_id=sample) base_dict = sample.to_sub_dict(full_data=True) exclude = ['submissions', 'excluded', 'colour', 'tooltip'] - # try: - # base_dict['excluded'] += exclude - # except KeyError: base_dict['excluded'] = exclude template = sample.get_details_template() template_path = Path(self.template.environment.loader.__getattribute__("searchpath")[0]) @@ -131,7 +128,6 @@ class SubmissionDetails(QDialog): html = template.render(reagent=base_dict, css=css) self.webview.setHtml(html) self.setWindowTitle(f"Reagent Details - {reagent.name} - {reagent.lot}") - # self.btn.setEnabled(False) @pyqtSlot(str) def submission_details(self, submission: str | BasicSubmission): @@ -160,8 +156,6 @@ class SubmissionDetails(QDialog): # logger.debug(f"Submission_details: {pformat(self.base_dict)}") # logger.debug(f"User is power user: {is_power_user()}") self.html = self.template.render(sub=self.base_dict, signing_permission=is_power_user(), css=css) - # with open("test.html", "w") as f: - # f.write(self.html) self.webview.setHtml(self.html) @pyqtSlot(str) @@ -178,11 +172,6 @@ class SubmissionDetails(QDialog): Renders submission to html, then creates and saves .pdf file to user selected file. """ fname = select_save_file(obj=self, default_name=self.export_plate, extension="pdf") - # page_layout = QPageLayout() - # page_layout.setPageSize(QPageSize(QPageSize.PageSizeId.A4)) - # page_layout.setOrientation(QPageLayout.Orientation.Portrait) - # page_layout.setMargins(QMarginsF(25, 25, 25, 25)) - # self.webview.page().printToPdf(fname.with_suffix(".pdf").__str__(), page_layout) save_pdf(obj=self, filename=fname) class SubmissionComment(QDialog): diff --git a/src/submissions/tools/__init__.py b/src/submissions/tools/__init__.py index 48f52d5..d13a0f6 100644 --- a/src/submissions/tools/__init__.py +++ b/src/submissions/tools/__init__.py @@ -17,6 +17,7 @@ from sqlalchemy import create_engine, text, MetaData from pydantic import field_validator, BaseModel, Field from pydantic_settings import BaseSettings, SettingsConfigDict from typing import Any, Tuple, Literal, List +print(inspect.stack()[1]) from __init__ import project_path from configparser import ConfigParser from tkinter import Tk # NOTE: This is for choosing database path before app is created. @@ -960,6 +961,6 @@ def report_result(func): except Exception as e: logger.error(f"Problem reporting due to {e}") logger.error(result.msg) - logger.debug(f"Returning: {output}") + # logger.debug(f"Returning: {output}") return output return wrapper