Updated controls to both Irida and PCR.

This commit is contained in:
lwark
2024-10-16 15:07:43 -05:00
parent 066d1af0f2
commit c3a4aac68b
11 changed files with 750 additions and 314 deletions

View File

@@ -2,14 +2,19 @@
All control related models.
"""
from __future__ import annotations
from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey
from sqlalchemy.orm import relationship, Query
from pprint import pformat
from PyQt6.QtWidgets import QWidget, QCheckBox, QLabel
from pandas import DataFrame
from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey, case, FLOAT
from sqlalchemy.orm import relationship, Query, validates
import logging, re
from operator import itemgetter
from . import BaseClass
from tools import setup_lookup
from datetime import date, datetime
from typing import List, Literal
from tools import setup_lookup, report_result, Result, Report, Settings, get_unique_values_in_df_column
from datetime import date, datetime, timedelta
from typing import List, Literal, Tuple, Generator
from dateutil.parser import parse
from re import Pattern
@@ -21,7 +26,7 @@ class ControlType(BaseClass):
Base class of a control archetype.
"""
id = Column(INTEGER, primary_key=True) #: primary key
name = Column(String(255), unique=True) #: controltype name (e.g. MCS)
name = Column(String(255), unique=True) #: controltype name (e.g. Irida Control)
targets = Column(JSON) #: organisms checked for
instances = relationship("Control", back_populates="controltype") #: control samples created of this type.
@@ -53,7 +58,7 @@ class ControlType(BaseClass):
pass
return cls.execute_query(query=query, limit=limit)
def get_subtypes(self, mode: Literal['kraken', 'matches', 'contains']) -> List[str]:
def get_modes(self, mode: Literal['kraken', 'matches', 'contains']) -> List[str]:
"""
Get subtypes associated with this controltype (currently used only for Kraken)
@@ -65,8 +70,10 @@ class ControlType(BaseClass):
"""
# NOTE: Get first instance since all should have same subtypes
# NOTE: Get mode of instance
if not self.instances:
return
jsoner = getattr(self.instances[0], mode)
# logger.debug(f"JSON out: {jsoner.keys()}")
# logger.debug(f"JSON retrieved: {jsoner.keys()}")
try:
# NOTE: Pick genera (all should have same subtypes)
genera = list(jsoner.keys())[0]
@@ -74,10 +81,14 @@ class ControlType(BaseClass):
return []
# NOTE: remove items that don't have relevant data
subtypes = [item for item in jsoner[genera] if "_hashes" not in item and "_ratio" not in item]
logger.debug(f"subtypes out: {pformat(subtypes)}")
return subtypes
def get_instance_class(self):
return Control.find_polymorphic_subclass(polymorphic_identity=self.name)
@classmethod
def get_positive_control_types(cls) -> List[ControlType]:
def get_positive_control_types(cls) -> Generator[ControlType, None, None]:
"""
Gets list of Control types if they have targets
@@ -104,35 +115,234 @@ class Control(BaseClass):
"""
id = Column(INTEGER, primary_key=True) #: primary key
parent_id = Column(String,
ForeignKey("_controltype.id", name="fk_control_parent_id")) #: primary key of control type
controltype_name = Column(String, ForeignKey("_controltype.name", ondelete="SET NULL",
name="fk_BC_subtype_name")) #: name of joined submission type
controltype = relationship("ControlType", back_populates="instances",
foreign_keys=[parent_id]) #: reference to parent control type
foreign_keys=[controltype_name]) #: reference to parent control type
name = Column(String(255), unique=True) #: Sample ID
submitted_date = Column(TIMESTAMP) #: Date submitted to Robotics
submission_id = Column(INTEGER, ForeignKey("_basicsubmission.id")) #: parent submission id
submission = relationship("BasicSubmission", back_populates="controls",
foreign_keys=[submission_id]) #: parent submission
__mapper_args__ = {
"polymorphic_identity": "Basic Control",
"polymorphic_on": case(
(controltype_name == "PCR Control", "PCR Control"),
(controltype_name == "Irida Control", "Irida Control"),
else_="Basic Control"
),
"with_polymorphic": "*",
}
def __repr__(self) -> str:
return f"<{self.controltype_name}({self.name})>"
@classmethod
def find_polymorphic_subclass(cls, polymorphic_identity: str | ControlType | None = None,
attrs: dict | None = None):
"""
Find subclass based on polymorphic identity or relevant attributes.
Args:
polymorphic_identity (str | None, optional): String representing polymorphic identity. Defaults to None.
attrs (str | SubmissionType | None, optional): Attributes of the relevant class. Defaults to None.
Returns:
_type_: Subclass of interest.
"""
if isinstance(polymorphic_identity, dict):
# logger.debug(f"Controlling for dict value")
polymorphic_identity = polymorphic_identity['value']
if isinstance(polymorphic_identity, ControlType):
polymorphic_identity = polymorphic_identity.name
model = cls
match polymorphic_identity:
case str():
try:
model = cls.__mapper__.polymorphic_map[polymorphic_identity].class_
except Exception as e:
logger.error(
f"Could not get polymorph {polymorphic_identity} of {cls} due to {e}, falling back to BasicSubmission")
case _:
pass
if attrs and any([not hasattr(cls, attr) for attr in attrs.keys()]):
# NOTE: looks for first model that has all included kwargs
try:
model = next(subclass for subclass in cls.__subclasses__() if
all([hasattr(subclass, attr) for attr in attrs.keys()]))
except StopIteration as e:
raise AttributeError(
f"Couldn't find existing class/subclass of {cls} with all attributes:\n{pformat(attrs.keys())}")
logger.info(f"Recruiting model: {model}")
return model
@classmethod
def make_parent_buttons(cls, parent: QWidget) -> None:
"""
Args:
parent (QWidget): chart holding widget to add buttons to.
Returns:
"""
pass
@classmethod
def make_chart(cls, parent, chart_settings: dict, ctx):
"""
Args:
chart_settings (dict): settings passed down from chart widget
ctx (Settings): settings passed down from gui
Returns:
"""
return None
class PCRControl(Control):
id = Column(INTEGER, ForeignKey('_control.id'), primary_key=True)
subtype = Column(String(16)) #: PC or NC
target = Column(String(16)) #: N1, N2, etc.
ct = Column(FLOAT)
reagent_lot = Column(String(64), ForeignKey("_reagent.name", ondelete="SET NULL",
name="fk_reagent_lot"))
reagent = relationship("Reagent", foreign_keys=reagent_lot)
__mapper_args__ = dict(polymorphic_identity="PCR Control",
polymorphic_load="inline",
inherit_condition=(id == Control.id))
def to_sub_dict(self):
return dict(name=self.name, ct=self.ct, subtype=self.subtype, target=self.target, reagent_lot=self.reagent_lot,
submitted_date=self.submitted_date.date())
@classmethod
@setup_lookup
def query(cls,
sub_type: str | None = None,
start_date: date | str | int | None = None,
end_date: date | str | int | None = None,
control_name: str | None = None,
limit: int = 0
) -> Control | List[Control]:
"""
Lookup control objects in the database based on a number of parameters.
Args:
sub_type (models.ControlType | str | None, optional): Control archetype. Defaults to None.
start_date (date | str | int | None, optional): Beginning date to search by. Defaults to 2023-01-01 if end_date not None.
end_date (date | str | int | None, optional): End date to search by. Defaults to today if start_date not None.
control_name (str | None, optional): Name of control. Defaults to None.
limit (int, optional): Maximum number of results to return (0 = all). Defaults to 0.
Returns:
models.Control|List[models.Control]: Control object of interest.
"""
query: Query = cls.__database_session__.query(cls)
# NOTE: by date range
if start_date is not None and end_date is None:
logger.warning(f"Start date with no end date, using today.")
end_date = date.today()
if end_date is not None and start_date is None:
logger.warning(f"End date with no start date, using Jan 1, 2023")
start_date = date(2023, 1, 1)
if start_date is not None:
match start_date:
case date():
# logger.debug(f"Lookup control by start date({start_date})")
start_date = start_date.strftime("%Y-%m-%d")
case int():
# logger.debug(f"Lookup control by ordinal start date {start_date}")
start_date = datetime.fromordinal(
datetime(1900, 1, 1).toordinal() + start_date - 2).date().strftime("%Y-%m-%d")
case _:
# logger.debug(f"Lookup control with parsed start date {start_date}")
start_date = parse(start_date).strftime("%Y-%m-%d")
match end_date:
case date():
# logger.debug(f"Lookup control by end date({end_date})")
end_date = end_date.strftime("%Y-%m-%d")
case int():
# logger.debug(f"Lookup control by ordinal end date {end_date}")
end_date = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + end_date - 2).date().strftime(
"%Y-%m-%d")
case _:
# logger.debug(f"Lookup control with parsed end date {end_date}")
end_date = parse(end_date).strftime("%Y-%m-%d")
# logger.debug(f"Looking up BasicSubmissions from start date: {start_date} and end date: {end_date}")
query = query.filter(cls.submitted_date.between(start_date, end_date))
match sub_type:
case str():
from backend import BasicSubmission, SubmissionType
query = query.join(BasicSubmission).join(SubmissionType).filter(SubmissionType.name == sub_type)
case _:
pass
match control_name:
case str():
# logger.debug(f"Lookup control by name {control_name}")
query = query.filter(cls.name.startswith(control_name))
limit = 1
case _:
pass
return cls.execute_query(query=query, limit=limit)
@classmethod
def make_chart(cls, parent, chart_settings: dict, ctx):
from frontend.visualizations.pcr_charts import PCRFigure
parent.mode_typer.clear()
parent.mode_typer.setEnabled(False)
report = Report()
controls = cls.query(sub_type=chart_settings['sub_type'], start_date=chart_settings['start_date'], end_date=chart_settings['end_date'])
data = [control.to_sub_dict() for control in controls]
df = DataFrame.from_records(data)
try:
df = df[df.ct > 0.0]
except AttributeError:
df = df
fig = PCRFigure(df=df, modes=None)
return report, fig
class IridaControl(Control):
id = Column(INTEGER, ForeignKey('_control.id'), primary_key=True)
contains = Column(JSON) #: unstructured hashes in contains.tsv for each organism
matches = Column(JSON) #: unstructured hashes in matches.tsv for each organism
kraken = Column(JSON) #: unstructured output from kraken_report
submission_id = Column(INTEGER, ForeignKey("_basicsubmission.id")) #: parent submission id
submission = relationship("BacterialCulture", back_populates="controls",
foreign_keys=[submission_id]) #: parent submission
sub_type = Column(String(16), nullable=False) #: EN-NOS, MCS-NOS, etc
refseq_version = Column(String(16)) #: version of refseq used in fastq parsing
kraken2_version = Column(String(16)) #: version of kraken2 used in fastq parsing
kraken2_db_version = Column(String(32)) #: folder name of kraken2 db
sample = relationship("BacterialCultureSample", back_populates="control") #: This control's submission sample
sample_id = Column(INTEGER,
ForeignKey("_basicsample.id", ondelete="SET NULL", name="cont_BCS_id")) #: sample id key
# submission_id = Column(INTEGER, ForeignKey("_basicsubmission.id")) #: parent submission id
# submission = relationship("BacterialCulture", back_populates="controls",
# foreign_keys=[submission_id]) #: parent submission
def __repr__(self) -> str:
return f"<Control({self.name})>"
__mapper_args__ = dict(polymorphic_identity="Irida Control",
polymorphic_load="inline",
inherit_condition=(id == Control.id))
@validates("sub_type")
def enforce_subtype_literals(self, key: str, value: str):
acceptables = ['ATCC49226', 'ATCC49619', 'EN-NOS', "EN-SSTI", "MCS-NOS", "MCS-SSTI", "SN-NOS", "SN-SSTI"]
if value.upper() not in acceptables:
raise KeyError(f"Sub-type must be in {acceptables}")
return value
def to_sub_dict(self) -> dict:
"""
Converts object into convenient dictionary for use in submission summary
Converts object into convenient dictionary for use in submission summary
Returns:
dict: output dictionary containing: Name, Type, Targets, Top Kraken results
"""
Returns:
dict: output dictionary containing: Name, Type, Targets, Top Kraken results
"""
# logger.debug("loading json string into dict")
try:
kraken = self.kraken
@@ -153,25 +363,27 @@ class Control(BaseClass):
else:
targets = ["None"]
# logger.debug("constructing output dictionary")
output = {
"name": self.name,
"type": self.controltype.name,
"targets": ", ".join(targets),
"kraken": new_kraken[0:10]
}
output = dict(
name=self.name,
type=self.controltype.name,
targets=", ".join(targets),
kraken=new_kraken[0:10]
)
return output
def convert_by_mode(self, mode: Literal['kraken', 'matches', 'contains']) -> List[dict]:
def convert_by_mode(self, control_sub_type: str, mode: Literal['kraken', 'matches', 'contains'],
consolidate: bool = False) -> Generator[dict, None, None]:
"""
split this instance into analysis types for controls graphs
Args:
mode (str): analysis type, 'contains', etc
consolidate (bool): whether to merge all off-target genera. Defaults to False
control_sub_type (str): control subtype, 'MCS-NOS', etc.
mode (str): analysis type, 'contains', etc.
Returns:
List[dict]: list of records
"""
output = []
# logger.debug("load json string for mode (i.e. contains, matches, kraken2)")
try:
data = self.__getattribute__(mode)
@@ -179,6 +391,18 @@ class Control(BaseClass):
data = {}
if data is None:
data = {}
# NOTE: Data truncation and consolidation.
if "kraken" in mode:
data = {k: v for k, v in sorted(data.items(), key=lambda d: d[1][f"{mode}_count"], reverse=True)[:50]}
else:
if consolidate:
on_tar = {k: v for k, v in data.items() if k.strip("*") in self.controltype.targets[control_sub_type]}
# logger.debug(f"Consolidating off-targets to: {self.controltype.targets[control_sub_type]}")
off_tar = sum(v[f'{mode}_ratio'] for k, v in data.items() if
k.strip("*") not in self.controltype.targets[control_sub_type])
on_tar['Off-target'] = {f"{mode}_ratio": off_tar}
data = on_tar
# logger.debug(pformat(data))
# logger.debug(f"Length of data: {len(data)}")
# logger.debug("dict keys are genera of bacteria, e.g. 'Streptococcus'")
for genus in data:
@@ -186,17 +410,13 @@ class Control(BaseClass):
name=self.name,
submitted_date=self.submitted_date,
genus=genus,
target='Target' if genus.strip("*") in self.controltype.targets else "Off-target"
target='Target' if genus.strip("*") in self.controltype.targets[control_sub_type] else "Off-target"
)
# logger.debug("get Target or Off-target of genus")
# logger.debug("set 'contains_hashes', etc for genus")
for key in data[genus]:
_dict[key] = data[genus][key]
output.append(_dict)
# logger.debug("Have to triage kraken data to keep program from getting overwhelmed")
if "kraken" in mode:
output = sorted(output, key=lambda d: d[f"{mode}_count"], reverse=True)[:50]
return output
yield _dict
@classmethod
def get_modes(cls) -> List[str]:
@@ -217,7 +437,7 @@ class Control(BaseClass):
@classmethod
@setup_lookup
def query(cls,
control_type: ControlType | str | None = None,
sub_type: str | None = None,
start_date: date | str | int | None = None,
end_date: date | str | int | None = None,
control_name: str | None = None,
@@ -227,7 +447,7 @@ class Control(BaseClass):
Lookup control objects in the database based on a number of parameters.
Args:
control_type (models.ControlType | str | None, optional): Control archetype. Defaults to None.
sub_type (models.ControlType | str | None, optional): Control archetype. Defaults to None.
start_date (date | str | int | None, optional): Beginning date to search by. Defaults to 2023-01-01 if end_date not None.
end_date (date | str | int | None, optional): End date to search by. Defaults to today if start_date not None.
control_name (str | None, optional): Name of control. Defaults to None.
@@ -238,13 +458,14 @@ class Control(BaseClass):
"""
query: Query = cls.__database_session__.query(cls)
# NOTE: by control type
match control_type:
case ControlType():
# logger.debug(f"Looking up control by control type: {control_type}")
query = query.filter(cls.controltype == control_type)
match sub_type:
# case ControlType():
# # logger.debug(f"Looking up control by control type: {sub_type}")
# query = query.filter(cls.controltype == sub_type)
case str():
# logger.debug(f"Looking up control by control type: {control_type}")
query = query.join(ControlType).filter(ControlType.name == control_type)
# logger.debug(f"Looking up control by control type: {sub_type}")
# query = query.join(ControlType).filter(ControlType.name == sub_type)
query = query.filter(cls.sub_type == sub_type)
case _:
pass
# NOTE: by date range
@@ -287,3 +508,241 @@ class Control(BaseClass):
case _:
pass
return cls.execute_query(query=query, limit=limit)
@classmethod
def make_parent_buttons(cls, parent: QWidget) -> None:
"""
Args:
parent (QWidget): chart holding widget to add buttons to.
Returns:
"""
super().make_parent_buttons(parent=parent)
rows = parent.layout.rowCount()
logger.debug(f"Parent rows: {rows}")
checker = QCheckBox(parent)
checker.setChecked(True)
checker.setObjectName("irida_check")
checker.setToolTip("Pools off-target genera to save time.")
parent.layout.addWidget(QLabel("Consolidate Off-targets"), rows, 0, 1, 1)
parent.layout.addWidget(checker, rows, 1, 1, 2)
checker.checkStateChanged.connect(parent.controls_getter_function)
@classmethod
@report_result
def make_chart(cls, chart_settings: dict, parent, ctx) -> Tuple[Report, "IridaFigure" | None]:
from frontend.visualizations import IridaFigure
try:
checker = parent.findChild(QCheckBox, name="irida_check")
if chart_settings['mode'] == "kraken":
checker.setEnabled(False)
checker.setChecked(False)
else:
checker.setEnabled(True)
consolidate = checker.isChecked()
except AttributeError:
consolidate = False
report = Report()
# logger.debug(f"settings: {pformat(chart_settings)}")
controls = cls.query(sub_type=chart_settings['sub_type'], start_date=chart_settings['start_date'],
end_date=chart_settings['end_date'])
# logger.debug(f"Controls found: {controls}")
if not controls:
report.add_result(Result(status="Critical", msg="No controls found in given date range."))
return report, None
# NOTE: change each control to list of dictionaries
data = [control.convert_by_mode(control_sub_type=chart_settings['sub_type'], mode=chart_settings['mode'],
consolidate=consolidate) for
control in controls]
# NOTE: flatten data to one dimensional list
data = [item for sublist in data for item in sublist]
# logger.debug(f"Control objects going into df conversion: {pformat(data)}")
if not data:
report.add_result(Result(status="Critical", msg="No data found for controls in given date range."))
return report, None
df = cls.convert_data_list_to_df(input_df=data, sub_mode=chart_settings['sub_mode'])
# logger.debug(f"Chart df: \n {df}")
if chart_settings['sub_mode'] is None:
title = chart_settings['sub_mode']
else:
title = f"{chart_settings['mode']} - {chart_settings['sub_mode']}"
# NOTE: send dataframe to chart maker
df, modes = cls.prep_df(ctx=ctx, df=df)
# logger.debug(f"prepped df: \n {df}")
# assert modes
# logger.debug(f"modes: {modes}")
fig = IridaFigure(df=df, ytitle=title, modes=modes, parent=parent,
months=chart_settings['months'])
return report, fig
@classmethod
def convert_data_list_to_df(cls, input_df: list[dict], sub_mode) -> DataFrame:
"""
Convert list of control records to dataframe
Args:
ctx (dict): settings passed from gui
input_df (list[dict]): list of dictionaries containing records
sub_type (str | None, optional): sub_type of submission type. Defaults to None.
Returns:
DataFrame: dataframe of controls
"""
# logger.debug(f"Subtype: {sub_mode}")
df = DataFrame.from_records(input_df)
# logger.debug(f"DF from records: {df}")
safe = ['name', 'submitted_date', 'genus', 'target']
for column in df.columns:
if column not in safe:
if sub_mode is not None and column != sub_mode:
continue
else:
safe.append(column)
if "percent" in column:
# count_col = [item for item in df.columns if "count" in item][0]
try:
count_col = next(item for item in df.columns if "count" in item)
except StopIteration:
continue
# NOTE: The actual percentage from kraken was off due to exclusion of NaN, recalculating.
df[column] = 100 * df[count_col] / df.groupby('name')[count_col].transform('sum')
df = df[[c for c in df.columns if c in safe]]
# NOTE: move date of sample submitted on same date as previous ahead one.
df = cls.displace_date(df=df)
# NOTE: ad hoc method to make data labels more accurate.
df = cls.df_column_renamer(df=df)
return df
@classmethod
def df_column_renamer(cls, df: DataFrame) -> DataFrame:
"""
Ad hoc function I created to clarify some fields
Args:
df (DataFrame): input dataframe
Returns:
DataFrame: dataframe with 'clarified' column names
"""
df = df[df.columns.drop(list(df.filter(regex='_hashes')))]
return df.rename(columns={
"contains_ratio": "contains_shared_hashes_ratio",
"matches_ratio": "matches_shared_hashes_ratio",
"kraken_count": "kraken2_read_count_(top_50)",
"kraken_percent": "kraken2_read_percent_(top_50)"
})
@classmethod
def displace_date(cls, df: DataFrame) -> DataFrame:
"""
This function serves to split samples that were submitted on the same date by incrementing dates.
It will shift the date forward by one day if it is the same day as an existing date in a list.
Args:
df (DataFrame): input dataframe composed of control records
Returns:
DataFrame: output dataframe with dates incremented.
"""
# logger.debug(f"Unique items: {df['name'].unique()}")
# NOTE: get submitted dates for each control
dict_list = [dict(name=item, date=df[df.name == item].iloc[0]['submitted_date']) for item in
sorted(df['name'].unique())]
previous_dates = set()
for item in dict_list:
df, previous_dates = cls.check_date(df=df, item=item, previous_dates=previous_dates)
return df
@classmethod
def check_date(cls, df: DataFrame, item: dict, previous_dates: set) -> Tuple[DataFrame, list]:
"""
Checks if an items date is already present in df and adjusts df accordingly
Args:
df (DataFrame): input dataframe
item (dict): control for checking
previous_dates (list): list of dates found in previous controls
Returns:
Tuple[DataFrame, list]: Output dataframe and appended list of previous dates
"""
try:
check = item['date'] in previous_dates
except IndexError:
check = False
previous_dates.add(item['date'])
if check:
# logger.debug(f"We found one! Increment date!\n\t{item['date']} to {item['date'] + timedelta(days=1)}")
# NOTE: get df locations where name == item name
mask = df['name'] == item['name']
# NOTE: increment date in dataframe
df.loc[mask, 'submitted_date'] = df.loc[mask, 'submitted_date'].apply(lambda x: x + timedelta(days=1))
item['date'] += timedelta(days=1)
passed = False
else:
passed = True
# logger.debug(f"\n\tCurrent date: {item['date']}\n\tPrevious dates:{previous_dates}")
# logger.debug(f"DF: {type(df)}, previous_dates: {type(previous_dates)}")
# NOTE: if run didn't lead to changed date, return values
if passed:
# logger.debug(f"Date check passed, returning.")
return df, previous_dates
# NOTE: if date was changed, rerun with new date
else:
logger.warning(f"Date check failed, running recursion")
df, previous_dates = cls.check_date(df, item, previous_dates)
return df, previous_dates
@classmethod
def prep_df(cls, ctx: Settings, df: DataFrame) -> Tuple[DataFrame | None, list]:
"""
Constructs figures based on parsed pandas dataframe.
Args:
ctx (Settings): settings passed down from gui
df (pd.DataFrame): input dataframe
ytitle (str | None, optional): title for the y-axis. Defaults to None.
Returns:
Figure: Plotly figure
"""
# NOTE: converts starred genera to normal and splits off list of starred
if df.empty:
return None, []
df['genus'] = df['genus'].replace({'\*': ''}, regex=True).replace({"NaN": "Unknown"})
df['genera'] = [item[-1] if item and item[-1] == "*" else "" for item in df['genus'].to_list()]
# NOTE: remove original runs, using reruns if applicable
df = cls.drop_reruns_from_df(ctx=ctx, df=df)
# NOTE: sort by and exclude from
sorts = ['submitted_date', "target", "genus"]
exclude = ['name', 'genera']
# logger.debug(df.columns)
modes = [item for item in df.columns if item not in sorts and item not in exclude]
# logger.debug(f"Modes coming out: {modes}")
# NOTE: Set descending for any columns that have "{mode}" in the header.
ascending = [False if item == "target" else True for item in sorts]
df = df.sort_values(by=sorts, ascending=ascending)
# logger.debug(df[df.isna().any(axis=1)])
# NOTE: actual chart construction is done by
return df, modes
@classmethod
def drop_reruns_from_df(cls, ctx: Settings, df: DataFrame) -> DataFrame:
"""
Removes semi-duplicates from dataframe after finding sequencing repeats.
Args:
ctx (Settings): settings passed from gui
df (DataFrame): initial dataframe
Returns:
DataFrame: dataframe with originals removed in favour of repeats.
"""
if 'rerun_regex' in ctx:
sample_names = get_unique_values_in_df_column(df, column_name="name")
rerun_regex = re.compile(fr"{ctx.rerun_regex}")
exclude = [re.sub(rerun_regex, "", sample) for sample in sample_names if rerun_regex.search(sample)]
df = df[df.name not in exclude]
return df

View File

@@ -13,7 +13,7 @@ from tempfile import TemporaryDirectory, TemporaryFile
from operator import itemgetter
from pprint import pformat
from . import BaseClass, Reagent, SubmissionType, KitType, Organization, Contact
from sqlalchemy import Column, String, TIMESTAMP, INTEGER, ForeignKey, JSON, FLOAT, case, desc
from sqlalchemy import Column, String, TIMESTAMP, INTEGER, ForeignKey, JSON, FLOAT, case
from sqlalchemy.orm import relationship, validates, Query
from sqlalchemy.orm.attributes import flag_modified
from sqlalchemy.ext.associationproxy import association_proxy
@@ -22,7 +22,6 @@ from sqlalchemy.exc import OperationalError as AlcOperationalError, IntegrityErr
from sqlite3 import OperationalError as SQLOperationalError, IntegrityError as SQLIntegrityError
import pandas as pd
from openpyxl import Workbook
from openpyxl.worksheet.worksheet import Worksheet
from openpyxl.drawing.image import Image as OpenpyxlImage
from tools import row_map, setup_lookup, jinja_template_loading, rreplace, row_keys, check_key_or_attr, Result, Report, \
report_result
@@ -32,8 +31,6 @@ from dateutil.parser import parse
from pathlib import Path
from jinja2.exceptions import TemplateNotFound
from jinja2 import Template
from docxtpl import InlineImage
from docx.shared import Inches
from PIL import Image
logger = logging.getLogger(f"submissions.{__name__}")
@@ -74,6 +71,8 @@ class BasicSubmission(BaseClass):
contact_id = Column(INTEGER, ForeignKey("_contact.id", ondelete="SET NULL",
name="fk_BS_contact_id")) #: client lab id from _organizations
custom = Column(JSON)
controls = relationship("Control", back_populates="submission",
uselist=True) #: A control sample added to submission
submission_sample_associations = relationship(
"SubmissionSampleAssociation",
@@ -114,7 +113,6 @@ class BasicSubmission(BaseClass):
# NOTE: Allows for subclassing into ex. BacterialCulture, Wastewater, etc.
__mapper_args__ = {
"polymorphic_identity": "Basic Submission",
# "polymorphic_on": submission_type_name,
"polymorphic_on": case(
(submission_type_name == "Wastewater", "Wastewater"),
@@ -190,7 +188,7 @@ class BasicSubmission(BaseClass):
# NOTE: Singles tells the query which fields to set limit to 1
dicto['singles'] = parent_defs['singles']
# logger.debug(dicto['singles'])
# NOTE: Grab subtype specific info.
# NOTE: Grab mode_sub_type specific info.
output = {}
for k, v in dicto.items():
if len(args) > 0 and k not in args:
@@ -960,7 +958,6 @@ class BasicSubmission(BaseClass):
pcr_sample_map = cls.get_submission_type().sample_map['pcr_samples']
# logger.debug(f'sample map: {pcr_sample_map}')
main_sheet = xl[pcr_sample_map['main_sheet']]
# samples = []
fields = {k: v for k, v in pcr_sample_map.items() if k not in ['main_sheet', 'start_row']}
for row in main_sheet.iter_rows(min_row=pcr_sample_map['start_row']):
idx = row[0].row
@@ -969,12 +966,11 @@ class BasicSubmission(BaseClass):
sheet = xl[v['sheet']]
sample[k] = sheet.cell(row=idx, column=v['column']).value
yield sample
# samples.append(sample)
# return samples
@classmethod
def parse_pcr_controls(cls, xl: Workbook) -> list:
def parse_pcr_controls(cls, xl: Workbook, rsl_plate_num: str) -> list:
location_map = cls.get_submission_type().sample_map['pcr_controls']
submission = cls.query(rsl_plate_num=rsl_plate_num)
name_column = 1
for item in location_map:
logger.debug(f"Looking for {item['name']}")
@@ -983,7 +979,29 @@ class BasicSubmission(BaseClass):
for cell in row:
if cell.value == item['name']:
logger.debug(f"Pulling from row {iii}, column {item['ct_column']}")
yield dict(name=item['name'], ct=worksheet.cell(row=iii, column=item['ct_column']).value)
subtype, target = item['name'].split("-")
ct = worksheet.cell(row=iii, column=item['ct_column']).value
if subtype == "PC":
ctrl = next((assoc.reagent for assoc in submission.submission_reagent_associations
if any(["positive control" in item.name.lower() for item in assoc.reagent.role])), None)
elif subtype == "NC":
ctrl = next((assoc.reagent for assoc in submission.submission_reagent_associations
if any(["molecular grade water" in item.name.lower() for item in assoc.reagent.role])), None)
try:
ct = float(ct)
except ValueError:
ct = 0.0
if ctrl:
ctrl = ctrl.lot
else:
ctrl = None
yield dict(
name=f"{rsl_plate_num}<{item['name']}>",
ct=ct,
subtype=subtype,
target=target,
reagent_lot=ctrl
)
@classmethod
def filename_template(cls) -> str:
@@ -996,21 +1014,6 @@ class BasicSubmission(BaseClass):
"""
return "{{ rsl_plate_num }}"
# @classmethod
# def custom_sample_autofill_row(cls, sample, worksheet: Worksheet) -> int:
# """
# Updates row information
#
# Args:
# sample (_type_): _description_
# worksheet (Workbook): _description_
#
# Returns:
# int: New row number
# """
# logger.debug(f"Sample from args: {sample}")
# return None
@classmethod
def adjust_autofill_samples(cls, samples: List[Any]) -> List[Any]:
"""
@@ -1025,19 +1028,6 @@ class BasicSubmission(BaseClass):
logger.info(f"Hello from {cls.__mapper_args__['polymorphic_identity']} sampler")
return samples
# def adjust_to_dict_samples(self, backup: bool = False) -> List[dict]:
# """
# Updates sample dictionaries with custom values
#
# Args:
# backup (bool, optional): Whether to perform backup. Defaults to False.
#
# Returns:
# List[dict]: Updated dictionaries
# """
# # logger.debug(f"Hello from {self.__class__.__name__} dictionary sample adjuster.")
# return [item.to_sub_dict() for item in self.submission_sample_associations]
@classmethod
def get_details_template(cls, base_dict: dict) -> Template:
"""
@@ -1380,8 +1370,7 @@ class BacterialCulture(BasicSubmission):
derivative submission type from BasicSubmission
"""
id = Column(INTEGER, ForeignKey('_basicsubmission.id'), primary_key=True)
controls = relationship("Control", back_populates="submission",
uselist=True) #: A control sample added to submission
__mapper_args__ = dict(polymorphic_identity="Bacterial Culture",
polymorphic_load="inline",
inherit_condition=(id == BasicSubmission.id))
@@ -1442,25 +1431,6 @@ class BacterialCulture(BasicSubmission):
pos_control_reg.missing = False
return pyd
# @classmethod
# def custom_sample_autofill_row(cls, sample, worksheet: Worksheet) -> int:
# """
# Extends parent
# """
# # logger.debug(f"Checking {sample.well}")
# # logger.debug(f"here's the worksheet: {worksheet}")
# row = super().custom_sample_autofill_row(sample, worksheet)
# df = pd.DataFrame(list(worksheet.values))
# # logger.debug(f"Here's the dataframe: {df}")
# idx = df[df[0] == sample.well]
# if idx.empty:
# new = f"{sample.well[0]}{sample.well[1:].zfill(2)}"
# # logger.debug(f"Checking: {new}")
# idx = df[df[0] == new]
# # logger.debug(f"Here is the row: {idx}")
# row = idx.index.to_list()[0]
# return row + 1
@classmethod
def custom_info_parser(cls, input_dict: dict, xl: Workbook | None = None, custom_fields: dict = {}) -> dict:
input_dict = super().custom_info_parser(input_dict=input_dict, xl=xl, custom_fields=custom_fields)
@@ -1548,7 +1518,7 @@ class Wastewater(BasicSubmission):
for sample in samples:
# NOTE: remove '-{target}' from controls
sample['sample'] = re.sub('-N\\d$', '', sample['sample'])
# # NOTE: if sample is already in output skip
# NOTE: if sample is already in output skip
if sample['sample'] in [item['sample'] for item in output]:
logger.warning(f"Already have {sample['sample']}")
continue
@@ -1577,8 +1547,6 @@ class Wastewater(BasicSubmission):
# @classmethod
# def parse_pcr_controls(cls, xl: Workbook, location_map: list) -> list:
@classmethod
def enforce_name(cls, instr: str, data: dict | None = {}) -> str:
"""
@@ -1681,15 +1649,17 @@ class Wastewater(BasicSubmission):
obj (_type_): Parent widget
"""
from backend.excel import PCRParser
from backend.db import PCRControl, ControlType
from frontend.widgets import select_open_file
report = Report()
fname = select_open_file(obj=obj, file_extension="xlsx")
if not fname:
report.add_result(Result(msg="No file selected, cancelling.", status="Warning"))
return report
parser = PCRParser(filepath=fname)
parser = PCRParser(filepath=fname, submission=self)
self.set_attribute("pcr_info", parser.pcr)
pcr_samples = [sample for sample in parser.samples]
pcr_controls = [control for control in parser.controls]
self.save(original=False)
# logger.debug(f"Got {len(parser.samples)} samples to update!")
# logger.debug(f"Parser samples: {parser.samples}")
@@ -1700,6 +1670,16 @@ class Wastewater(BasicSubmission):
except StopIteration:
continue
self.update_subsampassoc(sample=sample, input_dict=sample_dict)
controltype = ControlType.query(name="PCR Control")
logger.debug(parser.pcr)
submitted_date = datetime.strptime(" ".join(parser.pcr['run_start_date/time'].split(" ")[:-1]),
"%Y-%m-%d %I:%M:%S %p")
for control in pcr_controls:
new_control = PCRControl(**control)
new_control.submitted_date = submitted_date
new_control.controltype = controltype
new_control.submission = self
new_control.save()
class WastewaterArtic(BasicSubmission):
@@ -2207,7 +2187,7 @@ class BasicSample(BaseClass):
id = Column(INTEGER, primary_key=True) #: primary key
submitter_id = Column(String(64), nullable=False, unique=True) #: identification from submitter
sample_type = Column(String(32)) #: subtype of sample
sample_type = Column(String(32)) #: mode_sub_type of sample
sample_submission_associations = relationship(
"SubmissionSampleAssociation",
@@ -2632,7 +2612,7 @@ class BacterialCultureSample(BasicSample):
id = Column(INTEGER, ForeignKey('_basicsample.id'), primary_key=True)
organism = Column(String(64)) #: bacterial specimen
concentration = Column(String(16)) #: sample concentration
control = relationship("Control", back_populates="sample", uselist=False)
control = relationship("IridaControl", back_populates="sample", uselist=False)
__mapper_args__ = dict(polymorphic_identity="Bacterial Culture Sample",
polymorphic_load="inline",
inherit_condition=(id == BasicSample.id))
@@ -2677,7 +2657,7 @@ class SubmissionSampleAssociation(BaseClass):
# reference to the Sample object
sample = relationship(BasicSample, back_populates="sample_submission_associations") #: associated sample
base_sub_type = Column(String) #: string of subtype name
base_sub_type = Column(String) #: string of mode_sub_type name
# Refers to the type of parent.
# Hooooooo boy, polymorphic association type, now we're getting into the weeds!

View File

@@ -675,7 +675,7 @@ class PCRParser(object):
rsl_plate_num = self.submission_obj.rsl_plate_num
self.pcr = self.parse_general()
self.samples = self.submission_obj.parse_pcr(xl=self.xl, rsl_plate_num=rsl_plate_num)
self.controls = self.submission_obj.parse_pcr_controls(xl=self.xl)
self.controls = self.submission_obj.parse_pcr_controls(xl=self.xl, rsl_plate_num=rsl_plate_num)
def parse_general(self):
"""

View File

@@ -1,4 +1,52 @@
'''
Contains all operations for creating charts, graphs and visual effects.
'''
from .control_charts import *
from PyQt6.QtWidgets import QWidget
import plotly
from plotly.graph_objects import Figure
import pandas as pd
from frontend.widgets.functions import select_save_file
class CustomFigure(Figure):
def __init__(self, df: pd.DataFrame, modes: list, ytitle: str | None = None, parent: QWidget | None = None,
months: int = 6):
super().__init__()
def save_figure(self, group_name: str = "plotly_output", parent: QWidget | None = None):
"""
Writes plotly figure to html file.
Args:
figs ():
settings (dict): settings passed down from click
fig (Figure): input figure object
group_name (str): controltype
"""
output = select_save_file(obj=parent, default_name=group_name, extension="png")
self.write_image(output.absolute().__str__(), engine="kaleido")
def to_html(self) -> str:
"""
Creates final html code from plotly
Args:
figure (Figure): input figure
Returns:
str: html string
"""
html = '<html><body>'
if self is not None:
html += plotly.offline.plot(self, output_type='div',
include_plotlyjs='cdn') #, image = 'png', auto_open=True, image_filename='plot_image')
else:
html += "<h1>No data was retrieved for the given parameters.</h1>"
html += '</body></html>'
return html
from .irida_charts import IridaFigure
from .pcr_charts import PCRFigure

View File

@@ -1,15 +1,13 @@
"""
Functions for constructing controls graphs using plotly.
Functions for constructing irida controls graphs using plotly.
"""
from copy import deepcopy
from datetime import date
from pprint import pformat
import plotly
import plotly.express as px
import pandas as pd
from PyQt6.QtWidgets import QWidget
from plotly.graph_objects import Figure
from . import CustomFigure
import logging
from tools import get_unique_values_in_df_column, divide_chunks
from frontend.widgets.functions import select_save_file
@@ -17,11 +15,12 @@ from frontend.widgets.functions import select_save_file
logger = logging.getLogger(f"submissions.{__name__}")
class CustomFigure(Figure):
class IridaFigure(CustomFigure):
def __init__(self, df: pd.DataFrame, modes: list, ytitle: str | None = None, parent: QWidget | None = None,
months: int = 6):
super().__init__()
super().__init__(df=df, modes=modes)
self.construct_chart(df=df, modes=modes)
self.generic_figure_markers(modes=modes, ytitle=ytitle, months=months)
@@ -105,25 +104,16 @@ class CustomFigure(Figure):
self.update_xaxes(
rangeslider_visible=True,
rangeselector=dict(
# buttons=list([
# dict(count=1, label="1m", step="month", stepmode="backward"),
# dict(count=3, label="3m", step="month", stepmode="backward"),
# dict(count=6, label="6m", step="month", stepmode="backward"),
# dict(count=1, label="YTD", step="year", stepmode="todate"),
# dict(count=12, label="1y", step="month", stepmode="backward"),
# dict(step="all")
# ])
buttons=[button for button in self.make_plotly_buttons(months=months)]
)
)
assert isinstance(self, Figure)
# return fig
assert isinstance(self, CustomFigure)
def make_plotly_buttons(self, months:int=6):
def make_plotly_buttons(self, months: int = 6):
rng = [1]
if months > 2:
rng += [iii for iii in range(3, months, 3)]
logger.debug(f"Making buttons for months: {rng}")
# logger.debug(f"Making buttons for months: {rng}")
buttons = [dict(count=iii, label=f"{iii}m", step="month", stepmode="backward") for iii in rng]
if months > date.today().month:
buttons += [dict(count=1, label="YTD", step="year", stepmode="todate")]
@@ -161,35 +151,3 @@ class CustomFigure(Figure):
{"yaxis.title.text": mode},
])
def save_figure(self, group_name: str = "plotly_output", parent: QWidget | None = None):
"""
Writes plotly figure to html file.
Args:
figs ():
settings (dict): settings passed down from click
fig (Figure): input figure object
group_name (str): controltype
"""
output = select_save_file(obj=parent, default_name=group_name, extension="png")
self.write_image(output.absolute().__str__(), engine="kaleido")
def to_html(self) -> str:
"""
Creates final html code from plotly
Args:
figure (Figure): input figure
Returns:
str: html string
"""
html = '<html><body>'
if self is not None:
html += plotly.offline.plot(self, output_type='div',
include_plotlyjs='cdn') #, image = 'png', auto_open=True, image_filename='plot_image')
else:
html += "<h1>No data was retrieved for the given parameters.</h1>"
html += '</body></html>'
return html

View File

@@ -0,0 +1,34 @@
"""
Functions for constructing irida controls graphs using plotly.
"""
from datetime import date
from pprint import pformat
import plotly
from . import CustomFigure
import plotly.express as px
import pandas as pd
from PyQt6.QtWidgets import QWidget
from plotly.graph_objects import Figure
import logging
from tools import get_unique_values_in_df_column, divide_chunks
from frontend.widgets.functions import select_save_file
logger = logging.getLogger(f"submissions.{__name__}")
class PCRFigure(CustomFigure):
def __init__(self, df: pd.DataFrame, modes: list, ytitle: str | None = None, parent: QWidget | None = None,
months: int = 6):
super().__init__(df=df, modes=modes)
self.construct_chart(df=df)
# self.generic_figure_markers(modes=modes, ytitle=ytitle, months=months)
def construct_chart(self, df: pd.DataFrame):
logger.debug(f"PCR df: {df}")
try:
scatter = px.scatter(data_frame=df, x='submitted_date', y="ct", hover_data=["name", "target", "ct", "reagent_lot"], color='target')
except ValueError:
scatter = px.scatter()
self.add_traces(scatter.data)

View File

@@ -4,7 +4,7 @@ Constructs main application.
from pprint import pformat
from PyQt6.QtWidgets import (
QTabWidget, QWidget, QVBoxLayout,
QHBoxLayout, QScrollArea, QMainWindow,
QHBoxLayout, QScrollArea, QMainWindow,
QToolBar
)
from PyQt6.QtGui import QAction
@@ -13,7 +13,7 @@ from pathlib import Path
from markdown import markdown
from __init__ import project_path
from tools import check_if_app, Settings, Report, jinja_template_loading, check_authorization, page_size
from .functions import select_save_file,select_open_file
from .functions import select_save_file, select_open_file
from datetime import date
from .pop_ups import HTMLPop, AlertPop
from .misc import LogParser, Pagifier
@@ -29,6 +29,7 @@ from .summary import Summary
logger = logging.getLogger(f'submissions.{__name__}')
logger.info("Hello, I am a logger")
class App(QMainWindow):
def __init__(self, ctx: Settings = None):
@@ -61,11 +62,11 @@ class App(QMainWindow):
self.show()
self.statusBar().showMessage('Ready', 5000)
self.backup_database()
def _createMenuBar(self):
"""
adds items to menu bar
"""
"""
# logger.debug(f"Creating menu bar...")
menuBar = self.menuBar()
fileMenu = menuBar.addMenu("&File")
@@ -85,11 +86,11 @@ class App(QMainWindow):
# reportMenu.addAction(self.generateReportAction)
maintenanceMenu.addAction(self.joinExtractionAction)
maintenanceMenu.addAction(self.joinPCRAction)
def _createToolBar(self):
"""
adds items to toolbar
"""
"""
# logger.debug(f"Creating toolbar...")
toolbar = QToolBar("My main toolbar")
self.addToolBar(toolbar)
@@ -100,7 +101,7 @@ class App(QMainWindow):
def _createActions(self):
"""
creates actions
"""
"""
# logger.debug(f"Creating actions...")
self.importAction = QAction("&Import Submission", self)
self.addReagentAction = QAction("Add Reagent", self)
@@ -139,7 +140,7 @@ class App(QMainWindow):
def showAbout(self):
"""
Show the 'about' message
"""
"""
j_env = jinja_template_loading()
template = j_env.get_template("project.html")
html = template.render(info=self.ctx.package.__dict__)
@@ -150,7 +151,7 @@ class App(QMainWindow):
def openDocs(self):
"""
Open the documentation html pages
"""
"""
if check_if_app():
url = Path(sys._MEIPASS).joinpath("files", "docs", "index.html")
else:
@@ -182,14 +183,14 @@ class App(QMainWindow):
def runSampleSearch(self):
"""
Create a search for samples.
"""
"""
dlg = SearchBox(self)
dlg.exec()
def backup_database(self):
"""
Copies the database into the backup directory the first time it is opened every month.
"""
"""
month = date.today().strftime("%Y-%m")
current_month_bak = Path(self.ctx.backup_path).joinpath(f"submissions_backup-{month}").resolve()
logger.info(f"Here is the db directory: {self.ctx.database_path}")
@@ -244,8 +245,8 @@ class App(QMainWindow):
class AddSubForm(QWidget):
def __init__(self, parent:QWidget):
def __init__(self, parent: QWidget):
# logger.debug(f"Initializating subform...")
super(QWidget, self).__init__(parent)
self.layout = QVBoxLayout(self)
@@ -255,11 +256,12 @@ class AddSubForm(QWidget):
self.tab2 = QWidget()
self.tab3 = QWidget()
self.tab4 = QWidget()
self.tabs.resize(300,200)
self.tabs.resize(300, 200)
# NOTE: Add tabs
self.tabs.addTab(self.tab1,"Submissions")
self.tabs.addTab(self.tab2,"Controls")
self.tabs.addTab(self.tab3, "Summary Report")
self.tabs.addTab(self.tab1, "Submissions")
self.tabs.addTab(self.tab2, "Irida Controls")
self.tabs.addTab(self.tab3, "PCR Controls")
self.tabs.addTab(self.tab4, "Cost Report")
# self.tabs.addTab(self.tab4, "Add Kit")
# NOTE: Create submission adder form
self.formwidget = SubmissionFormContainer(self)
@@ -276,7 +278,7 @@ class AddSubForm(QWidget):
self.sheetlayout = QVBoxLayout(self)
self.sheetwidget.setLayout(self.sheetlayout)
self.sub_wid = SubmissionsSheet(parent=parent)
self.pager = Pagifier(page_max=self.sub_wid.total_count/page_size)
self.pager = Pagifier(page_max=self.sub_wid.total_count / page_size)
self.sheetlayout.addWidget(self.sub_wid)
self.sheetlayout.addWidget(self.pager)
# NOTE: Create layout of first tab to hold form and sheet
@@ -285,15 +287,19 @@ class AddSubForm(QWidget):
self.tab1.layout.addWidget(self.interior)
self.tab1.layout.addWidget(self.sheetwidget)
self.tab2.layout = QVBoxLayout(self)
self.controls_viewer = ControlsViewer(self)
self.tab2.layout.addWidget(self.controls_viewer)
self.irida_viewer = ControlsViewer(self, archetype="Irida Control")
self.tab2.layout.addWidget(self.irida_viewer)
self.tab2.setLayout(self.tab2.layout)
self.tab3.layout = QVBoxLayout(self)
self.pcr_viewer = ControlsViewer(self, archetype="PCR Control")
self.tab3.layout.addWidget(self.pcr_viewer)
self.tab3.setLayout(self.tab3.layout)
# NOTE: create custom widget to add new tabs
# ST_adder = SubmissionTypeAdder(self)
summary_report = Summary(self)
self.tab3.layout = QVBoxLayout(self)
self.tab3.layout.addWidget(summary_report)
self.tab3.setLayout(self.tab3.layout)
self.tab4.layout = QVBoxLayout(self)
self.tab4.layout.addWidget(summary_report)
self.tab4.setLayout(self.tab4.layout)
# kit_adder = KitAdder(self)
# self.tab4.layout = QVBoxLayout(self)
# self.tab4.layout.addWidget(kit_adder)

View File

@@ -4,6 +4,7 @@ Handles display of control charts
import re
import sys
from datetime import timedelta, date
from pprint import pformat
from typing import Tuple
from PyQt6.QtWebEngineWidgets import QWebEngineView
from PyQt6.QtWidgets import (
@@ -11,20 +12,26 @@ from PyQt6.QtWidgets import (
QDateEdit, QLabel, QSizePolicy, QPushButton, QGridLayout
)
from PyQt6.QtCore import QSignalBlocker
from backend.db import ControlType, Control
from backend.db import ControlType, IridaControl
from PyQt6.QtCore import QDate, QSize
import logging
from pandas import DataFrame
from tools import Report, Result, get_unique_values_in_df_column, Settings, report_result
from frontend.visualizations.control_charts import CustomFigure
from frontend.visualizations import IridaFigure, PCRFigure
from .misc import StartEndDatePicker
logger = logging.getLogger(f"submissions.{__name__}")
class ControlsViewer(QWidget):
def __init__(self, parent: QWidget) -> None:
def __init__(self, parent: QWidget, archetype: str) -> None:
super().__init__(parent)
logger.debug(f"Incoming Archetype: {archetype}")
self.archetype = ControlType.query(name=archetype)
if not self.archetype:
return
logger.debug(f"Archetype set as: {self.archetype}")
self.app = self.parent().parent()
# logger.debug(f"\n\n{self.app}\n\n")
self.report = Report()
@@ -32,51 +39,53 @@ class ControlsViewer(QWidget):
self.webengineview = QWebEngineView()
# NOTE: set tab2 layout
self.layout = QGridLayout(self)
self.control_typer = QComboBox()
self.control_sub_typer = QComboBox()
# NOTE: fetch types of controls
con_types = [item.name for item in ControlType.query()]
self.control_typer.addItems(con_types)
con_sub_types = [item for item in self.archetype.targets.keys()]
self.control_sub_typer.addItems(con_sub_types)
# NOTE: create custom widget to get types of analysis
self.mode_typer = QComboBox()
mode_types = Control.get_modes()
mode_types = IridaControl.get_modes()
self.mode_typer.addItems(mode_types)
# NOTE: create custom widget to get subtypes of analysis
self.sub_typer = QComboBox()
self.sub_typer.setEnabled(False)
self.mode_sub_typer = QComboBox()
self.mode_sub_typer.setEnabled(False)
# NOTE: add widgets to tab2 layout
self.layout.addWidget(self.datepicker, 0,0,1,2)
self.layout.addWidget(self.datepicker, 0, 0, 1, 2)
self.save_button = QPushButton("Save Chart", parent=self)
self.layout.addWidget(self.save_button, 0,2,1,1)
self.layout.addWidget(self.control_typer, 1,0,1,3)
self.layout.addWidget(self.mode_typer, 2,0,1,3)
self.layout.addWidget(self.sub_typer, 3,0,1,3)
self.layout.addWidget(self.webengineview, 4,0,1,3)
self.layout.addWidget(self.save_button, 0, 2, 1, 1)
self.layout.addWidget(self.control_sub_typer, 1, 0, 1, 3)
self.layout.addWidget(self.mode_typer, 2, 0, 1, 3)
self.layout.addWidget(self.mode_sub_typer, 3, 0, 1, 3)
self.archetype.get_instance_class().make_parent_buttons(parent=self)
self.layout.addWidget(self.webengineview, self.layout.rowCount(), 0, 1, 3)
self.setLayout(self.layout)
self.controls_getter()
self.control_typer.currentIndexChanged.connect(self.controls_getter)
self.mode_typer.currentIndexChanged.connect(self.controls_getter)
self.datepicker.start_date.dateChanged.connect(self.controls_getter)
self.datepicker.end_date.dateChanged.connect(self.controls_getter)
self.controls_getter_function()
self.control_sub_typer.currentIndexChanged.connect(self.controls_getter_function)
self.mode_typer.currentIndexChanged.connect(self.controls_getter_function)
self.datepicker.start_date.dateChanged.connect(self.controls_getter_function)
self.datepicker.end_date.dateChanged.connect(self.controls_getter_function)
self.save_button.pressed.connect(self.save_chart_function)
def save_chart_function(self):
self.fig.save_figure(parent=self)
def controls_getter(self):
"""
Lookup controls from database and send to chartmaker
"""
self.controls_getter_function()
# def controls_getter(self):
# """
# Lookup controls from database and send to chartmaker
# """
# self.controls_getter_function()
@report_result
def controls_getter_function(self):
def controls_getter_function(self, *args, **kwargs):
"""
Get controls based on start/end dates
"""
report = Report()
# NOTE: subtype defaults to disabled
# NOTE: mode_sub_type defaults to disabled
try:
self.sub_typer.disconnect()
self.mode_sub_typer.disconnect()
except TypeError:
pass
# NOTE: correct start date being more recent than end date and rerun
@@ -93,37 +102,32 @@ class ControlsViewer(QWidget):
# NOTE: convert to python useable date objects
self.start_date = self.datepicker.start_date.date().toPyDate()
self.end_date = self.datepicker.end_date.date().toPyDate()
self.con_type = self.control_typer.currentText()
self.con_sub_type = self.control_sub_typer.currentText()
self.mode = self.mode_typer.currentText()
self.sub_typer.clear()
self.mode_sub_typer.clear()
# NOTE: lookup subtypes
try:
sub_types = ControlType.query(name=self.con_type).get_subtypes(mode=self.mode)
sub_types = self.archetype.get_modes(mode=self.mode)
except AttributeError:
sub_types = []
if sub_types != []:
# NOTE: block signal that will rerun controls getter and update sub_typer
with QSignalBlocker(self.sub_typer) as blocker:
self.sub_typer.addItems(sub_types)
self.sub_typer.setEnabled(True)
self.sub_typer.currentTextChanged.connect(self.chart_maker)
if sub_types:
# NOTE: block signal that will rerun controls getter and update mode_sub_typer
with QSignalBlocker(self.mode_sub_typer) as blocker:
self.mode_sub_typer.addItems(sub_types)
self.mode_sub_typer.setEnabled(True)
self.mode_sub_typer.currentTextChanged.connect(self.chart_maker_function)
else:
self.sub_typer.clear()
self.sub_typer.setEnabled(False)
self.chart_maker()
self.mode_sub_typer.clear()
self.mode_sub_typer.setEnabled(False)
self.chart_maker_function()
return report
def diff_month(self, d1:date, d2:date):
def diff_month(self, d1: date, d2: date):
return abs((d1.year - d2.year) * 12 + d1.month - d2.month)
def chart_maker(self):
"""
Creates plotly charts for webview
"""
self.chart_maker_function()
@report_result
def chart_maker_function(self):
def chart_maker_function(self, *args, **kwargs):
# TODO: Generalize this by moving as much code as possible to IridaControl
"""
Create html chart for controls reporting
@@ -134,44 +138,26 @@ class ControlsViewer(QWidget):
Tuple[QMainWindow, dict]: Collection of new main app window and result dict
"""
report = Report()
# logger.debug(f"Control getter context: \n\tControl type: {self.con_type}\n\tMode: {self.mode}\n\tStart
# Date: {self.start_date}\n\tEnd Date: {self.end_date}")
# NOTE: set the subtype for kraken
if self.sub_typer.currentText() == "":
self.subtype = None
# logger.debug(f"Control getter context: \n\tControl type: {self.con_sub_type}\n\tMode: {self.mode}\n\tStart \
# Date: {self.start_date}\n\tEnd Date: {self.end_date}")
# NOTE: set the mode_sub_type for kraken
if self.mode_sub_typer.currentText() == "":
self.mode_sub_type = None
else:
self.subtype = self.sub_typer.currentText()
# logger.debug(f"Subtype: {self.subtype}")
self.mode_sub_type = self.mode_sub_typer.currentText()
logger.debug(f"Subtype: {self.mode_sub_type}")
months = self.diff_month(self.start_date, self.end_date)
# NOTE: query all controls using the type/start and end dates from the gui
controls = Control.query(control_type=self.con_type, start_date=self.start_date, end_date=self.end_date)
# NOTE: if no data found from query set fig to none for reporting in webview
if controls is None:
fig = None
self.save_button.setEnabled(False)
else:
# NOTE: change each control to list of dictionaries
data = [control.convert_by_mode(mode=self.mode) for control in controls]
# NOTE: flatten data to one dimensional list
data = [item for sublist in data for item in sublist]
# logger.debug(f"Control objects going into df conversion: {type(data)}")
if not data:
report.add_result(Result(status="Critical", msg="No data found for controls in given date range."))
return
# NOTE send to dataframe creator
df = self.convert_data_list_to_df(input_df=data)
if self.subtype is None:
title = self.mode
else:
title = f"{self.mode} - {self.subtype}"
# NOTE: send dataframe to chart maker
df, modes = self.prep_df(ctx=self.app.ctx, df=df)
months = self.diff_month(self.start_date, self.end_date)
fig = CustomFigure(df=df, ytitle=title, modes=modes, parent=self, months=months)
self.save_button.setEnabled(True)
chart_settings = dict(sub_type=self.con_sub_type, start_date=self.start_date, end_date=self.end_date,
mode=self.mode,
sub_mode=self.mode_sub_type, parent=self, months=months)
_, self.fig = self.archetype.get_instance_class().make_chart(chart_settings=chart_settings, parent=self, ctx=self.app.ctx)
# if isinstance(self.fig, IridaFigure):
# self.save_button.setEnabled(True)
# logger.debug(f"Updating figure...")
self.fig = fig
# self.fig = fig
# NOTE: construct html for webview
html = fig.to_html()
html = self.fig.to_html()
# logger.debug(f"The length of html code is: {len(html)}")
self.webengineview.setHtml(html)
self.webengineview.update()
@@ -185,7 +171,7 @@ class ControlsViewer(QWidget):
Args:
ctx (dict): settings passed from gui
input_df (list[dict]): list of dictionaries containing records
subtype (str | None, optional): sub_type of submission type. Defaults to None.
mode_sub_type (str | None, optional): sub_type of submission type. Defaults to None.
Returns:
DataFrame: dataframe of controls
@@ -195,7 +181,7 @@ class ControlsViewer(QWidget):
safe = ['name', 'submitted_date', 'genus', 'target']
for column in df.columns:
if column not in safe:
if self.subtype is not None and column != self.subtype:
if self.mode_sub_type is not None and column != self.mode_sub_type:
continue
else:
safe.append(column)
@@ -338,34 +324,4 @@ class ControlsViewer(QWidget):
rerun_regex = re.compile(fr"{ctx.rerun_regex}")
exclude = [re.sub(rerun_regex, "", sample) for sample in sample_names if rerun_regex.search(sample)]
df = df[df.name not in exclude]
# for sample in sample_names:
# if rerun_regex.search(sample):
# first_run = re.sub(rerun_regex, "", sample)
# df = df.drop(df[df.name == first_run].index)
return df
# class ControlsDatePicker(QWidget):
# """
# custom widget to pick start and end dates for controls graphs
# """
#
# def __init__(self) -> None:
# super().__init__()
# self.start_date = QDateEdit(calendarPopup=True)
# # NOTE: start date is two months prior to end date by default
# sixmonthsago = QDate.currentDate().addDays(-180)
# self.start_date.setDate(sixmonthsago)
# self.end_date = QDateEdit(calendarPopup=True)
# self.end_date.setDate(QDate.currentDate())
# self.layout = QHBoxLayout()
# self.layout.addWidget(QLabel("Start Date"))
# self.layout.addWidget(self.start_date)
# self.layout.addWidget(QLabel("End Date"))
# self.layout.addWidget(self.end_date)
# self.setLayout(self.layout)
# self.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed)
#
#
# def sizeHint(self) -> QSize:
# return QSize(80, 20)

View File

@@ -7,7 +7,7 @@ from PyQt6.QtWidgets import (QDialog, QPushButton, QVBoxLayout,
QDialogButtonBox, QTextEdit, QGridLayout)
from PyQt6.QtWebEngineWidgets import QWebEngineView
from PyQt6.QtWebChannel import QWebChannel
from PyQt6.QtCore import Qt, pyqtSlot, QMarginsF
from PyQt6.QtCore import Qt, pyqtSlot, QMarginsF, QSize
from jinja2 import TemplateNotFound
from backend.db.models import BasicSubmission, BasicSample, Reagent, KitType
from tools import is_power_user, jinja_template_loading
@@ -38,7 +38,7 @@ class SubmissionDetails(QDialog):
self.app = None
self.webview = QWebEngineView(parent=self)
self.webview.setMinimumSize(900, 500)
self.webview.setMaximumSize(900, 700)
self.webview.setMaximumWidth(900)
self.webview.loadFinished.connect(self.activate_export)
self.layout = QGridLayout()
# self.setFixedSize(900, 500)
@@ -98,9 +98,6 @@ class SubmissionDetails(QDialog):
sample = BasicSample.query(submitter_id=sample)
base_dict = sample.to_sub_dict(full_data=True)
exclude = ['submissions', 'excluded', 'colour', 'tooltip']
# try:
# base_dict['excluded'] += exclude
# except KeyError:
base_dict['excluded'] = exclude
template = sample.get_details_template()
template_path = Path(self.template.environment.loader.__getattribute__("searchpath")[0])
@@ -131,7 +128,6 @@ class SubmissionDetails(QDialog):
html = template.render(reagent=base_dict, css=css)
self.webview.setHtml(html)
self.setWindowTitle(f"Reagent Details - {reagent.name} - {reagent.lot}")
# self.btn.setEnabled(False)
@pyqtSlot(str)
def submission_details(self, submission: str | BasicSubmission):
@@ -160,8 +156,6 @@ class SubmissionDetails(QDialog):
# logger.debug(f"Submission_details: {pformat(self.base_dict)}")
# logger.debug(f"User is power user: {is_power_user()}")
self.html = self.template.render(sub=self.base_dict, signing_permission=is_power_user(), css=css)
# with open("test.html", "w") as f:
# f.write(self.html)
self.webview.setHtml(self.html)
@pyqtSlot(str)
@@ -178,11 +172,6 @@ class SubmissionDetails(QDialog):
Renders submission to html, then creates and saves .pdf file to user selected file.
"""
fname = select_save_file(obj=self, default_name=self.export_plate, extension="pdf")
# page_layout = QPageLayout()
# page_layout.setPageSize(QPageSize(QPageSize.PageSizeId.A4))
# page_layout.setOrientation(QPageLayout.Orientation.Portrait)
# page_layout.setMargins(QMarginsF(25, 25, 25, 25))
# self.webview.page().printToPdf(fname.with_suffix(".pdf").__str__(), page_layout)
save_pdf(obj=self, filename=fname)
class SubmissionComment(QDialog):

View File

@@ -17,6 +17,7 @@ from sqlalchemy import create_engine, text, MetaData
from pydantic import field_validator, BaseModel, Field
from pydantic_settings import BaseSettings, SettingsConfigDict
from typing import Any, Tuple, Literal, List
print(inspect.stack()[1])
from __init__ import project_path
from configparser import ConfigParser
from tkinter import Tk # NOTE: This is for choosing database path before app is created.
@@ -960,6 +961,6 @@ def report_result(func):
except Exception as e:
logger.error(f"Problem reporting due to {e}")
logger.error(result.msg)
logger.debug(f"Returning: {output}")
# logger.debug(f"Returning: {output}")
return output
return wrapper