landon/Submissions-App
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

controls working

Browse Source
This commit is contained in:
Landon Wark
2023-01-23 14:28:24 -06:00
parent d17ee5862d
commit 7a53cfd9a1
9 changed files with 625 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files

257
src/submissions/frontend/__init__.py
View File

@@ -7,7 +7,7 @@ from PyQt6.QtWidgets import (
QSpinBox
)
from PyQt6.QtGui import QAction, QIcon
from PyQt6.QtCore import QDateTime, QDate
from PyQt6.QtCore import QDateTime, QDate, QSignalBlocker
from PyQt6.QtCore import pyqtSlot
from PyQt6.QtWebEngineWidgets import QWebEngineView
@@ -19,17 +19,21 @@ import plotly.express as px
import yaml
from backend.excel.parser import SheetParser
from backend.excel.reports import convert_control_by_mode, convert_data_list_to_df
from backend.db import (construct_submission_info, lookup_reagent,
construct_reagent, store_reagent, store_submission, lookup_kittype_by_use,
lookup_regent_by_type_name_and_kit_name, lookup_all_orgs, lookup_submissions_by_date_range,
get_all_Control_Types_names, create_kit_from_yaml
get_all_Control_Types_names, create_kit_from_yaml, get_all_available_modes, get_all_controls_by_type,
get_control_subtypes
)
from backend.excel.reports import make_report_xlsx
import numpy
from frontend.custom_widgets import AddReagentQuestion, AddReagentForm, SubmissionsSheet, ReportDatePicker, KitAdder
from frontend.custom_widgets import AddReagentQuestion, AddReagentForm, SubmissionsSheet, ReportDatePicker, KitAdder, ControlsDatePicker
import logging
import difflib
from frontend.visualizations.charts import create_charts
logger = logging.getLogger(__name__)
logger.info("Hello, I am a logger")
@@ -54,7 +58,8 @@ class App(QMainWindow):
self._createMenuBar()
self._createToolBar()
self._connectActions()
self.renderPage()
# self.renderPage()
self.controls_getter()
self.show()
def _createMenuBar(self):
@@ -86,6 +91,10 @@ class App(QMainWindow):
self.addReagentAction.triggered.connect(self.add_reagent)
self.generateReportAction.triggered.connect(self.generateReport)
self.addKitAction.triggered.connect(self.add_kit)
self.table_widget.control_typer.currentIndexChanged.connect(self.controls_getter)
self.table_widget.mode_typer.currentIndexChanged.connect(self.controls_getter)
self.table_widget.datepicker.start_date.dateChanged.connect(self.controls_getter)
self.table_widget.datepicker.end_date.dateChanged.connect(self.controls_getter)
def importSubmission(self):
@@ -207,6 +216,28 @@ class App(QMainWindow):
html += '</body></html>'
self.table_widget.webengineview.setHtml(html)
self.table_widget.webengineview.update()
# type = self.table_widget.control_typer.currentText()
# mode = self.table_widget.mode_typer.currentText()
# controls = get_all_controls_by_type(ctx=self.ctx, type=type)
# data = []
# for control in controls:
# dicts = convert_control_by_mode(ctx=self.ctx, control=control, mode=mode)
# data.append(dicts)
# data = [item for sublist in data for item in sublist]
# # print(data)
# df = convert_data_list_to_df(ctx=self.ctx, input=data)
# fig = create_charts(ctx=self.ctx, df=df)
# print(fig)
# html = '<html><body>'
# html += plotly.offline.plot(fig, output_type='div', auto_open=True, image = 'png', image_filename='plot_image')
# html += '</body></html>'
# html = plotly.io.to_html(fig)
# # print(html)
# # with open("C:\\Users\\lwark\\Desktop\\test.html", "w") as f:
# # f.write(html)
# self.table_widget.webengineview.setHtml(html)
# self.table_widget.webengineview.update()
def submit_new_sample(self):
@@ -294,19 +325,101 @@ class App(QMainWindow):
def add_kit(self):
home_dir = str(Path(self.ctx["directory_path"]))
fname = Path(QFileDialog.getOpenFileName(self, 'Open file', home_dir)[0])
fname = Path(QFileDialog.getOpenFileName(self, 'Open file', home_dir, filter = "yml(*.yml)")[0])
assert fname.exists()
with open(fname.__str__(), "r") as stream:
try:
exp = yaml.load(stream, Loader=yaml.Loader)
except yaml.YAMLError as exc:
logger.error(f'Error reading yaml file {fname}: {exc}')
return {}
try:
with open(fname.__str__(), "r") as stream:
try:
exp = yaml.load(stream, Loader=yaml.Loader)
except yaml.YAMLError as exc:
logger.error(f'Error reading yaml file {fname}: {exc}')
return {}
except PermissionError:
return
create_kit_from_yaml(ctx=self.ctx, exp=exp)
def controls_getter(self):
# self.table_widget.webengineview.setHtml("")
try:
self.table_widget.sub_typer.disconnect()
except TypeError:
pass
if self.table_widget.datepicker.start_date.date() > self.table_widget.datepicker.end_date.date():
print("that is not allowed!")
# self.table_widget.datepicker.start_date.setDate(e_date)
threemonthsago = self.table_widget.datepicker.end_date.date().addDays(-90)
with QSignalBlocker(self.table_widget.datepicker.start_date) as blocker:
self.table_widget.datepicker.start_date.setDate(threemonthsago)
self.controls_getter()
return
self.start_date = self.table_widget.datepicker.start_date.date().toPyDate()
self.end_date = self.table_widget.datepicker.end_date.date().toPyDate()
self.con_type = self.table_widget.control_typer.currentText()
self.mode = self.table_widget.mode_typer.currentText()
self.table_widget.sub_typer.clear()
sub_types = get_control_subtypes(ctx=self.ctx, type=self.con_type, mode=self.mode)
if sub_types != []:
with QSignalBlocker(self.table_widget.sub_typer) as blocker:
self.table_widget.sub_typer.addItems(sub_types)
self.table_widget.sub_typer.setEnabled(True)
self.table_widget.sub_typer.currentTextChanged.connect(self.chart_maker)
else:
self.table_widget.sub_typer.clear()
self.table_widget.sub_typer.setEnabled(False)
self.chart_maker()
def chart_maker(self):
print(f"Control getter context: \n\tControl type: {self.con_type}\n\tMode: {self.mode}\n\tStart Date: {self.start_date}\n\tEnd Date: {self.end_date}")
if self.table_widget.sub_typer.currentText() == "":
self.subtype = None
else:
self.subtype = self.table_widget.sub_typer.currentText()
print(f"Subtype: {self.subtype}")
controls = get_all_controls_by_type(ctx=self.ctx, con_type=self.con_type, start_date=self.start_date, end_date=self.end_date)
if controls == None:
return
data = []
for control in controls:
dicts = convert_control_by_mode(ctx=self.ctx, control=control, mode=self.mode)
data.append(dicts)
data = [item for sublist in data for item in sublist]
# print(data)
df = convert_data_list_to_df(ctx=self.ctx, input=data, subtype=self.subtype)
if self.subtype == None:
title = self.mode
else:
title = f"{self.mode} - {self.subtype}"
fig = create_charts(ctx=self.ctx, df=df, ytitle=title)
print(f"Updating figure...")
html = '<html><body>'
if fig != None:
html += plotly.offline.plot(fig, output_type='div', include_plotlyjs='cdn')#, image = 'png', auto_open=True, image_filename='plot_image')
else:
html += "<h1>No data was retrieved for the given parameters.</h1>"
html += '</body></html>'
# with open("C:\\Users\\lwark\\Desktop\\test.html", "w") as f:
# f.write(html)
self.table_widget.webengineview.setHtml(html)
self.table_widget.webengineview.update()
print("Figure updated... I hope.")
# def datechange(self):
# s_date = self.table_widget.datepicker.start_date.date()
# e_date = self.table_widget.datepicker.end_date.date()
# if s_date > e_date:
# print("that is not allowed!")
# # self.table_widget.datepicker.start_date.setDate(e_date)
# threemonthsago = e_date.addDays(-90)
# self.table_widget.datepicker.start_date.setDate(threemonthsago)
# self.chart_maker()
class AddSubForm(QWidget):
def __init__(self, parent):
@@ -354,7 +467,7 @@ class AddSubForm(QWidget):
# self.tab1.layout.addWidget(self.scroller)
# self.tab1.setWidget(self.scroller)
# self.tab1.setMinimumHeight(300)
self.datepicker = ControlsDatePicker()
self.webengineview = QWebEngineView()
# data = '''<html>Hello World</html>'''
# self.webengineview.setHtml(data)
@@ -362,7 +475,15 @@ class AddSubForm(QWidget):
self.control_typer = QComboBox()
con_types = get_all_Control_Types_names(ctx=parent.ctx)
self.control_typer.addItems(con_types)
self.mode_typer = QComboBox()
mode_types = get_all_available_modes(ctx=parent.ctx)
self.mode_typer.addItems(mode_types)
self.sub_typer = QComboBox()
self.sub_typer.setEnabled(False)
self.tab2.layout.addWidget(self.datepicker)
self.tab2.layout.addWidget(self.control_typer)
self.tab2.layout.addWidget(self.mode_typer)
self.tab2.layout.addWidget(self.sub_typer)
self.tab2.layout.addWidget(self.webengineview)
self.tab2.setLayout(self.tab2.layout)
# Add tabs to widget
@@ -372,113 +493,3 @@ class AddSubForm(QWidget):
self.tab3.setLayout(self.tab3.layout)
self.layout.addWidget(self.tabs)
self.setLayout(self.layout)
# @pyqtSlot()
# def on_click(self):
# print("\n")
# for currentQTableWidgetItem in self.tableWidget.selectedItems():
# print(currentQTableWidgetItem.row(), currentQTableWidgetItem.column(), currentQTableWidgetItem.text())
# import sys
# from pathlib import Path
# from textual import events
# from textual.app import App, ComposeResult
# from textual.containers import Container, Vertical
# from textual.reactive import var
# from textual.widgets import DirectoryTree, Footer, Header, Input, Label
# from textual.css.query import NoMatches
# sys.path.append(Path(__file__).absolute().parents[1].__str__())
# from backend.excel.parser import SheetParser
# class FormField(Input):
# def on_mount(self):
# self.placeholder = "Value not set."
# def update(self, input:str):
# self.value = input
# class DataBrowser(App):
# """
# File browser input
# """
# CSS_PATH = "static/css/data_browser.css"
# BINDINGS = [
# ("ctrl+f", "toggle_files", "Toggle Files"),
# ("ctrl+q", "quit", "Quit"),
# ]
# show_tree = var(True)
# context = {}
# def watch_show_tree(self, show_tree: bool) -> None:
# """Called when show_tree is modified."""
# self.set_class(show_tree, "-show-tree")
# def compose(self) -> ComposeResult:
# """Compose our UI."""
# if 'directory_path' in self.context:
# path = self.context['directory_path']
# else:
# path = "."
# yield Header()
# yield Container(
# DirectoryTree(path, id="tree-view"),
# Vertical(
# Label("[b]File Name[/b]", classes='box'), FormField(id="file-name", classes='box'),
# # Label("[b]Sample Type[/b]", classes='box'), FormField(id="sample-type", classes='box'),
# id="form-view"
# )
# )
# yield Footer()
# def on_mount(self, event: events.Mount) -> None:
# self.query_one(DirectoryTree).focus()
# def on_directory_tree_file_selected(self, event: DirectoryTree.FileSelected) -> None:
# """Called when the user click a file in the directory tree."""
# event.stop()
# sample = SheetParser(Path(event.path), **self.context)
# sample_view = self.query_one("#file-name", FormField)
# # sample_type = self.query_one("#sample-type", FormField)
# sample_view.update(event.path)
# # sample_type.update(sample.sub['sample_type'])
# form_view = self.query_one("#form-view", Vertical)
# if sample.sub != None:
# for var in sample.sub.keys():
# # if var == "sample_type":
# # continue
# try:
# deleter = self.query_one(f"#{var}_label")
# deleter.remove()
# except NoMatches:
# pass
# try:
# deleter = self.query_one(f"#{var}")
# deleter.remove()
# except NoMatches:
# pass
# form_view.mount(Label(var.replace("_", " ").upper(), id=f"{var}_label", classes='box added'))
# form_view.mount(FormField(id=var, classes='box added', value=sample.sub[var]))
# else:
# adds = self.query(".added")
# for add in adds:
# add.remove()
# def action_toggle_files(self) -> None:
# """Called in response to key binding."""
# self.show_tree = not self.show_tree
# if __name__ == "__main__":
# app = DataBrowser()
# app.run()

31
src/submissions/frontend/custom_widgets/__init__.py
View File

@@ -4,9 +4,9 @@ from PyQt6.QtWidgets import (
QDialogButtonBox, QDateEdit, QTableView,
QTextEdit, QSizePolicy, QWidget,
QGridLayout, QPushButton, QSpinBox,
QScrollBar, QScrollArea
QScrollBar, QScrollArea, QHBoxLayout
)
from PyQt6.QtCore import Qt, QDate, QAbstractTableModel
from PyQt6.QtCore import Qt, QDate, QAbstractTableModel, QSize
from PyQt6.QtGui import QFontMetrics
from backend.db import get_all_reagenttype_names, submissions_to_df, lookup_submission_by_id, lookup_all_sample_types, create_kit_from_yaml
@@ -35,7 +35,7 @@ class AddReagentQuestion(QDialog):
self.buttonBox.rejected.connect(self.reject)
self.layout = QVBoxLayout()
message = QLabel(f"Couldn't find reagent type {reagent_type.replace('_', ' ').title()}: {reagent_lot} in the database.\nWould you like to add it?")
message = QLabel(f"Couldn't find reagent type {reagent_type.replace('_', ' ').title().strip('Lot')}: {reagent_lot} in the database.\nWould you like to add it?")
self.layout.addWidget(message)
self.layout.addWidget(self.buttonBox)
self.setLayout(self.layout)
@@ -151,6 +151,7 @@ class SubmissionDetails(QDialog):
interior.setParent(self)
data = lookup_submission_by_id(ctx=ctx, id=id)
base_dict = data.to_dict()
del base_dict['id']
base_dict['reagents'] = [item.to_sub_dict() for item in data.reagents]
base_dict['samples'] = [item.to_sub_dict() for item in data.samples]
template = env.get_template("submission_details.txt")
@@ -306,4 +307,26 @@ class ReagentTypeForm(QWidget):
grid.addWidget(QLabel("Extension of Life (months):"),0,2)
eol = QSpinBox()
eol.setMinimum(0)
grid.addWidget(eol, 0,3)
grid.addWidget(eol, 0,3)
class ControlsDatePicker(QWidget):
def __init__(self) -> None:
super().__init__()
self.start_date = QDateEdit(calendarPopup=True)
threemonthsago = QDate.currentDate().addDays(-90)
self.start_date.setDate(threemonthsago)
self.end_date = QDateEdit(calendarPopup=True)
self.end_date.setDate(QDate.currentDate())
self.layout = QHBoxLayout()
self.layout.addWidget(QLabel("Start Date"))
self.layout.addWidget(self.start_date)
self.layout.addWidget(QLabel("End Date"))
self.layout.addWidget(self.end_date)
self.setLayout(self.layout)
self.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed)
def sizeHint(self):
return QSize(80,20)

0
src/submissions/frontend/visualizations/__init__.py Normal file
View File

278
src/submissions/frontend/visualizations/charts.py Normal file
View File

@@ -0,0 +1,278 @@
import plotly.express as px
import pandas as pd
from pathlib import Path
from plotly.graph_objects import Figure
import logging
from backend.excel import get_unique_values_in_df_column
logger = logging.getLogger("controls.tools.vis_functions")
def create_charts(ctx:dict, df:pd.DataFrame, ytitle:str|None=None) -> Figure:
"""
Constructs figures based on parsed pandas dataframe.
Args:
settings (dict): settings passed down from click
df (pd.DataFrame): input dataframe
group_name (str): controltype
Returns:
Figure: _description_
"""
from backend.excel import drop_reruns_from_df
genera = []
if df.empty:
return None
for item in df['genus'].to_list():
try:
if item[-1] == "*":
genera.append(item[-1])
else:
genera.append("")
except IndexError:
genera.append("")
df['genus'] = df['genus'].replace({'\*':''}, regex=True)
df['genera'] = genera
df = df.dropna()
df = drop_reruns_from_df(ctx=ctx, df=df)
sorts = ['submitted_date', "target", "genus"]
exclude = ['name', 'genera']
modes = [item for item in df.columns if item not in sorts and item not in exclude and "_hashes" not in item]
# Set descending for any columns that have "{mode}" in the header.
ascending = [False if item == "target" else True for item in sorts]
df = df.sort_values(by=sorts, ascending=ascending)
fig = construct_chart(ctx=ctx, df=df, modes=modes, ytitle=ytitle)
return fig
def generic_figure_markers(fig:Figure, modes:list=[], ytitle:str|None=None) -> Figure:
"""
Adds standard layout to figure.
Args:
fig (Figure): Input figure.
modes (list, optional): List of modes included in figure. Defaults to [].
Returns:
Figure: Output figure with updated titles, rangeslider, buttons.
"""
if modes != []:
ytitle = modes[0]
# Creating visibles list for each mode.
fig.update_layout(
xaxis_title="Submitted Date (* - Date parsed from fastq file creation date)",
yaxis_title=ytitle,
showlegend=True,
barmode='stack',
updatemenus=[
dict(
type="buttons",
direction="right",
x=0.7,
y=1.2,
showactive=True,
buttons=make_buttons(modes=modes, fig_len=len(fig.data)),
)
]
)
fig.update_xaxes(
rangeslider_visible=True,
rangeselector=dict(
buttons=list([
dict(count=1, label="1m", step="month", stepmode="backward"),
dict(count=3, label="3m", step="month", stepmode="backward"),
dict(count=6, label="6m", step="month", stepmode="backward"),
dict(count=1, label="YTD", step="year", stepmode="todate"),
dict(count=1, label="1y", step="year", stepmode="backward"),
dict(step="all")
])
)
)
logger.debug(f"Returning figure {fig}")
assert type(fig) == Figure
return fig
def make_buttons(modes:list, fig_len:int) -> list:
"""
Creates list of buttons with one for each mode to be used in showing/hiding mode traces.
Args:
modes (list): list of modes used by main parser.
fig_len (int): number of traces in the figure
Returns:
list: list of buttons.
"""
buttons = []
if len(modes) > 1:
for ii, mode in enumerate(modes):
# What I need to do is create a list of bools with the same length as the fig.data
mode_vis = [True] * fig_len
# And break it into {len(modes)} chunks
mode_vis = list(divide_chunks(mode_vis, len(modes)))
# Then, for each chunk, if the chunk index isn't equal to the index of the current mode, set to false
for jj, sublist in enumerate(mode_vis):
if jj != ii:
mode_vis[jj] = [not elem for elem in mode_vis[jj]]
# Finally, flatten list.
mode_vis = [item for sublist in mode_vis for item in sublist]
# Now, make button to add to list
buttons.append(dict(label=mode, method="update", args=[
{"visible": mode_vis},
{"yaxis.title.text": mode},
]
))
return buttons
def output_figures(settings:dict, figs:list, group_name:str):
"""
Writes plotly figure to html file.
Args:
settings (dict): settings passed down from click
fig (Figure): input figure object
group_name (str): controltype
"""
with open(Path(settings['folder']['output']).joinpath(f'{group_name}.html'), "w") as f:
for fig in figs:
try:
f.write(fig.to_html(full_html=False, include_plotlyjs='cdn'))
except AttributeError:
logger.error(f"The following figure was a string: {fig}")
# Below are the individual construction functions. They must be named "construct_{mode}_chart" and
# take only json_in and mode to hook into the main processor.
def construct_chart(ctx:dict, df:pd.DataFrame, modes:list, ytitle:str|None=None) -> Figure:
fig = Figure()
for ii, mode in enumerate(modes):
if "count" in mode:
df[mode] = pd.to_numeric(df[mode],errors='coerce')
color = "genus"
color_discrete_sequence=None
elif 'percent' in mode:
color = "genus"
color_discrete_sequence=None
else:
color = "target"
print(get_unique_values_in_df_column(df, 'target'))
match get_unique_values_in_df_column(df, 'target'):
case ['Target']:
color_discrete_sequence=["blue"]
case ['Off-target']:
color_discrete_sequence=['red']
case _:
color_discrete_sequence=['blue', 'red']
bar = px.bar(df, x="submitted_date",
y=mode,
color=color,
title=mode,
barmode='stack',
hover_data=["genus", "name", "target", mode],
text="genera",
color_discrete_sequence=color_discrete_sequence
)
bar.update_traces(visible = ii == 0)
fig.add_traces(bar.data)
# sys.exit(f"number of traces={len(fig.data)}")
return generic_figure_markers(fig=fig, modes=modes, ytitle=ytitle)
def construct_refseq_chart(settings:dict, df:pd.DataFrame, group_name:str, mode:str) -> Figure:
"""
Constructs intial refseq chart for both contains and matches.
Args:
settings (dict): settings passed down from click.
df (pd.DataFrame): dataframe containing all sample data for the group.
group_name (str): name of the group being processed.
mode (str): contains or matches, overwritten by hardcoding, so don't think about it too hard.
Returns:
Figure: initial figure with contains and matches traces.
"""
# This overwrites the mode from the signature, might get confusing.
fig = Figure()
modes = ['contains', 'matches']
for ii, mode in enumerate(modes):
bar = px.bar(df, x="submitted_date",
y=f"{mode}_ratio",
color="target",
title=f"{group_name}_{mode}",
barmode='stack',
hover_data=["genus", "name", f"{mode}_hashes"],
text="genera"
)
bar.update_traces(visible = ii == 0)
# Plotly express returns a full figure, so we have to use the data from that figure only.
fig.add_traces(bar.data)
# sys.exit(f"number of traces={len(fig.data)}")
return generic_figure_markers(fig=fig, modes=modes)
def construct_kraken_chart(settings:dict, df:pd.DataFrame, group_name:str, mode:str) -> Figure:
"""
Constructs intial refseq chart for each mode in the kraken config settings.
Args:
settings (dict): settings passed down from click.
df (pd.DataFrame): dataframe containing all sample data for the group.
group_name (str): name of the group being processed.
mode (str): kraken modes retrieved from config file by setup.
Returns:
Figure: initial figure with traces for modes
"""
df[f'{mode}_count'] = pd.to_numeric(df[f'{mode}_count'],errors='coerce')
# The actual percentage from kraken was off due to exclusion of NaN, recalculating.
df[f'{mode}_percent'] = 100 * df[f'{mode}_count'] / df.groupby('submitted_date')[f'{mode}_count'].transform('sum')
modes = settings['modes'][mode]
# This overwrites the mode from the signature, might get confusing.
fig = Figure()
for ii, entry in enumerate(modes):
bar = px.bar(df, x="submitted_date",
y=entry,
color="genus",
title=f"{group_name}_{entry}",
barmode="stack",
hover_data=["genus", "name", "target"],
text="genera",
)
bar.update_traces(visible = ii == 0)
fig.add_traces(bar.data)
return generic_figure_markers(fig=fig, modes=modes)
def divide_chunks(input_list:list, chunk_count:int):
"""
Divides a list into {chunk_count} equal parts
Args:
input_list (list): Initials list
chunk_count (int): size of each chunk
Returns:
tuple: tuple containing sublists.
"""
k, m = divmod(len(input_list), chunk_count)
return (input_list[i*k+min(i, m):(i+1)*k+min(i+1, m)] for i in range(chunk_count))
########This must be at bottom of module###########
function_map = {}
for item in dict(locals().items()):
try:
if dict(locals().items())[item].__module__ == __name__:
try:
function_map[item] = dict(locals().items())[item]
except KeyError:
pass
except AttributeError:
pass
###################################################