From 5fe5c222227a1942fa736759895aa689603d7e39 Mon Sep 17 00:00:00 2001 From: lwark Date: Fri, 4 Oct 2024 15:24:00 -0500 Subject: [PATCH] Added report tab with HTML and excel export. --- .../backend/db/models/submissions.py | 41 ++++---- src/submissions/backend/validators/pydant.py | 2 +- src/submissions/frontend/widgets/app.py | 10 +- src/submissions/frontend/widgets/misc.py | 46 ++++++++- .../frontend/widgets/submission_table.py | 95 +++++++------------ 5 files changed, 106 insertions(+), 88 deletions(-) diff --git a/src/submissions/backend/db/models/submissions.py b/src/submissions/backend/db/models/submissions.py index c3134fd..c32a178 100644 --- a/src/submissions/backend/db/models/submissions.py +++ b/src/submissions/backend/db/models/submissions.py @@ -13,7 +13,7 @@ from tempfile import TemporaryDirectory, TemporaryFile from operator import itemgetter from pprint import pformat from . import BaseClass, Reagent, SubmissionType, KitType, Organization, Contact -from sqlalchemy import Column, String, TIMESTAMP, INTEGER, ForeignKey, JSON, FLOAT, case +from sqlalchemy import Column, String, TIMESTAMP, INTEGER, ForeignKey, JSON, FLOAT, case, desc from sqlalchemy.orm import relationship, validates, Query from sqlalchemy.orm.attributes import flag_modified from sqlalchemy.ext.associationproxy import association_proxy @@ -283,7 +283,7 @@ class BasicSubmission(BaseClass): del input_dict['id'] return input_dict - def generate_associations(self, name:str, extra:str|None=None): + def generate_associations(self, name: str, extra: str | None = None): try: field = self.__getattribute__(name) except AttributeError: @@ -479,15 +479,6 @@ class BasicSubmission(BaseClass): Returns: str: html output string. """ - # output_samples = [] - # logger.debug("Setting locations.") - # for column in range(1, plate_columns + 1): - # for row in range(1, plate_rows + 1): - # try: - # well = next((item for item in sample_list if item['row'] == row and item['column'] == column), dict(name="", row=row, column=column, background_color="#ffffff")) - # except StopIteration: - # well = dict(name="", row=row, column=column, background_color="#ffffff") - # output_samples.append(well) rows = range(1, plate_rows + 1) columns = range(1, plate_columns + 1) # NOTE: An overly complicated list comprehension create a list of sample locations @@ -512,11 +503,12 @@ class BasicSubmission(BaseClass): @classmethod def submissions_to_df(cls, submission_type: str | None = None, limit: int = 0, - chronologic: bool = True) -> pd.DataFrame: + chronologic: bool = True, page: int = 1, page_size: int = 250) -> pd.DataFrame: """ Convert all submissions to dataframe Args: + page (int, optional): Limits the number of submissions to a page size. Defaults to 1. chronologic (bool, optional): Sort submissions in chronologic order. Defaults to True. submission_type (str | None, optional): Filter by SubmissionType. Defaults to None. limit (int, optional): Maximum number of results to return. Defaults to 0. @@ -528,16 +520,16 @@ class BasicSubmission(BaseClass): # logger.debug(f"Using limit: {limit}") # NOTE: use lookup function to create list of dicts subs = [item.to_dict() for item in - cls.query(submission_type=submission_type, limit=limit, chronologic=chronologic)] + cls.query(submission_type=submission_type, limit=limit, chronologic=chronologic, page=page, page_size=page_size)] # logger.debug(f"Got {len(subs)} submissions.") df = pd.DataFrame.from_records(subs) # logger.debug(f"Column names: {df.columns}") # NOTE: Exclude sub information exclude = ['controls', 'extraction_info', 'pcr_info', 'comment', 'comments', 'samples', 'reagents', - 'equipment', 'gel_info', 'gel_image', 'dna_core_submission_number', 'gel_controls', - 'source_plates', 'pcr_technician', 'ext_technician', 'artic_technician', 'cost_centre', - 'signed_by', 'artic_date', 'gel_barcode', 'gel_date', 'ngs_date', 'contact_phone', 'contact', - 'tips', 'gel_image_path', 'custom'] + 'equipment', 'gel_info', 'gel_image', 'dna_core_submission_number', 'gel_controls', + 'source_plates', 'pcr_technician', 'ext_technician', 'artic_technician', 'cost_centre', + 'signed_by', 'artic_date', 'gel_barcode', 'gel_date', 'ngs_date', 'contact_phone', 'contact', + 'tips', 'gel_image_path', 'custom'] df = df.loc[:, ~df.columns.isin(exclude)] # for item in excluded: # try: @@ -829,7 +821,7 @@ class BasicSubmission(BaseClass): return input_dict @classmethod - def custom_validation(cls, pyd:"PydSubmission") -> dict: + def custom_validation(cls, pyd: "PydSubmission") -> dict: """ Performs any final custom parsing of the excel file. @@ -886,7 +878,6 @@ class BasicSubmission(BaseClass): ws.cell(row=item['row'], column=item['column'], value=item['value']) return input_excel - @classmethod def enforce_name(cls, instr: str, data: dict | None = {}) -> str: """ @@ -1061,6 +1052,8 @@ class BasicSubmission(BaseClass): reagent: Reagent | str | None = None, chronologic: bool = False, limit: int = 0, + page: int = 1, + page_size: int = 250, **kwargs ) -> BasicSubmission | List[BasicSubmission]: """ @@ -1161,7 +1154,13 @@ class BasicSubmission(BaseClass): case _: pass if chronologic: - query.order_by(cls.submitted_date) + logger.debug("Attempting sort by date descending") + query = query.order_by(cls.submitted_date.desc()) + if page_size is not None: + query = query.limit(page_size) + page = page - 1 + if page is not None: + query = query.offset(page * page_size) return cls.execute_query(query=query, model=model, limit=limit, **kwargs) @classmethod @@ -2483,7 +2482,7 @@ class BasicSample(BaseClass): df = pd.DataFrame.from_records(samples) # NOTE: Exclude sub information exclude = ['concentration', 'organism', 'colour', 'tooltip', 'comments', 'samples', 'reagents', - 'equipment', 'gel_info', 'gel_image', 'dna_core_submission_number', 'gel_controls'] + 'equipment', 'gel_info', 'gel_image', 'dna_core_submission_number', 'gel_controls'] df = df.loc[:, ~df.columns.isin(exclude)] return df diff --git a/src/submissions/backend/validators/pydant.py b/src/submissions/backend/validators/pydant.py index b6acdac..98946cc 100644 --- a/src/submissions/backend/validators/pydant.py +++ b/src/submissions/backend/validators/pydant.py @@ -611,7 +611,7 @@ class PydSubmission(BaseModel, extra='allow'): @classmethod def expand_samples(cls, value): if isinstance(value, Generator): - # logger.debug("We have a generator") + # logger.debug("We have a generator")[ return [PydSample(**sample) for sample in value] return value diff --git a/src/submissions/frontend/widgets/app.py b/src/submissions/frontend/widgets/app.py index e3c361d..0fceabd 100644 --- a/src/submissions/frontend/widgets/app.py +++ b/src/submissions/frontend/widgets/app.py @@ -12,11 +12,11 @@ from pathlib import Path from markdown import markdown from __init__ import project_path -from tools import check_if_app, Settings, Report, jinja_template_loading, check_authorization +from tools import check_if_app, Settings, Report, jinja_template_loading, check_authorization, page_size from .functions import select_save_file,select_open_file from datetime import date from .pop_ups import HTMLPop, AlertPop -from .misc import LogParser +from .misc import LogParser, Pagifier import logging, webbrowser, sys, shutil from .submission_table import SubmissionsSheet from .submission_widget import SubmissionFormContainer @@ -49,6 +49,7 @@ class App(QMainWindow): self.height = 1000 self.setWindowTitle(self.title) self.setGeometry(self.left, self.top, self.width, self.height) + self.page_size = page_size # NOTE: insert tabs into main app self.table_widget = AddSubForm(self) self.setCentralWidget(self.table_widget) @@ -133,6 +134,7 @@ class App(QMainWindow): self.githubAction.triggered.connect(self.openGithub) self.yamlExportAction.triggered.connect(self.export_ST_yaml) self.yamlImportAction.triggered.connect(self.import_ST_yaml) + self.table_widget.pager.current_page.textChanged.connect(self.update_data) def showAbout(self): """ @@ -237,6 +239,8 @@ class App(QMainWindow): else: logger.warning("Save of submission type cancelled.") + def update_data(self): + self.table_widget.sub_wid.setData(page=int(self.table_widget.pager.current_page.text()), page_size=page_size) class AddSubForm(QWidget): @@ -272,7 +276,9 @@ class AddSubForm(QWidget): self.sheetlayout = QVBoxLayout(self) self.sheetwidget.setLayout(self.sheetlayout) self.sub_wid = SubmissionsSheet(parent=parent) + self.pager = Pagifier(page_max=self.sub_wid.total_count/page_size) self.sheetlayout.addWidget(self.sub_wid) + self.sheetlayout.addWidget(self.pager) # NOTE: Create layout of first tab to hold form and sheet self.tab1.layout = QHBoxLayout(self) self.tab1.setLayout(self.tab1.layout) diff --git a/src/submissions/frontend/widgets/misc.py b/src/submissions/frontend/widgets/misc.py index 89c3db8..1710bfd 100644 --- a/src/submissions/frontend/widgets/misc.py +++ b/src/submissions/frontend/widgets/misc.py @@ -1,21 +1,22 @@ ''' Contains miscellaneous widgets for frontend functions ''' +import math from datetime import date -from PyQt6.QtGui import QPageLayout, QPageSize, QStandardItem, QStandardItemModel +from PyQt6.QtGui import QPageLayout, QPageSize, QStandardItem, QIcon from PyQt6.QtWebEngineWidgets import QWebEngineView from PyQt6.QtWidgets import ( QLabel, QVBoxLayout, QLineEdit, QComboBox, QDialog, QDialogButtonBox, QDateEdit, QPushButton, QFormLayout, QWidget, QHBoxLayout, QSizePolicy ) -from PyQt6.QtCore import Qt, QDate, QSize, QMarginsF, pyqtSlot, pyqtSignal, QEvent +from PyQt6.QtCore import Qt, QDate, QSize, QMarginsF from tools import jinja_template_loading from backend.db.models import * import logging from .pop_ups import AlertPop -from .functions import select_open_file, select_save_file +from .functions import select_open_file logger = logging.getLogger(f"submissions.{__name__}") @@ -245,3 +246,42 @@ class CheckableComboBox(QComboBox): def changed(self): logger.debug("emitting updated") self.updated.emit() + + +class Pagifier(QWidget): + + def __init__(self, page_max:int): + super().__init__() + self.page_max = math.ceil(page_max) + + next = QPushButton(parent=self, icon = QIcon.fromTheme(QIcon.ThemeIcon.GoNext)) + next.pressed.connect(self.increment_page) + previous = QPushButton(parent=self, icon=QIcon.fromTheme(QIcon.ThemeIcon.GoPrevious)) + previous.pressed.connect(self.decrement_page) + label = QLabel(f"/ {self.page_max}") + label.setMinimumWidth(200) + label.setAlignment(Qt.AlignmentFlag.AlignCenter) + self.current_page = QLineEdit(self) + self.current_page.setEnabled(False) + # onlyInt = QIntValidator() + # onlyInt.setRange(1, 4) + # self.current_page.setValidator(onlyInt) + self.current_page.setText("1") + self.current_page.setAlignment(Qt.AlignmentFlag.AlignCenter) + self.layout = QHBoxLayout() + self.layout.addWidget(previous) + self.layout.addWidget(self.current_page) + self.layout.addWidget(label) + self.layout.addWidget(next) + self.setLayout(self.layout) + + def increment_page(self): + new = int(self.current_page.text())+1 + if new <= self.page_max: + self.current_page.setText(str(new)) + + def decrement_page(self): + new = int(self.current_page.text())-1 + if new >= 1: + self.current_page.setText(str(new)) + diff --git a/src/submissions/frontend/widgets/submission_table.py b/src/submissions/frontend/widgets/submission_table.py index a7328ba..e8db9e7 100644 --- a/src/submissions/frontend/widgets/submission_table.py +++ b/src/submissions/frontend/widgets/submission_table.py @@ -7,9 +7,9 @@ from PyQt6.QtWidgets import QTableView, QMenu from PyQt6.QtCore import Qt, QAbstractTableModel, QSortFilterProxyModel from PyQt6.QtGui import QAction, QCursor from backend.db.models import BasicSubmission -from backend.excel import ReportMaker from tools import Report, Result, report_result -from .functions import select_save_file, select_open_file +from .functions import select_open_file + # from .misc import ReportDatePicker logger = logging.getLogger(f"submissions.{__name__}") @@ -20,6 +20,7 @@ class pandasModel(QAbstractTableModel): pandas model for inserting summary sheet into gui NOTE: Copied from Stack Overflow. I have no idea how it actually works. """ + def __init__(self, data) -> None: QAbstractTableModel.__init__(self) self._data = data @@ -45,10 +46,10 @@ class pandasModel(QAbstractTableModel): Returns: int: number of columns in data - """ + """ return self._data.shape[1] - def data(self, index, role=Qt.ItemDataRole.DisplayRole) -> str|None: + def data(self, index, role=Qt.ItemDataRole.DisplayRole) -> str | None: if index.isValid(): if role == Qt.ItemDataRole.DisplayRole: return str(self._data.iloc[index.row(), index.column()]) @@ -63,28 +64,30 @@ class pandasModel(QAbstractTableModel): class SubmissionsSheet(QTableView): """ presents submission summary to user in tab1 - """ + """ + def __init__(self, parent) -> None: """ initialize Args: ctx (dict): settings passed from gui - """ + """ super().__init__(parent) self.app = self.parent() self.report = Report() - self.setData() + self.setData(page=1, page_size=self.app.page_size) self.resizeColumnsToContents() self.resizeRowsToContents() self.setSortingEnabled(True) self.doubleClicked.connect(lambda x: BasicSubmission.query(id=x.sibling(x.row(), 0).data()).show_details(self)) - - def setData(self) -> None: + self.total_count = BasicSubmission.__database_session__.query(BasicSubmission).count() + + def setData(self, page: int = 1, page_size: int = 250) -> None: """ sets data in model - """ - self.data = BasicSubmission.submissions_to_df() + """ + self.data = BasicSubmission.submissions_to_df(page=page) try: self.data['Id'] = self.data['Id'].apply(str) self.data['Id'] = self.data['Id'].str.zfill(4) @@ -93,17 +96,17 @@ class SubmissionsSheet(QTableView): proxyModel = QSortFilterProxyModel() proxyModel.setSourceModel(pandasModel(self.data)) self.setModel(proxyModel) - + def contextMenuEvent(self, event): """ Creates actions for right click menu events. Args: event (_type_): the item of interest - """ + """ # logger.debug(event().__dict__) id = self.selectionModel().currentIndex() - id = id.sibling(id.row(),0).data() + id = id.sibling(id.row(), 0).data() submission = BasicSubmission.query(id=id) self.menu = QMenu(self) self.con_actions = submission.custom_context_events() @@ -115,13 +118,13 @@ class SubmissionsSheet(QTableView): # add other required actions self.menu.popup(QCursor.pos()) - def triggered_action(self, action_name:str): + def triggered_action(self, action_name: str): """ Calls the triggered action from the context menu Args: action_name (str): name of the action from the menu - """ + """ # logger.debug(f"Action: {action_name}") # logger.debug(f"Responding with {self.con_actions[action_name]}") func = self.con_actions[action_name] @@ -155,16 +158,16 @@ class SubmissionsSheet(QTableView): count = 0 for run in runs: new_run = dict( - start_time=run[0].strip(), - rsl_plate_num=run[1].strip(), - sample_count=run[2].strip(), - status=run[3].strip(), - experiment_name=run[4].strip(), - end_time=run[5].strip() - ) + start_time=run[0].strip(), + rsl_plate_num=run[1].strip(), + sample_count=run[2].strip(), + status=run[3].strip(), + experiment_name=run[4].strip(), + end_time=run[5].strip() + ) # NOTE: elution columns are item 6 in the comma split list to the end for ii in range(6, len(run)): - new_run[f"column{str(ii-5)}_vol"] = run[ii] + new_run[f"column{str(ii - 5)}_vol"] = run[ii] # NOTE: Lookup imported submissions sub = BasicSubmission.query(rsl_plate_num=new_run['rsl_plate_num']) # NOTE: If no such submission exists, move onto the next run @@ -208,13 +211,13 @@ class SubmissionsSheet(QTableView): count = 0 for run in runs: new_run = dict( - start_time=run[0].strip(), - rsl_plate_num=run[1].strip(), - biomek_status=run[2].strip(), - quant_status=run[3].strip(), - experiment_name=run[4].strip(), - end_time=run[5].strip() - ) + start_time=run[0].strip(), + rsl_plate_num=run[1].strip(), + biomek_status=run[2].strip(), + quant_status=run[3].strip(), + experiment_name=run[4].strip(), + end_time=run[5].strip() + ) # NOTE: lookup imported submission sub = BasicSubmission.query(rsl_number=new_run['rsl_plate_num']) # NOTE: if imported submission doesn't exist move on to next run @@ -225,33 +228,3 @@ class SubmissionsSheet(QTableView): sub.save() report.add_result(Result(msg=f"We added {count} logs to the database.", status='Information')) return report - - # @report_result - # def generate_report(self, *args): - # """ - # Make a report - # """ - # report = Report() - # result = self.generate_report_function() - # report.add_result(result) - # return report - # - # def generate_report_function(self): - # """ - # Generate a summary of activities for a time period - # - # Args: - # obj (QMainWindow): original app window - # - # Returns: - # Tuple[QMainWindow, dict]: Collection of new main app window and result dict - # """ - # report = Report() - # # NOTE: ask for date ranges - # dlg = ReportDatePicker() - # if dlg.exec(): - # info = dlg.parse_form() - # fname = select_save_file(obj=self, default_name=f"Submissions_Report_{info['start_date']}-{info['end_date']}.xlsx", extension="xlsx") - # rp = ReportMaker(start_date=info['start_date'], end_date=info['end_date']) - # rp.write_report(filename=fname, obj=self) - # return report