From 20952f2eddaafd9fc7ad725fb7c0f174799bfd34 Mon Sep 17 00:00:00 2001 From: lwark Date: Tue, 6 May 2025 13:21:03 -0500 Subject: [PATCH] New table view. --- src/submissions/backend/db/models/__init__.py | 6 +- src/submissions/backend/db/models/kits.py | 1 + .../backend/db/models/submissions.py | 188 +++++++++++++++++- src/submissions/backend/excel/parser.py | 17 ++ .../backend/validators/__init__.py | 2 +- src/submissions/backend/validators/pydant.py | 32 +++ src/submissions/frontend/widgets/app.py | 5 +- .../frontend/widgets/sample_checker.py | 6 +- .../frontend/widgets/submission_table.py | 114 ++++++++++- .../frontend/widgets/submission_widget.py | 28 ++- 10 files changed, 382 insertions(+), 17 deletions(-) diff --git a/src/submissions/backend/db/models/__init__.py b/src/submissions/backend/db/models/__init__.py index 4556eb4..ddc0355 100644 --- a/src/submissions/backend/db/models/__init__.py +++ b/src/submissions/backend/db/models/__init__.py @@ -221,10 +221,10 @@ class BaseClass(Base): Returns: Any | List[Any]: Single result if limit = 1 or List if other. """ - logger.debug(f"Kwargs: {kwargs}") + # logger.debug(f"Kwargs: {kwargs}") if model is None: model = cls - logger.debug(f"Model: {model}") + # logger.debug(f"Model: {model}") if query is None: query: Query = cls.__database_session__.query(model) singles = model.get_default_info('singles') @@ -516,7 +516,7 @@ from .controls import * from .organizations import * from .kits import * from .submissions import * -from .audit import * +from .audit import AuditLog # NOTE: Add a creator to the submission for reagent association. Assigned here due to circular import constraints. # https://docs.sqlalchemy.org/en/20/orm/extensions/associationproxy.html#sqlalchemy.ext.associationproxy.association_proxy.params.creator diff --git a/src/submissions/backend/db/models/kits.py b/src/submissions/backend/db/models/kits.py index 8744dd2..5702aba 100644 --- a/src/submissions/backend/db/models/kits.py +++ b/src/submissions/backend/db/models/kits.py @@ -1289,6 +1289,7 @@ class SubmissionType(BaseClass): query: Query = cls.__database_session__.query(cls) match name: case str(): + logger.debug(f"querying with {name}") query = query.filter(cls.name == name) limit = 1 case _: diff --git a/src/submissions/backend/db/models/submissions.py b/src/submissions/backend/db/models/submissions.py index 6dc0495..5eae92a 100644 --- a/src/submissions/backend/db/models/submissions.py +++ b/src/submissions/backend/db/models/submissions.py @@ -54,7 +54,7 @@ class ClientSubmission(BaseClass, LogMixin): _submission_category = Column( String(64)) #: ["Research", "Diagnostic", "Surveillance", "Validation"], else defaults to submission_type_name sample_count = Column(INTEGER) #: Number of samples in the submission - + comment = Column(JSON) runs = relationship("BasicSubmission", back_populates="client_submission") #: many-to-one relationship contact = relationship("Contact", back_populates="submissions") #: client org @@ -92,6 +92,192 @@ class ClientSubmission(BaseClass, LogMixin): except AttributeError: self._submission_category = "NA" + @classmethod + def recruit_parser(cls): + pass + + @classmethod + @setup_lookup + def query(cls, + submissiontype: str | SubmissionType | None = None, + submission_type_name: str | None = None, + id: int | str | None = None, + submitter_plate_num: str | None = None, + start_date: date | datetime | str | int | None = None, + end_date: date | datetime | str | int | None = None, + chronologic: bool = False, + limit: int = 0, + page: int = 1, + page_size: None | int = 250, + **kwargs + ) -> BasicSubmission | List[BasicSubmission]: + """ + Lookup submissions based on a number of parameters. Overrides parent. + + Args: + submission_type (str | models.SubmissionType | None, optional): Submission type of interest. Defaults to None. + id (int | str | None, optional): Submission id in the database (limits results to 1). Defaults to None. + rsl_plate_num (str | None, optional): Submission name in the database (limits results to 1). Defaults to None. + start_date (date | str | int | None, optional): Beginning date to search by. Defaults to None. + end_date (date | str | int | None, optional): Ending date to search by. Defaults to None. + reagent (models.Reagent | str | None, optional): A reagent used in the submission. Defaults to None. + chronologic (bool, optional): Return results in chronologic order. Defaults to False. + limit (int, optional): Maximum number of results to return. Defaults to 0. + + Returns: + models.BasicSubmission | List[models.BasicSubmission]: Submission(s) of interest + """ + # from ... import SubmissionReagentAssociation + # NOTE: if you go back to using 'model' change the appropriate cls to model in the query filters + query: Query = cls.__database_session__.query(cls) + if start_date is not None and end_date is None: + logger.warning(f"Start date with no end date, using today.") + end_date = date.today() + if end_date is not None and start_date is None: + # NOTE: this query returns a tuple of (object, datetime), need to get only datetime. + start_date = cls.__database_session__.query(cls, func.min(cls.submitted_date)).first()[1] + logger.warning(f"End date with no start date, using first submission date: {start_date}") + if start_date is not None: + start_date = cls.rectify_query_date(start_date) + end_date = cls.rectify_query_date(end_date, eod=True) + logger.debug(f"Start date: {start_date}, end date: {end_date}") + query = query.filter(cls.submitted_date.between(start_date, end_date)) + # NOTE: by rsl number (returns only a single value) + match submitter_plate_num: + case str(): + query = query.filter(cls.submitter_plate_num == submitter_plate_num) + limit = 1 + case _: + pass + match submission_type_name: + case str(): + query = query.filter(cls.submission_type_name == submission_type_name) + case _: + pass + # NOTE: by id (returns only a single value) + match id: + case int(): + query = query.filter(cls.id == id) + limit = 1 + case str(): + query = query.filter(cls.id == int(id)) + limit = 1 + case _: + pass + # query = query.order_by(cls.submitted_date.desc()) + # NOTE: Split query results into pages of size {page_size} + if page_size > 0: + query = query.limit(page_size) + page = page - 1 + if page is not None: + query = query.offset(page * page_size) + return cls.execute_query(query=query, model=cls, limit=limit, **kwargs) + + @classmethod + def submissions_to_df(cls, submission_type: str | None = None, limit: int = 0, + chronologic: bool = True, page: int = 1, page_size: int = 250) -> pd.DataFrame: + """ + Convert all submissions to dataframe + + Args: + page_size (int, optional): Number of items to include in query result. Defaults to 250. + page (int, optional): Limits the number of submissions to a page size. Defaults to 1. + chronologic (bool, optional): Sort submissions in chronologic order. Defaults to True. + submission_type (str | None, optional): Filter by SubmissionType. Defaults to None. + limit (int, optional): Maximum number of results to return. Defaults to 0. + + Returns: + pd.DataFrame: Pandas Dataframe of all relevant submissions + """ + # NOTE: use lookup function to create list of dicts + subs = [item.to_dict() for item in + cls.query(submissiontype=submission_type, limit=limit, chronologic=chronologic, page=page, + page_size=page_size)] + df = pd.DataFrame.from_records(subs) + # NOTE: Exclude sub information + exclude = ['controls', 'extraction_info', 'pcr_info', 'comment', 'comments', 'samples', 'reagents', + 'equipment', 'gel_info', 'gel_image', 'dna_core_submission_number', 'gel_controls', + 'source_plates', 'pcr_technician', 'ext_technician', 'artic_technician', 'cost_centre', + 'signed_by', 'artic_date', 'gel_barcode', 'gel_date', 'ngs_date', 'contact_phone', 'contact', + 'tips', 'gel_image_path', 'custom'] + # NOTE: dataframe equals dataframe of all columns not in exclude + df = df.loc[:, ~df.columns.isin(exclude)] + if chronologic: + try: + df.sort_values(by="id", axis=0, inplace=True, ascending=False) + except KeyError: + logger.error("No column named 'id'") + # NOTE: Human friendly column labels + df.columns = [item.replace("_", " ").title() for item in df.columns] + return df + + def to_dict(self, full_data: bool = False, backup: bool = False, report: bool = False) -> dict: + """ + Constructs dictionary used in submissions summary + + Args: + expand (bool, optional): indicates if generators to be expanded. Defaults to False. + report (bool, optional): indicates if to be used for a report. Defaults to False. + full_data (bool, optional): indicates if sample dicts to be constructed. Defaults to False. + backup (bool, optional): passed to adjust_to_dict_samples. Defaults to False. + + Returns: + dict: dictionary used in submissions summary and details + """ + # NOTE: get lab from nested organization object + try: + sub_lab = self.submitting_lab.name + except AttributeError: + sub_lab = None + try: + sub_lab = sub_lab.replace("_", " ").title() + except AttributeError: + pass + # NOTE: get extraction kit name from nested kit object + output = { + "id": self.id, + "submission_type": self.submission_type_name, + "submitter_plate_number": self.submitter_plate_num, + "submitted_date": self.submitted_date.strftime("%Y-%m-%d"), + "submitting_lab": sub_lab, + "sample_count": self.sample_count, + } + if report: + return output + if full_data: + # dicto, _ = self.extraction_kit.construct_xl_map_for_use(self.submission_type) + # samples = self.generate_associations(name="submission_sample_associations") + samples = None + runs = [item.to_dict() for item in self.runs] + # custom = self.custom + else: + samples = None + custom = None + runs = None + try: + comments = self.comment + except Exception as e: + logger.error(f"Error setting comment: {self.comment}, {e}") + comments = None + try: + contact = self.contact.name + except AttributeError as e: + try: + contact = f"Defaulted to: {self.submitting_lab.contacts[0].name}" + except (AttributeError, IndexError): + contact = "NA" + try: + contact_phone = self.contact.phone + except AttributeError: + contact_phone = "NA" + output["submission_category"] = self.submission_category + output["samples"] = samples + output["comment"] = comments + output["contact"] = contact + output["contact_phone"] = contact_phone + # output["custom"] = custom + output["runs"] = runs + return output class BasicSubmission(BaseClass, LogMixin): """ diff --git a/src/submissions/backend/excel/parser.py b/src/submissions/backend/excel/parser.py index 0fc7d15..e553348 100644 --- a/src/submissions/backend/excel/parser.py +++ b/src/submissions/backend/excel/parser.py @@ -546,6 +546,7 @@ class EquipmentParser(object): logger.error(f"Unable to add {eq} to list.") continue + class TipParser(object): """ Object to pull data for tips in excel sheet @@ -678,3 +679,19 @@ class ConcentrationParser(object): self.submission_obj = submission rsl_plate_num = self.submission_obj.rsl_plate_num self.samples = self.submission_obj.parse_concentration(xl=self.xl, rsl_plate_num=rsl_plate_num) + +# NOTE: Generified parsers below + +class InfoParserV2(object): + """ + Object for retrieving submitter info from sample list sheet + """ + + default_range = dict( + start_row=2, + end_row=18, + start_column=7, + end_column=8, + sheet="Sample List" + ) + diff --git a/src/submissions/backend/validators/__init__.py b/src/submissions/backend/validators/__init__.py index f672da8..17047cb 100644 --- a/src/submissions/backend/validators/__init__.py +++ b/src/submissions/backend/validators/__init__.py @@ -205,4 +205,4 @@ class RSLNamer(object): from .pydant import PydSubmission, PydKitType, PydContact, PydOrganization, PydSample, PydReagent, PydReagentRole, \ - PydEquipment, PydEquipmentRole, PydTips, PydPCRControl, PydIridaControl, PydProcess, PydElastic + PydEquipment, PydEquipmentRole, PydTips, PydPCRControl, PydIridaControl, PydProcess, PydElastic, PydClientSubmission diff --git a/src/submissions/backend/validators/pydant.py b/src/submissions/backend/validators/pydant.py index 9feb5d1..6852f6c 100644 --- a/src/submissions/backend/validators/pydant.py +++ b/src/submissions/backend/validators/pydant.py @@ -1328,3 +1328,35 @@ class PydElastic(BaseModel, extra="allow", arbitrary_types_allowed=True): field_value = getattr(self, field) self.instance.__setattr__(field, field_value) return self.instance + +# NOTE: Generified objects below: + +class PydClientSubmission(BaseModel, extra="allow"): + + filepath: Path + submission_type: dict | None + submitter_plate_num: dict | None = Field(default=dict(value=None, missing=True), validate_default=True) + submitted_date: dict | None + submitted_date: dict | None = Field(default=dict(value=date.today(), missing=True), validate_default=True) + submitting_lab: dict | None + sample_count: dict | None + kittype: dict | None + submission_category: dict | None = Field(default=dict(value=None, missing=True), validate_default=True) + comment: dict | None = Field(default=dict(value="", missing=True), validate_default=True) + cost_centre: dict | None = Field(default=dict(value=None, missing=True), validate_default=True) + contact: dict | None = Field(default=dict(value=None, missing=True), validate_default=True) + + + def to_form(self, parent: QWidget, disable: list | None = None): + """ + Converts this instance into a frontend.widgets.submission_widget.SubmissionFormWidget + + Args: + disable (list, optional): a list of widgets to be disabled in the form. Defaults to None. + parent (QWidget): parent widget of the constructed object + + Returns: + SubmissionFormWidget: Submission form widget + """ + from frontend.widgets.submission_widget import ClientSubmissionFormWidget + return ClientSubmissionFormWidget(parent=parent, submission=self, disable=disable) diff --git a/src/submissions/frontend/widgets/app.py b/src/submissions/frontend/widgets/app.py index f3fef25..b3e8701 100644 --- a/src/submissions/frontend/widgets/app.py +++ b/src/submissions/frontend/widgets/app.py @@ -22,7 +22,7 @@ from .date_type_picker import DateTypePicker from .functions import select_save_file from .pop_ups import HTMLPop from .misc import Pagifier -from .submission_table import SubmissionsSheet +from .submission_table import SubmissionsSheet, SubmissionsTree, ClientRunModel from .submission_widget import SubmissionFormContainer from .controls_chart import ControlsViewer from .summary import Summary @@ -253,7 +253,8 @@ class AddSubForm(QWidget): self.sheetwidget = QWidget(self) self.sheetlayout = QVBoxLayout(self) self.sheetwidget.setLayout(self.sheetlayout) - self.sub_wid = SubmissionsSheet(parent=parent) + # self.sub_wid = SubmissionsSheet(parent=parent) + self.sub_wid = SubmissionsTree(parent=parent, model=ClientRunModel(self)) self.pager = Pagifier(page_max=self.sub_wid.total_count / page_size) self.sheetlayout.addWidget(self.sub_wid) self.sheetlayout.addWidget(self.pager) diff --git a/src/submissions/frontend/widgets/sample_checker.py b/src/submissions/frontend/widgets/sample_checker.py index b5760b1..90296ca 100644 --- a/src/submissions/frontend/widgets/sample_checker.py +++ b/src/submissions/frontend/widgets/sample_checker.py @@ -34,7 +34,11 @@ class SampleChecker(QDialog): template_path = Path(template.environment.loader.__getattribute__("searchpath")[0]) with open(template_path.joinpath("css", "styles.css"), "r") as f: css = f.read() - html = template.render(samples=self.formatted_list, css=css) + try: + samples = self.formatted_list + except AttributeError: + samples = [] + html = template.render(samples=samples, css=css) self.webview.setHtml(html) QBtn = QDialogButtonBox.StandardButton.Ok | QDialogButtonBox.StandardButton.Cancel self.buttonBox = QDialogButtonBox(QBtn) diff --git a/src/submissions/frontend/widgets/submission_table.py b/src/submissions/frontend/widgets/submission_table.py index 9d7e463..8128e39 100644 --- a/src/submissions/frontend/widgets/submission_table.py +++ b/src/submissions/frontend/widgets/submission_table.py @@ -2,11 +2,13 @@ Contains widgets specific to the submission summary and submission details. """ import logging +import sys from pprint import pformat -from PyQt6.QtWidgets import QTableView, QMenu -from PyQt6.QtCore import Qt, QAbstractTableModel, QSortFilterProxyModel -from PyQt6.QtGui import QAction, QCursor -from backend.db.models import BasicSubmission +from PyQt6.QtWidgets import QTableView, QMenu, QTreeView, QStyledItemDelegate, QStyle, QStyleOptionViewItem, \ + QHeaderView, QAbstractItemView +from PyQt6.QtCore import Qt, QAbstractTableModel, QSortFilterProxyModel, pyqtSlot, QModelIndex +from PyQt6.QtGui import QAction, QCursor, QStandardItemModel, QStandardItem, QIcon, QColor +from backend.db.models import BasicSubmission, ClientSubmission from tools import Report, Result, report_result from .functions import select_open_file @@ -84,6 +86,7 @@ class SubmissionsSheet(QTableView): """ sets data in model """ + # self.data = ClientSubmission.submissions_to_df(page=page, page_size=page_size) self.data = BasicSubmission.submissions_to_df(page=page, page_size=page_size) try: self.data['Id'] = self.data['Id'].apply(str) @@ -222,3 +225,106 @@ class SubmissionsSheet(QTableView): sub.save() report.add_result(Result(msg=f"We added {count} logs to the database.", status='Information')) return report + + +class RunDelegate(QStyledItemDelegate): + def __init__(self, parent=None): + super(RunDelegate, self).__init__(parent) + self._plus_icon = QIcon("plus.png") + self._minus_icon = QIcon("minus.png") + + def initStyleOption(self, option, index): + super(RunDelegate, self).initStyleOption(option, index) + if not index.parent().isValid(): + is_open = bool(option.state & QStyle.StateFlag.State_Open) + option.features |= QStyleOptionViewItem.ViewItemFeature.HasDecoration + option.icon = self._minus_icon if is_open else self._plus_icon + +class SubmissionsTree(QTreeView): + """ + https://stackoverflow.com/questions/54385437/how-can-i-make-a-table-that-can-collapse-its-rows-into-categories-in-qt + """ + def __init__(self, model, parent=None): + super(SubmissionsTree, self).__init__(parent) + self.total_count = 1 + self.setIndentation(0) + self.setExpandsOnDoubleClick(False) + self.clicked.connect(self.on_clicked) + delegate = RunDelegate(self) + self.setItemDelegateForColumn(0, delegate) + self.model = model + self.setModel(self.model) + # self.header().setSectionResizeMode(0, QHeaderView.sectionResizeMode(self,0).ResizeToContents) + self.setSelectionBehavior(QAbstractItemView.selectionBehavior(self).SelectRows) + # self.setStyleSheet("background-color: #0D1225;") + self.set_data() + + @pyqtSlot(QModelIndex) + def on_clicked(self, index): + if not index.parent().isValid() and index.column() == 0: + self.setExpanded(index, not self.isExpanded(index)) + + def set_data(self, page: int = 1, page_size: int = 250) -> None: + """ + sets data in model + """ + # self.data = ClientSubmission.submissions_to_df(page=page, page_size=page_size) + self.data = [item.to_dict(full_data=True) for item in ClientSubmission.query(chronologic=True, page=page, page_size=page_size)] + logger.debug(pformat(self.data)) + # sys.exit() + for submission in self.data: + group_item = self.model.add_group(submission['submitter_plate_number']) + for run in submission['runs']: + self.model.append_element_to_group(group_item=group_item, texts=run['plate_number']) + + + def link_extractions(self): + pass + + def link_pcr(self): + pass + + +class ClientRunModel(QStandardItemModel): + def __init__(self, parent=None): + super(ClientRunModel, self).__init__(parent) + self.setColumnCount(8) + self.setHorizontalHeaderLabels(["id", "Name", "Library", "Release Date", "Genre(s)", "Last Played", "Time Played", ""]) + for i in range(self.columnCount()): + it = self.horizontalHeaderItem(i) + # it.setForeground(QColor("#F2F2F2")) + + def add_group(self, group_name): + item_root = QStandardItem() + item_root.setEditable(False) + item = QStandardItem(group_name) + item.setEditable(False) + ii = self.invisibleRootItem() + i = ii.rowCount() + for j, it in enumerate((item_root, item)): + ii.setChild(i, j, it) + ii.setEditable(False) + for j in range(self.columnCount()): + it = ii.child(i, j) + if it is None: + it = QStandardItem() + ii.setChild(i, j, it) + # it.setBackground(QColor("#002842")) + # it.setForeground(QColor("#F2F2F2")) + return item_root + + def append_element_to_group(self, group_item, texts): + j = group_item.rowCount() + item_icon = QStandardItem() + item_icon.setEditable(False) + item_icon.setIcon(QIcon("game.png")) + # item_icon.setBackground(QColor("#0D1225")) + group_item.setChild(j, 0, item_icon) + for i, text in enumerate(texts): + item = QStandardItem(text) + item.setEditable(False) + # item.setBackground(QColor("#0D1225")) + # item.setForeground(QColor("#F2F2F2")) + group_item.setChild(j, i+1, item) + + diff --git a/src/submissions/frontend/widgets/submission_widget.py b/src/submissions/frontend/widgets/submission_widget.py index 229b203..e91d740 100644 --- a/src/submissions/frontend/widgets/submission_widget.py +++ b/src/submissions/frontend/widgets/submission_widget.py @@ -10,7 +10,7 @@ from .functions import select_open_file, select_save_file import logging from pathlib import Path from tools import Report, Result, check_not_nan, main_form_style, report_result, get_application_from_parent -from backend.excel.parser import SheetParser +from backend.excel.parsers import SheetParser, InfoParserV2 from backend.validators import PydSubmission, PydReagent from backend.db import ( Organization, SubmissionType, Reagent, @@ -121,13 +121,14 @@ class SubmissionFormContainer(QWidget): return report # NOTE: create sheetparser using excel sheet and context from gui try: - self.prsr = SheetParser(filepath=fname) + # self.prsr = SheetParser(filepath=fname) + self.parser = InfoParserV2(filepath=fname) except PermissionError: logger.error(f"Couldn't get permission to access file: {fname}") return except AttributeError: - self.prsr = SheetParser(filepath=fname) - self.pyd = self.prsr.to_pydantic() + self.parser = InfoParserV2(filepath=fname) + self.pyd = self.parser.to_pydantic() # logger.debug(f"Samples: {pformat(self.pyd.samples)}") checker = SampleChecker(self, "Sample Checker", self.pyd) if checker.exec(): @@ -177,11 +178,13 @@ class SubmissionFormWidget(QWidget): self.missing_info = [] self.submission_type = SubmissionType.query(name=self.pyd.submission_type['value']) basic_submission_class = self.submission_type.submission_class + logger.debug(f"Basic submission class: {basic_submission_class}") defaults = basic_submission_class.get_default_info("form_recover", "form_ignore", submission_type=self.pyd.submission_type['value']) self.recover = defaults['form_recover'] self.ignore = defaults['form_ignore'] self.layout = QVBoxLayout() for k in list(self.pyd.model_fields.keys()) + list(self.pyd.model_extra.keys()): + logger.debug(f"Pydantic field: {k}") if k in self.ignore: logger.warning(f"{k} in form_ignore {self.ignore}, not creating widget") continue @@ -197,6 +200,7 @@ class SubmissionFormWidget(QWidget): value = self.pyd.model_extra[k] except KeyError: value = dict(value=None, missing=True) + logger.debug(f"Pydantic value: {value}") add_widget = self.create_widget(key=k, value=value, submission_type=self.submission_type, sub_obj=basic_submission_class, disable=check) if add_widget is not None: @@ -208,7 +212,8 @@ class SubmissionFormWidget(QWidget): self.layout.addWidget(self.disabler) self.disabler.checkbox.checkStateChanged.connect(self.disable_reagents) self.setStyleSheet(main_form_style) - self.scrape_reagents(self.extraction_kit) + # self.scrape_reagents(self.extraction_kit) + self.setLayout(self.layout) def disable_reagents(self): """ @@ -774,3 +779,16 @@ class SubmissionFormWidget(QWidget): layout.addWidget(self.label) layout.addWidget(self.checkbox) self.setLayout(layout) + + +class ClientSubmissionFormWidget(SubmissionFormWidget): + + def __init__(self, parent: QWidget, submission: PydSubmission, disable: list | None = None) -> None: + super().__init__(parent, submission=submission, disable=disable) + save_btn = QPushButton("Save") + start_run_btn = QPushButton("Save && Start Run") + self.layout.addWidget(save_btn) + self.layout.addWidget(start_run_btn) + + +