New table view.

src/submissions/backend/db/models/__init__.py

@@ -221,10 +221,10 @@ class BaseClass(Base):
         Returns:
             Any | List[Any]: Single result if limit = 1 or List if other.
         """
-        logger.debug(f"Kwargs: {kwargs}")
+        # logger.debug(f"Kwargs: {kwargs}")
         if model is None:
             model = cls
-        logger.debug(f"Model: {model}")
+        # logger.debug(f"Model: {model}")
         if query is None:
             query: Query = cls.__database_session__.query(model)
         singles = model.get_default_info('singles')
@@ -516,7 +516,7 @@ from .controls import *
 from .organizations import *
 from .kits import *
 from .submissions import *
-from .audit import *
+from .audit import AuditLog
 
 # NOTE: Add a creator to the submission for reagent association. Assigned here due to circular import constraints.
 # https://docs.sqlalchemy.org/en/20/orm/extensions/associationproxy.html#sqlalchemy.ext.associationproxy.association_proxy.params.creator

src/submissions/backend/db/models/kits.py

@@ -1289,6 +1289,7 @@ class SubmissionType(BaseClass):
         query: Query = cls.__database_session__.query(cls)
         match name:
             case str():
+                logger.debug(f"querying with {name}")
                 query = query.filter(cls.name == name)
                 limit = 1
             case _:

src/submissions/backend/db/models/submissions.py

@@ -54,7 +54,7 @@ class ClientSubmission(BaseClass, LogMixin):
     _submission_category = Column(
         String(64))  #: ["Research", "Diagnostic", "Surveillance", "Validation"], else defaults to submission_type_name
     sample_count = Column(INTEGER)  #: Number of samples in the submission
-
+    comment = Column(JSON)
     runs = relationship("BasicSubmission", back_populates="client_submission")  #: many-to-one relationship
 
     contact = relationship("Contact", back_populates="submissions")  #: client org
@@ -92,6 +92,192 @@ class ClientSubmission(BaseClass, LogMixin):
             except AttributeError:
                 self._submission_category = "NA"
 
+    @classmethod
+    def recruit_parser(cls):
+        pass
+
+    @classmethod
+    @setup_lookup
+    def query(cls,
+              submissiontype: str | SubmissionType | None = None,
+              submission_type_name: str | None = None,
+              id: int | str | None = None,
+              submitter_plate_num: str | None = None,
+              start_date: date | datetime | str | int | None = None,
+              end_date: date | datetime | str | int | None = None,
+              chronologic: bool = False,
+              limit: int = 0,
+              page: int = 1,
+              page_size: None | int = 250,
+              **kwargs
+              ) -> BasicSubmission | List[BasicSubmission]:
+        """
+        Lookup submissions based on a number of parameters. Overrides parent.
+
+        Args:
+            submission_type (str | models.SubmissionType | None, optional): Submission type of interest. Defaults to None.
+            id (int | str | None, optional): Submission id in the database (limits results to 1). Defaults to None.
+            rsl_plate_num (str | None, optional): Submission name in the database (limits results to 1). Defaults to None.
+            start_date (date | str | int | None, optional): Beginning date to search by. Defaults to None.
+            end_date (date | str | int | None, optional): Ending date to search by. Defaults to None.
+            reagent (models.Reagent | str | None, optional): A reagent used in the submission. Defaults to None.
+            chronologic (bool, optional): Return results in chronologic order. Defaults to False.
+            limit (int, optional): Maximum number of results to return. Defaults to 0.
+
+        Returns:
+            models.BasicSubmission | List[models.BasicSubmission]: Submission(s) of interest
+        """
+        # from ... import SubmissionReagentAssociation
+        # NOTE: if you go back to using 'model' change the appropriate cls to model in the query filters
+        query: Query = cls.__database_session__.query(cls)
+        if start_date is not None and end_date is None:
+            logger.warning(f"Start date with no end date, using today.")
+            end_date = date.today()
+        if end_date is not None and start_date is None:
+            # NOTE: this query returns a tuple of (object, datetime), need to get only datetime.
+            start_date = cls.__database_session__.query(cls, func.min(cls.submitted_date)).first()[1]
+            logger.warning(f"End date with no start date, using first submission date: {start_date}")
+        if start_date is not None:
+            start_date = cls.rectify_query_date(start_date)
+            end_date = cls.rectify_query_date(end_date, eod=True)
+            logger.debug(f"Start date: {start_date}, end date: {end_date}")
+            query = query.filter(cls.submitted_date.between(start_date, end_date))
+        # NOTE: by rsl number (returns only a single value)
+        match submitter_plate_num:
+            case str():
+                query = query.filter(cls.submitter_plate_num == submitter_plate_num)
+                limit = 1
+            case _:
+                pass
+        match submission_type_name:
+            case str():
+                query = query.filter(cls.submission_type_name == submission_type_name)
+            case _:
+                pass
+        # NOTE: by id (returns only a single value)
+        match id:
+            case int():
+                query = query.filter(cls.id == id)
+                limit = 1
+            case str():
+                query = query.filter(cls.id == int(id))
+                limit = 1
+            case _:
+                pass
+        # query = query.order_by(cls.submitted_date.desc())
+        # NOTE: Split query results into pages of size {page_size}
+        if page_size > 0:
+            query = query.limit(page_size)
+        page = page - 1
+        if page is not None:
+            query = query.offset(page * page_size)
+        return cls.execute_query(query=query, model=cls, limit=limit, **kwargs)
+
+    @classmethod
+    def submissions_to_df(cls, submission_type: str | None = None, limit: int = 0,
+                          chronologic: bool = True, page: int = 1, page_size: int = 250) -> pd.DataFrame:
+        """
+        Convert all submissions to dataframe
+
+        Args:
+            page_size (int, optional): Number of items to include in query result. Defaults to 250.
+            page (int, optional): Limits the number of submissions to a page size. Defaults to 1.
+            chronologic (bool, optional): Sort submissions in chronologic order. Defaults to True.
+            submission_type (str | None, optional): Filter by SubmissionType. Defaults to None.
+            limit (int, optional): Maximum number of results to return. Defaults to 0.
+
+        Returns:
+            pd.DataFrame: Pandas Dataframe of all relevant submissions
+        """
+        # NOTE: use lookup function to create list of dicts
+        subs = [item.to_dict() for item in
+                cls.query(submissiontype=submission_type, limit=limit, chronologic=chronologic, page=page,
+                          page_size=page_size)]
+        df = pd.DataFrame.from_records(subs)
+        # NOTE: Exclude sub information
+        exclude = ['controls', 'extraction_info', 'pcr_info', 'comment', 'comments', 'samples', 'reagents',
+                   'equipment', 'gel_info', 'gel_image', 'dna_core_submission_number', 'gel_controls',
+                   'source_plates', 'pcr_technician', 'ext_technician', 'artic_technician', 'cost_centre',
+                   'signed_by', 'artic_date', 'gel_barcode', 'gel_date', 'ngs_date', 'contact_phone', 'contact',
+                   'tips', 'gel_image_path', 'custom']
+        # NOTE: dataframe equals dataframe of all columns not in exclude
+        df = df.loc[:, ~df.columns.isin(exclude)]
+        if chronologic:
+            try:
+                df.sort_values(by="id", axis=0, inplace=True, ascending=False)
+            except KeyError:
+                logger.error("No column named 'id'")
+        # NOTE: Human friendly column labels
+        df.columns = [item.replace("_", " ").title() for item in df.columns]
+        return df
+
+    def to_dict(self, full_data: bool = False, backup: bool = False, report: bool = False) -> dict:
+        """
+        Constructs dictionary used in submissions summary
+
+        Args:
+            expand (bool, optional): indicates if generators to be expanded. Defaults to False.
+            report (bool, optional): indicates if to be used for a report. Defaults to False.
+            full_data (bool, optional): indicates if sample dicts to be constructed. Defaults to False.
+            backup (bool, optional): passed to adjust_to_dict_samples. Defaults to False.
+
+        Returns:
+            dict: dictionary used in submissions summary and details
+        """
+        # NOTE: get lab from nested organization object
+        try:
+            sub_lab = self.submitting_lab.name
+        except AttributeError:
+            sub_lab = None
+        try:
+            sub_lab = sub_lab.replace("_", " ").title()
+        except AttributeError:
+            pass
+        # NOTE: get extraction kit name from nested kit object
+        output = {
+            "id": self.id,
+            "submission_type": self.submission_type_name,
+            "submitter_plate_number": self.submitter_plate_num,
+            "submitted_date": self.submitted_date.strftime("%Y-%m-%d"),
+            "submitting_lab": sub_lab,
+            "sample_count": self.sample_count,
+        }
+        if report:
+            return output
+        if full_data:
+            # dicto, _ = self.extraction_kit.construct_xl_map_for_use(self.submission_type)
+            # samples = self.generate_associations(name="submission_sample_associations")
+            samples = None
+            runs = [item.to_dict() for item in self.runs]
+            # custom = self.custom
+        else:
+            samples = None
+            custom = None
+            runs = None
+        try:
+            comments = self.comment
+        except Exception as e:
+            logger.error(f"Error setting comment: {self.comment}, {e}")
+            comments = None
+        try:
+            contact = self.contact.name
+        except AttributeError as e:
+            try:
+                contact = f"Defaulted to: {self.submitting_lab.contacts[0].name}"
+            except (AttributeError, IndexError):
+                contact = "NA"
+        try:
+            contact_phone = self.contact.phone
+        except AttributeError:
+            contact_phone = "NA"
+        output["submission_category"] = self.submission_category
+        output["samples"] = samples
+        output["comment"] = comments
+        output["contact"] = contact
+        output["contact_phone"] = contact_phone
+        # output["custom"] = custom
+        output["runs"] = runs
+        return output
 
 class BasicSubmission(BaseClass, LogMixin):
     """

src/submissions/backend/excel/parser.py

@@ -546,6 +546,7 @@ class EquipmentParser(object):
                     logger.error(f"Unable to add {eq} to list.")
                     continue
 
+
 class TipParser(object):
     """
     Object to pull data for tips in excel sheet
@@ -678,3 +679,19 @@ class ConcentrationParser(object):
             self.submission_obj = submission
             rsl_plate_num = self.submission_obj.rsl_plate_num
         self.samples = self.submission_obj.parse_concentration(xl=self.xl, rsl_plate_num=rsl_plate_num)
+
+# NOTE: Generified parsers below
+
+class InfoParserV2(object):
+    """
+    Object for retrieving submitter info from sample list sheet
+    """
+
+    default_range = dict(
+        start_row=2,
+        end_row=18,
+        start_column=7,
+        end_column=8,
+        sheet="Sample List"
+    )
+

src/submissions/backend/validators/__init__.py

@@ -205,4 +205,4 @@ class RSLNamer(object):
 
 
 from .pydant import PydSubmission, PydKitType, PydContact, PydOrganization, PydSample, PydReagent, PydReagentRole, \
-    PydEquipment, PydEquipmentRole, PydTips, PydPCRControl, PydIridaControl, PydProcess, PydElastic
+    PydEquipment, PydEquipmentRole, PydTips, PydPCRControl, PydIridaControl, PydProcess, PydElastic, PydClientSubmission

src/submissions/backend/validators/pydant.py

@@ -1328,3 +1328,35 @@ class PydElastic(BaseModel, extra="allow", arbitrary_types_allowed=True):
                     field_value = getattr(self, field)
             self.instance.__setattr__(field, field_value)
         return self.instance
+
+# NOTE: Generified objects below:
+
+class PydClientSubmission(BaseModel, extra="allow"):
+
+    filepath: Path
+    submission_type: dict | None
+    submitter_plate_num: dict | None = Field(default=dict(value=None, missing=True), validate_default=True)
+    submitted_date: dict | None
+    submitted_date: dict | None = Field(default=dict(value=date.today(), missing=True), validate_default=True)
+    submitting_lab: dict | None
+    sample_count: dict | None
+    kittype: dict | None
+    submission_category: dict | None = Field(default=dict(value=None, missing=True), validate_default=True)
+    comment: dict | None = Field(default=dict(value="", missing=True), validate_default=True)
+    cost_centre: dict | None = Field(default=dict(value=None, missing=True), validate_default=True)
+    contact: dict | None = Field(default=dict(value=None, missing=True), validate_default=True)
+
+
+    def to_form(self, parent: QWidget, disable: list | None = None):
+        """
+        Converts this instance into a frontend.widgets.submission_widget.SubmissionFormWidget
+
+        Args:
+            disable (list, optional): a list of widgets to be disabled in the form. Defaults to None.
+            parent (QWidget): parent widget of the constructed object
+
+        Returns:
+            SubmissionFormWidget: Submission form widget
+        """
+        from frontend.widgets.submission_widget import ClientSubmissionFormWidget
+        return ClientSubmissionFormWidget(parent=parent, submission=self, disable=disable)

src/submissions/frontend/widgets/app.py

@@ -22,7 +22,7 @@ from .date_type_picker import DateTypePicker
 from .functions import select_save_file
 from .pop_ups import HTMLPop
 from .misc import Pagifier
-from .submission_table import SubmissionsSheet
+from .submission_table import SubmissionsSheet, SubmissionsTree, ClientRunModel
 from .submission_widget import SubmissionFormContainer
 from .controls_chart import ControlsViewer
 from .summary import Summary
@@ -253,7 +253,8 @@ class AddSubForm(QWidget):
         self.sheetwidget = QWidget(self)
         self.sheetlayout = QVBoxLayout(self)
         self.sheetwidget.setLayout(self.sheetlayout)
-        self.sub_wid = SubmissionsSheet(parent=parent)
+        # self.sub_wid = SubmissionsSheet(parent=parent)
+        self.sub_wid = SubmissionsTree(parent=parent, model=ClientRunModel(self))
         self.pager = Pagifier(page_max=self.sub_wid.total_count / page_size)
         self.sheetlayout.addWidget(self.sub_wid)
         self.sheetlayout.addWidget(self.pager)

src/submissions/frontend/widgets/sample_checker.py

@@ -34,7 +34,11 @@ class SampleChecker(QDialog):
         template_path = Path(template.environment.loader.__getattribute__("searchpath")[0])
         with open(template_path.joinpath("css", "styles.css"), "r") as f:
             css = f.read()
-        html = template.render(samples=self.formatted_list, css=css)
+        try:
+            samples = self.formatted_list
+        except AttributeError:
+            samples = []
+        html = template.render(samples=samples, css=css)
         self.webview.setHtml(html)
         QBtn = QDialogButtonBox.StandardButton.Ok | QDialogButtonBox.StandardButton.Cancel
         self.buttonBox = QDialogButtonBox(QBtn)

src/submissions/frontend/widgets/submission_table.py

@@ -2,11 +2,13 @@
 Contains widgets specific to the submission summary and submission details.
 """
 import logging
+import sys
 from pprint import pformat
-from PyQt6.QtWidgets import QTableView, QMenu
-from PyQt6.QtCore import Qt, QAbstractTableModel, QSortFilterProxyModel
-from PyQt6.QtGui import QAction, QCursor
-from backend.db.models import BasicSubmission
+from PyQt6.QtWidgets import QTableView, QMenu, QTreeView, QStyledItemDelegate, QStyle, QStyleOptionViewItem, \
+    QHeaderView, QAbstractItemView
+from PyQt6.QtCore import Qt, QAbstractTableModel, QSortFilterProxyModel, pyqtSlot, QModelIndex
+from PyQt6.QtGui import QAction, QCursor, QStandardItemModel, QStandardItem, QIcon, QColor
+from backend.db.models import BasicSubmission, ClientSubmission
 from tools import Report, Result, report_result
 from .functions import select_open_file
 
@@ -84,6 +86,7 @@ class SubmissionsSheet(QTableView):
         """
         sets data in model
         """
+        # self.data = ClientSubmission.submissions_to_df(page=page, page_size=page_size)
         self.data = BasicSubmission.submissions_to_df(page=page, page_size=page_size)
         try:
             self.data['Id'] = self.data['Id'].apply(str)
@@ -222,3 +225,106 @@ class SubmissionsSheet(QTableView):
             sub.save()
         report.add_result(Result(msg=f"We added {count} logs to the database.", status='Information'))
         return report
+
+
+class RunDelegate(QStyledItemDelegate):
+    def __init__(self, parent=None):
+        super(RunDelegate, self).__init__(parent)
+        self._plus_icon = QIcon("plus.png")
+        self._minus_icon = QIcon("minus.png")
+
+    def initStyleOption(self, option, index):
+        super(RunDelegate, self).initStyleOption(option, index)
+        if not index.parent().isValid():
+            is_open = bool(option.state & QStyle.StateFlag.State_Open)
+            option.features |= QStyleOptionViewItem.ViewItemFeature.HasDecoration
+            option.icon = self._minus_icon if is_open else self._plus_icon
+
+class SubmissionsTree(QTreeView):
+    """
+    https://stackoverflow.com/questions/54385437/how-can-i-make-a-table-that-can-collapse-its-rows-into-categories-in-qt
+    """
+    def __init__(self, model, parent=None):
+        super(SubmissionsTree, self).__init__(parent)
+        self.total_count = 1
+        self.setIndentation(0)
+        self.setExpandsOnDoubleClick(False)
+        self.clicked.connect(self.on_clicked)
+        delegate = RunDelegate(self)
+        self.setItemDelegateForColumn(0, delegate)
+        self.model = model
+        self.setModel(self.model)
+        # self.header().setSectionResizeMode(0, QHeaderView.sectionResizeMode(self,0).ResizeToContents)
+        self.setSelectionBehavior(QAbstractItemView.selectionBehavior(self).SelectRows)
+        # self.setStyleSheet("background-color: #0D1225;")
+        self.set_data()
+
+    @pyqtSlot(QModelIndex)
+    def on_clicked(self, index):
+        if not index.parent().isValid() and index.column() == 0:
+            self.setExpanded(index, not self.isExpanded(index))
+
+    def set_data(self, page: int = 1, page_size: int = 250) -> None:
+        """
+        sets data in model
+        """
+        # self.data = ClientSubmission.submissions_to_df(page=page, page_size=page_size)
+        self.data = [item.to_dict(full_data=True) for item in ClientSubmission.query(chronologic=True, page=page, page_size=page_size)]
+        logger.debug(pformat(self.data))
+        # sys.exit()
+        for submission in self.data:
+            group_item = self.model.add_group(submission['submitter_plate_number'])
+            for run in submission['runs']:
+                self.model.append_element_to_group(group_item=group_item, texts=run['plate_number'])
+
+
+    def link_extractions(self):
+        pass
+
+    def link_pcr(self):
+        pass
+
+
+class ClientRunModel(QStandardItemModel):
+    def __init__(self, parent=None):
+        super(ClientRunModel, self).__init__(parent)
+        self.setColumnCount(8)
+        self.setHorizontalHeaderLabels(["id", "Name", "Library", "Release Date", "Genre(s)", "Last Played", "Time Played", ""])
+        for i in range(self.columnCount()):
+            it = self.horizontalHeaderItem(i)
+            # it.setForeground(QColor("#F2F2F2"))
+
+    def add_group(self, group_name):
+        item_root = QStandardItem()
+        item_root.setEditable(False)
+        item = QStandardItem(group_name)
+        item.setEditable(False)
+        ii = self.invisibleRootItem()
+        i = ii.rowCount()
+        for j, it in enumerate((item_root, item)):
+            ii.setChild(i, j, it)
+            ii.setEditable(False)
+        for j in range(self.columnCount()):
+            it = ii.child(i, j)
+            if it is None:
+                it = QStandardItem()
+                ii.setChild(i, j, it)
+            # it.setBackground(QColor("#002842"))
+            # it.setForeground(QColor("#F2F2F2"))
+        return item_root
+
+    def append_element_to_group(self, group_item, texts):
+        j = group_item.rowCount()
+        item_icon = QStandardItem()
+        item_icon.setEditable(False)
+        item_icon.setIcon(QIcon("game.png"))
+        # item_icon.setBackground(QColor("#0D1225"))
+        group_item.setChild(j, 0, item_icon)
+        for i, text in enumerate(texts):
+            item = QStandardItem(text)
+            item.setEditable(False)
+            # item.setBackground(QColor("#0D1225"))
+            # item.setForeground(QColor("#F2F2F2"))
+            group_item.setChild(j, i+1, item)
+
+

src/submissions/frontend/widgets/submission_widget.py

@@ -10,7 +10,7 @@ from .functions import select_open_file, select_save_file
 import logging
 from pathlib import Path
 from tools import Report, Result, check_not_nan, main_form_style, report_result, get_application_from_parent
-from backend.excel.parser import SheetParser
+from backend.excel.parsers import SheetParser, InfoParserV2
 from backend.validators import PydSubmission, PydReagent
 from backend.db import (
     Organization, SubmissionType, Reagent,
@@ -121,13 +121,14 @@ class SubmissionFormContainer(QWidget):
             return report
         # NOTE: create sheetparser using excel sheet and context from gui
         try:
-            self.prsr = SheetParser(filepath=fname)
+            # self.prsr = SheetParser(filepath=fname)
+            self.parser = InfoParserV2(filepath=fname)
         except PermissionError:
             logger.error(f"Couldn't get permission to access file: {fname}")
             return
         except AttributeError:
-            self.prsr = SheetParser(filepath=fname)
-        self.pyd = self.prsr.to_pydantic()
+            self.parser = InfoParserV2(filepath=fname)
+        self.pyd = self.parser.to_pydantic()
         # logger.debug(f"Samples: {pformat(self.pyd.samples)}")
         checker = SampleChecker(self, "Sample Checker", self.pyd)
         if checker.exec():
@@ -177,11 +178,13 @@ class SubmissionFormWidget(QWidget):
         self.missing_info = []
         self.submission_type = SubmissionType.query(name=self.pyd.submission_type['value'])
         basic_submission_class = self.submission_type.submission_class
+        logger.debug(f"Basic submission class: {basic_submission_class}")
         defaults = basic_submission_class.get_default_info("form_recover", "form_ignore", submission_type=self.pyd.submission_type['value'])
         self.recover = defaults['form_recover']
         self.ignore = defaults['form_ignore']
         self.layout = QVBoxLayout()
         for k in list(self.pyd.model_fields.keys()) + list(self.pyd.model_extra.keys()):
+            logger.debug(f"Pydantic field: {k}")
             if k in self.ignore:
                 logger.warning(f"{k} in form_ignore {self.ignore}, not creating widget")
                 continue
@@ -197,6 +200,7 @@ class SubmissionFormWidget(QWidget):
                     value = self.pyd.model_extra[k]
                 except KeyError:
                     value = dict(value=None, missing=True)
+            logger.debug(f"Pydantic value: {value}")
             add_widget = self.create_widget(key=k, value=value, submission_type=self.submission_type,
                                             sub_obj=basic_submission_class, disable=check)
             if add_widget is not None:
@@ -208,7 +212,8 @@ class SubmissionFormWidget(QWidget):
                 self.layout.addWidget(self.disabler)
                 self.disabler.checkbox.checkStateChanged.connect(self.disable_reagents)
         self.setStyleSheet(main_form_style)
-        self.scrape_reagents(self.extraction_kit)
+        # self.scrape_reagents(self.extraction_kit)
+        self.setLayout(self.layout)
 
     def disable_reagents(self):
         """
@@ -774,3 +779,16 @@ class SubmissionFormWidget(QWidget):
             layout.addWidget(self.label)
             layout.addWidget(self.checkbox)
             self.setLayout(layout)
+
+
+class ClientSubmissionFormWidget(SubmissionFormWidget):
+
+    def __init__(self, parent: QWidget, submission: PydSubmission, disable: list | None = None) -> None:
+        super().__init__(parent, submission=submission, disable=disable)
+        save_btn = QPushButton("Save")
+        start_run_btn = QPushButton("Save && Start Run")
+        self.layout.addWidget(save_btn)
+        self.layout.addWidget(start_run_btn)
+
+
+