Attempt to cleanup imports.

2025-09-12 10:14:53 -05:00
parent ba4912cab7
commit 11abaafcfc
23 changed files with 322 additions and 323 deletions
--- a/src/submissions/backend/init.py
+++ b/src/submissions/backend/init.py
@@ -1,6 +1,32 @@
 """
 Contains database, validators and excel operations.
 """
-from .db import *
-from .excel import *
-from .validators import *
+from .db import (
+    set_sqlite_pragma,
+    LogMixin, ConfigItem,
+    AuditLog,
+    ControlType, Control,
+    ClientLab, Contact,
+    ReagentRole, Reagent, ReagentLot, Discount, SubmissionType, ProcedureType, Procedure, ProcedureTypeReagentRoleAssociation,
+    ProcedureReagentLotAssociation, EquipmentRole, Equipment, EquipmentRoleEquipmentAssociation, Process, ProcessVersion,
+    Tips, TipsLot, ProcedureEquipmentAssociation,
+    ProcedureTypeEquipmentRoleAssociation, Results,
+    ClientSubmission, Run, Sample, ClientSubmissionSampleAssociation, RunSampleAssociation, ProcedureSampleAssociation,
+    update_log
+)
+from .excel import (
+    DefaultParser, DefaultKEYVALUEParser, DefaultTABLEParser, ProcedureInfoParser, ProcedureSampleParser,
+    ProcedureReagentParser, ProcedureEquipmentParser, DefaultResultsInfoParser, DefaultResultsSampleParser,
+    PCRSampleParser, PCRInfoParser, ClientSubmissionSampleParser, ClientSubmissionInfoParser, PCRInfoParser,
+    PCRSampleParser,
+    DefaultWriter, DefaultKEYVALUEWriter, DefaultTABLEWriter,
+    ProcedureInfoWriter, ProcedureSampleWriter, ProcedureReagentWriter, ProcedureEquipmentWriter,
+    PCRInfoWriter, PCRSampleWriter,
+    ClientSubmissionInfoWriter, ClientSubmissionSampleWriter,
+    ReportArchetype, ReportMaker, TurnaroundMaker, ConcentrationMaker, ChartReportMaker
+)
+from .validators import (
+    DefaultNamer, ClientSubmissionNamer, RSLNamer,
+    PydRun, PydContact, PydClientLab, PydSample, PydReagent, PydReagentRole, PydEquipment, PydEquipmentRole, PydTips,
+    PydProcess, PydElastic, PydClientSubmission, PydProcedure, PydResults, PydReagentLot
+)
--- a/src/submissions/backend/db/init.py
+++ b/src/submissions/backend/db/init.py
@@ -1,9 +1,14 @@
 """
 All database related operations.
 """
+from datetime import datetime
+from getpass import getuser
 from sqlalchemy import event, inspect
 from sqlalchemy.engine import Engine
 from tools import ctx
+import logging
+
+logger = logging.getLogger(f"submissions.{__name__}")


@event.listens_for(Engine, "connect")
@@ -28,7 +33,17 @@ def set_sqlite_pragma(dbapi_connection, connection_record):
    cursor.close()


-from .models import *
+from .models import (
+    LogMixin, ConfigItem,
+    AuditLog,
+    ReagentRole, Reagent, ReagentLot, Discount, SubmissionType, ProcedureType, Procedure, ProcedureTypeReagentRoleAssociation,
+    ProcedureReagentLotAssociation, EquipmentRole, Equipment, EquipmentRoleEquipmentAssociation, Process, ProcessVersion,
+    Tips, TipsLot, ProcedureEquipmentAssociation,
+    ProcedureTypeEquipmentRoleAssociation, Results,
+    ClientSubmission, Run, Sample, ClientSubmissionSampleAssociation, RunSampleAssociation, ProcedureSampleAssociation,
+    ControlType, Control,
+    ClientLab, Contact
+)


 def update_log(mapper, connection, target):
--- a/src/submissions/backend/db/models/init.py
+++ b/src/submissions/backend/db/models/init.py
@@ -3,11 +3,14 @@ Contains all models for sqlalchemy
 """
 from __future__ import annotations
 import sys, logging, json, inspect
+from datetime import datetime, date
+from pprint import pformat
+
 from dateutil.parser import parse
-from jinja2 import TemplateNotFound
+from jinja2 import TemplateNotFound, Template
 from pandas import DataFrame
 from pydantic import BaseModel
-from sqlalchemy import Column, INTEGER, String, JSON, DATETIME
+from sqlalchemy import Column, INTEGER, String, JSON, TIMESTAMP
 from sqlalchemy.ext.associationproxy import AssociationProxy
 from sqlalchemy.orm import DeclarativeMeta, declarative_base, Query, Session, InstrumentedAttribute, ColumnProperty
 from sqlalchemy.ext.declarative import declared_attr
@@ -16,7 +19,7 @@ from sqlalchemy.exc import ArgumentError
 from typing import Any, List, ClassVar
 from pathlib import Path
 from sqlalchemy.orm.relationships import _RelationshipDeclared
-from tools import report_result, list_sort_dict
+from tools import report_result, list_sort_dict, jinja_template_loading, Report, Result

 # NOTE: Load testing environment
 if 'pytest' in sys.modules:
@@ -46,7 +49,9 @@ class BaseClass(Base):
        except AttributeError:
            return f"<{self.__class__.__name__}(Name Unavailable)>"

-    @classproperty
+    # @classproperty
+    @classmethod
+    @declared_attr
    def aliases(cls) -> List[str]:
        """
        List of other names this class might be known by.
@@ -56,7 +61,8 @@ class BaseClass(Base):
        """
        return [cls.query_alias]

-    @classproperty
+    @classmethod
+    @declared_attr
    def query_alias(cls) -> str:
        """
        What to query this class as.
@@ -126,7 +132,9 @@ class BaseClass(Base):
        super().__init__(*args, **kwargs)
        self._misc_info = dict()

-    @classproperty
+    # @classproperty
+    @classmethod
+    @declared_attr
    def jsons(cls) -> List[str]:
        """
        Get list of JSON db columns
@@ -139,7 +147,9 @@ class BaseClass(Base):
        except AttributeError:
            return []

-    @classproperty
+    # @classproperty
+    @classmethod
+    @declared_attr
    def timestamps(cls) -> List[str]:
        """
        Get list of TIMESTAMP columns
@@ -169,9 +179,6 @@ class BaseClass(Base):
                continue
            if not isinstance(item[1].property, ColumnProperty):
                continue
-            # if isinstance(item[1], _RelationshipDeclared):
-            #     if "association" in item[0]:
-            #         continue
            if len(item[1].foreign_keys) > 0:
                continue
            if item[1].type.__class__.__name__ not in ["String"]:
@@ -245,14 +252,15 @@ class BaseClass(Base):
        """
        if not objects:
            try:
-                records = [obj.details_dict(**kwargs) for obj in cls.query()]
+                q = cls.query()
            except AttributeError:
-                records = [obj.details_dict(**kwargs) for obj in cls.query(page_size=0)]
+                q = cls.query(page_size=0)
        else:
-            try:
-                records = [obj.to_sub_dict(**kwargs) for obj in objects]
-            except AttributeError:
-                records = [{k: v['instance_attr'] for k, v in obj.omnigui_instance_dict.items()} for obj in objects]
+            q = objects
+        records = []
+        for obj in q:
+            dicto = obj.details_dict(**kwargs)
+            records.append({key: value for key, value in dicto.items() if key not in dicto['excluded']})
        return DataFrame.from_records(records)

    @classmethod
@@ -397,7 +405,9 @@ class BaseClass(Base):
            pass
        return dicto

-    @classproperty
+    # @classproperty
+    @classmethod
+    @declared_attr
    def pydantic_model(cls) -> BaseModel:
        """
        Gets the pydantic model corresponding to this object.
@@ -410,10 +420,15 @@ class BaseClass(Base):
            model = getattr(pydant, f"Pyd{cls.__name__}")
        except AttributeError:
            logger.warning(f"Couldn't get {cls.__name__} pydantic model.")
-            return pydant.PydElastic
+            try:
+                model = getattr(pydant, f"Pyd{cls.pyd_model_name}")
+            except AttributeError:
+                return pydant.PydElastic
        return model

-    @classproperty
+    # @classproperty
+    @classmethod
+    @declared_attr
    def add_edit_tooltips(cls) -> dict:
        """
        Gets tooltips for Omni-add-edit
@@ -423,7 +438,9 @@ class BaseClass(Base):
        """
        return dict()

-    @classproperty
+    # @classproperty
+    @classmethod
+    @declared_attr
    def details_template(cls) -> Template:
        """
        Get the details jinja template for the correct class
@@ -609,6 +626,11 @@ class BaseClass(Base):
                value = getattr(self, k)
            except AttributeError:
                continue
+            match value:
+                case str():
+                    value = value.strip('\"')
+                case _:
+                    pass
            output[k.strip("_")] = value
        if self._misc_info:
            for key, value in self._misc_info.items():
@@ -718,12 +740,20 @@ class ConfigItem(BaseClass):
        return config_items


-from .controls import *
 # NOTE: import order must go: orgs, kittype, run due to circular import issues
-from .organizations import *
-from .procedures import *
-from .submissions import *
 from .audit import AuditLog
+from .organizations import (
+    ClientLab, Contact, BaseClass # NOTE: For some reason I  need to import BaseClass at this point for queries to work.
+)
+from .procedures import (
+    ReagentRole, Reagent, ReagentLot, Discount, SubmissionType, ProcedureType, Procedure, ProcedureTypeReagentRoleAssociation,
+    ProcedureReagentLotAssociation, EquipmentRole, Equipment, EquipmentRoleEquipmentAssociation, Process, ProcessVersion,
+    Tips, TipsLot, ProcedureEquipmentAssociation, ProcedureTypeEquipmentRoleAssociation, Results
+)
+from .submissions import (
+    ClientSubmission, Run, Sample, ClientSubmissionSampleAssociation, RunSampleAssociation, ProcedureSampleAssociation
+)
+from .controls import ControlType, Control

 # NOTE: Add a creator to the procedure for reagent association. Assigned here due to circular import constraints.
 # https://docs.sqlalchemy.org/en/20/orm/extensions/associationproxy.html#sqlalchemy.ext.associationproxy.association_proxy.params.creator
--- a/src/submissions/backend/db/models/audit.py
+++ b/src/submissions/backend/db/models/audit.py
@@ -11,6 +11,7 @@ import logging

 logger = logging.getLogger(f"submissions.{__name__}")

+# NOTE: Need a seperate base for this.
 Base: DeclarativeMeta = declarative_base()

 class AuditLog(Base):
--- a/src/submissions/backend/db/models/organizations.py
+++ b/src/submissions/backend/db/models/organizations.py
@@ -4,8 +4,8 @@ All client organization related models.
 from __future__ import annotations
 import logging
 from sqlalchemy import Column, String, INTEGER, ForeignKey, Table
-from sqlalchemy.orm import relationship, Query
-from . import Base, BaseClass
+from sqlalchemy.orm import relationship, Query, declared_attr
+from . import BaseClass
 from tools import check_authorization, setup_lookup
 from typing import List

@@ -14,7 +14,8 @@ logger = logging.getLogger(f"submissions.{__name__}")
 # table containing clientlab/contact relationship
 clientlab_contact = Table(
    "_clientlab_contact",
-    Base.metadata,
+    # Base.metadata,
+    BaseClass.__base__.metadata,
    Column("clientlab_id", INTEGER, ForeignKey("_clientlab.id")),
    Column("contact_id", INTEGER, ForeignKey("_contact.id")),
    extend_existing=True
@@ -98,7 +99,9 @@ class Contact(BaseClass):
                             secondary=clientlab_contact)  #: relationship to joined clientlab
    clientsubmission = relationship("ClientSubmission", back_populates="contact")  #: procedure this contact has submitted

-    @classproperty
+    # @classproperty
+    @classmethod
+    @declared_attr
    def searchables(cls):
        return []

--- a/src/submissions/backend/db/models/procedures.py
+++ b/src/submissions/backend/db/models/procedures.py
@@ -7,13 +7,13 @@ from operator import itemgetter
 from pprint import pformat
 from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey, Interval, Table, FLOAT, func
 from sqlalchemy.ext.hybrid import hybrid_property
-from sqlalchemy.orm import relationship, validates, Query
+from sqlalchemy.orm import relationship, validates, Query, declared_attr
 from sqlalchemy.ext.associationproxy import association_proxy
 from datetime import date, datetime, timedelta
 from tools import check_authorization, setup_lookup, Report, Result, check_regex_match, timezone, \
    jinja_template_loading, flatten_list
 from typing import List, Literal, Generator, Any, Tuple, TYPE_CHECKING
-from . import Base, BaseClass, ClientLab, LogMixin
+from . import BaseClass, ClientLab, LogMixin
 from sqlalchemy.exc import OperationalError as AlcOperationalError, IntegrityError as AlcIntegrityError
 from sqlite3 import OperationalError as SQLOperationalError, IntegrityError as SQLIntegrityError

@@ -25,7 +25,7 @@ logger = logging.getLogger(f'submissions.{__name__}')

 reagentrole_reagent = Table(
    "_reagentrole_reagent",
-    Base.metadata,
+    BaseClass.__base__.metadata,
    Column("reagent_id", INTEGER, ForeignKey("_reagent.id")),
    Column("reagentrole_id", INTEGER, ForeignKey("_reagentrole.id")),
    extend_existing=True
@@ -33,7 +33,7 @@ reagentrole_reagent = Table(

 equipment_process = Table(
    "_equipment_process",
-    Base.metadata,
+    BaseClass.__base__.metadata,
    Column("process_id", INTEGER, ForeignKey("_process.id")),
    Column("equipment_id", INTEGER, ForeignKey("_equipment.id")),
    extend_existing=True
@@ -41,7 +41,7 @@ equipment_process = Table(

 process_tips = Table(
    "_process_tips",
-    Base.metadata,
+    BaseClass.__base__.metadata,
    Column("process_id", INTEGER, ForeignKey("_process.id")),
    Column("tips_id", INTEGER, ForeignKey("_tips.id")),
    extend_existing=True
@@ -49,7 +49,7 @@ process_tips = Table(

 submissiontype_proceduretype = Table(
    "_submissiontype_proceduretype",
-    Base.metadata,
+    BaseClass.__base__.metadata,
    Column("submissiontype_id", INTEGER, ForeignKey("_submissiontype.id")),
    Column("proceduretype_id", INTEGER, ForeignKey("_proceduretype.id")),
    extend_existing=True
@@ -217,7 +217,8 @@ class Reagent(BaseClass, LogMixin):
        self.name = name
        self.eol_ext = eol_ext

-    @classproperty
+    @classmethod
+    @declared_attr
    def searchables(cls):
        return [dict(label="Lot", field="lot")]

@@ -322,7 +323,8 @@ class Reagent(BaseClass, LogMixin):



-    @classproperty
+    @classmethod
+    @declared_attr
    def add_edit_tooltips(self):
        return dict(
            expiry="Use exact date on reagent.\nEOL will be calculated from kittype automatically"
@@ -342,6 +344,9 @@ class Reagent(BaseClass, LogMixin):


 class ReagentLot(BaseClass):
+
+    pyd_model_name = "Reagent"
+
    id = Column(INTEGER, primary_key=True)  #: primary key
    lot = Column(String(64), unique=True)  #: lot number of reagent
    expiry = Column(TIMESTAMP)  #: expiry date - extended by eol_ext of parent programmatically
@@ -368,7 +373,7 @@ class ReagentLot(BaseClass):
    def query(cls,
              lot: str | None = None,
              name: str | None = None,
-              limit: int = 1,
+              limit: int = 0,
              **kwargs) -> ReagentLot | List[ReagentLot]:
        """

@@ -386,6 +391,7 @@ class ReagentLot(BaseClass):
        match lot:
            case str():
                query = query.filter(cls.lot == lot)
+                limit = 1
            case _:
                pass
        match name:
@@ -414,13 +420,28 @@ class ReagentLot(BaseClass):
    @check_authorization
    def edit_from_search(self, obj, **kwargs):
        from frontend.widgets.omni_add_edit import AddEdit
+        from backend.validators.pydant import PydElastic
        dlg = AddEdit(parent=None, instance=self, disabled=['reagent'])
        if dlg.exec():
            pyd = dlg.parse_form()
-            for field in pyd.model_fields:
-                self.set_attribute(field, pyd.__getattribute__(field))
+            logger.debug(f"Pydantic returned: {type(pyd)} {pyd.model_fields}")
+            fields = pyd.model_fields
+            if isinstance(pyd, PydElastic):
+                fields.update(pyd.model_extra)
+            for field in fields:
+                if field in ['instance']:
+                    continue
+                field_value = pyd.__getattribute__(field)
+                logger.debug(f"Setting {field} in Reagent Lot to {field_value}")
+                self.set_attribute(field, field_value)
            self.save()

+    def details_dict(self, **kwargs) -> dict:
+        output = super().details_dict(**kwargs)
+        output['excluded'] += ["reagentlotprocedureassociation", "procedures"]
+        output['reagent'] = output['reagent'].name
+        return output
+
 class Discount(BaseClass):
    """
    Relationship table for client labs for certain kits.
@@ -508,7 +529,8 @@ class SubmissionType(BaseClass):
        """
        return f"<SubmissionType({self.name})>"

-    @classproperty
+    @classmethod
+    @declared_attr
    def aliases(cls) -> List[str]:
        """
        Gets other names the sql object of this class might go by.
@@ -604,12 +626,14 @@ class SubmissionType(BaseClass):
            sample_map=self.sample_map
        )

-    @classproperty
+    @classmethod
+    @declared_attr
    def info_map_json_edit_fields(cls):
        dicto = dict()
        return dicto

-    @classproperty
+    @classmethod
+    @declared_attr
    def regex(cls) -> re.Pattern:
        """
        Constructs catchall regex.
@@ -1182,7 +1206,8 @@ class ProcedureTypeReagentRoleAssociation(BaseClass):
        dicto['required']['instance_attr'] = bool(dicto['required']['instance_attr'])
        return dicto

-    @classproperty
+    @classmethod
+    @declared_attr
    def json_edit_fields(cls) -> dict:
        dicto = dict(
            sheet="str",
@@ -1607,7 +1632,8 @@ class Equipment(BaseClass, LogMixin):
        creation_dict['equipmentrole'] = equipmentrole or creation_dict['equipmentrole']
        return PydEquipment(**creation_dict)

-    @classproperty
+    @classmethod
+    @declared_attr
    def manufacturer_regex(cls) -> re.Pattern:
        """
        Creates regex to determine tip manufacturer
--- a/src/submissions/backend/db/models/submissions.py
+++ b/src/submissions/backend/db/models/submissions.py
@@ -11,9 +11,9 @@ from pprint import pformat
 from pandas import DataFrame
 from sqlalchemy.ext.hybrid import hybrid_property
 from frontend.widgets.functions import select_save_file
-from . import Base, BaseClass, SubmissionType, ClientLab, Contact, LogMixin, Procedure
+from . import BaseClass, SubmissionType, ClientLab, Contact, LogMixin, Procedure
 from sqlalchemy import Column, String, TIMESTAMP, INTEGER, ForeignKey, JSON, FLOAT, case, func
-from sqlalchemy.orm import relationship, Query
+from sqlalchemy.orm import relationship, Query, declared_attr
 from sqlalchemy.orm.attributes import flag_modified
 from sqlalchemy.ext.associationproxy import association_proxy
 from sqlalchemy.exc import OperationalError as AlcOperationalError, IntegrityError as AlcIntegrityError, StatementError
@@ -947,7 +947,8 @@ class Run(BaseClass, LogMixin):

    # NOTE: Polymorphic functions

-    @classproperty
+    @classmethod
+    @declared_attr
    def regex(cls) -> re.Pattern:
        """
        Constructs catchall regex.
@@ -1297,7 +1298,8 @@ class Sample(BaseClass, LogMixin):
    def __repr__(self) -> str:
        return f"<Sample({self.sample_id})>"

-    @classproperty
+    @classmethod
+    @declared_attr
    def searchables(cls):
        return [dict(label="Submitter ID", field="sample_id")]

--- a/src/submissions/backend/excel/init.py
+++ b/src/submissions/backend/excel/init.py
@@ -2,4 +2,18 @@
 Contains pandas and openpyxl convenience functions for interacting with excel workbooks
 """

-from backend.excel.parsers.clientsubmission_parser import ClientSubmissionInfoParser, ClientSubmissionSampleParser
+# from backend.excel.parsers.clientsubmission_parser import ClientSubmissionInfoParser, ClientSubmissionSampleParser
+from .parsers import (
+    DefaultParser, DefaultKEYVALUEParser, DefaultTABLEParser,
+    ProcedureInfoParser, ProcedureSampleParser, ProcedureReagentParser, ProcedureEquipmentParser,
+    DefaultResultsInfoParser, DefaultResultsSampleParser, PCRSampleParser, PCRInfoParser,
+    ClientSubmissionSampleParser, ClientSubmissionInfoParser,
+    PCRInfoParser, PCRSampleParser
+)
+from .writers import (
+    DefaultWriter, DefaultKEYVALUEWriter, DefaultTABLEWriter,
+    ProcedureInfoWriter, ProcedureSampleWriter, ProcedureReagentWriter, ProcedureEquipmentWriter,
+    PCRInfoWriter, PCRSampleWriter,
+    ClientSubmissionInfoWriter, ClientSubmissionSampleWriter
+)
+from .reports import ReportArchetype, ReportMaker, TurnaroundMaker, ConcentrationMaker, ChartReportMaker
--- a/src/submissions/backend/excel/parsers/init.py
+++ b/src/submissions/backend/excel/parsers/init.py
@@ -137,5 +137,10 @@ class DefaultTABLEParser(DefaultParser):
        return [self._pyd_object(**output) for output in self.parsed_info]


+from .procedure_parsers import ProcedureInfoParser, ProcedureSampleParser, ProcedureReagentParser, ProcedureEquipmentParser
+from .results_parsers import (
+    DefaultResultsInfoParser, DefaultResultsSampleParser,
+    PCRSampleParser, PCRInfoParser
+)
 from .clientsubmission_parser import ClientSubmissionSampleParser, ClientSubmissionInfoParser
-from backend.excel.parsers.results_parsers.pcr_results_parser import PCRInfoParser, PCRSampleParser
+from .results_parsers.pcr_results_parser import PCRInfoParser, PCRSampleParser
--- a/src/submissions/backend/excel/writers/init.py
+++ b/src/submissions/backend/excel/writers/init.py
@@ -245,4 +245,8 @@ class DefaultTABLEWriter(DefaultWriter):
        return worksheet


+from .procedure_writers import ProcedureInfoWriter, ProcedureSampleWriter, ProcedureReagentWriter, ProcedureEquipmentWriter
+from .results_writers import (
+    PCRInfoWriter, PCRSampleWriter
+)
 from .clientsubmission_writer import ClientSubmissionInfoWriter, ClientSubmissionSampleWriter
--- a/src/submissions/backend/validators/init.py
+++ b/src/submissions/backend/validators/init.py
@@ -265,5 +265,7 @@ class RSLNamer(object):
            return ""


-from .pydant import PydRun, PydContact, PydClientLab, PydSample, PydReagent, PydReagentRole, \
-    PydEquipment, PydEquipmentRole, PydTips, PydProcess, PydElastic, PydClientSubmission, PydProcedure, PydResults
+from .pydant import (
+    PydRun, PydContact, PydClientLab, PydSample, PydReagent, PydReagentRole, PydEquipment, PydEquipmentRole, PydTips,
+    PydProcess, PydElastic, PydClientSubmission, PydProcedure, PydResults, PydReagentLot
+)
--- a/src/submissions/backend/validators/pydant.py
+++ b/src/submissions/backend/validators/pydant.py
@@ -3,6 +3,7 @@ Contains pydantic models and accompanying validators
 """
 from __future__ import annotations
 import re, logging, csv, sys, string
+from pprint import pformat
 from pydantic import BaseModel, field_validator, Field, model_validator
 from datetime import date, datetime, timedelta
 from dateutil.parser import parse
@@ -11,7 +12,8 @@ from typing import List, Tuple, Literal
 from types import GeneratorType
 from . import RSLNamer
 from pathlib import Path
-from tools import check_not_nan, convert_nans_to_nones, Report, Result, timezone, sort_dict_by_list, row_keys
+from tools import check_not_nan, convert_nans_to_nones, Report, Result, timezone, sort_dict_by_list, row_keys, \
+    flatten_list
 from backend.db import models
 from backend.db.models import *
 from sqlalchemy.exc import StatementError
@@ -136,6 +138,12 @@ class PydBaseClass(BaseModel, extra='allow', validate_assignment=True):
        return list(set(output))


+class PydReagentLot(PydBaseClass):
+    lot: str | None
+    expiry: date | datetime | Literal['NA'] | None = Field(default=None, validate_default=True)
+    missing: bool = Field(default=True)
+    comment: str | None = Field(default="", validate_default=True)
+
 class PydReagent(PydBaseClass):
    lot: str | None
    reagentrole: str | None
@@ -220,6 +228,7 @@ class PydReagent(PydBaseClass):
        else:
            return values.data['reagentrole'].strip()

+
    def improved_dict(self) -> dict:
        """
        Constructs a dictionary consisting of model.fields and model.extras
--- a/src/submissions/frontend/init.py
+++ b/src/submissions/frontend/init.py
@@ -1,5 +1,32 @@
-'''
+"""
 Constructs main application.
-'''
-from .widgets import *
-from .visualizations import *
+"""
+from .widgets import (
+    pandasModel,
+    App,
+    Concentrations,
+    ControlsViewer,
+    DateTypePicker,
+    EquipmentUsage, RoleComboBox,
+    select_open_file, select_save_file, save_pdf,
+    GelBox, ControlsForm,
+    InfoPane,
+    StartEndDatePicker, CheckableComboBox, Pagifier,
+    AddEdit, EditProperty,
+    SearchBox, SearchResults, FieldSearch,
+    QuestionAsker, AlertPop, HTMLPop, ObjectSelector,
+    ProcedureCreation,
+    SampleChecker,
+    SubmissionDetails, SubmissionComment,
+    SubmissionsTree, ClientSubmissionRunModel,
+    MyQComboBox, MyQDateEdit, SubmissionFormContainer, SubmissionFormWidget, ClientSubmissionFormWidget,
+    Summary,
+    TurnaroundMaker
+)
+from .visualizations import (
+    CustomFigure,
+    IridaFigure,
+    PCRFigure,
+    ConcentrationsChart,
+    TurnaroundChart
+)
--- a/src/submissions/frontend/widgets/init.py
+++ b/src/submissions/frontend/widgets/init.py
@@ -1,20 +1,71 @@
 """
 Contains all custom generated PyQT6 derivative widgets.
 """
+from PyQt6.QtCore import QAbstractTableModel, Qt
+
+
+class pandasModel(QAbstractTableModel):
+    """
+    pandas model for inserting summary sheet into gui
+    NOTE: Copied from Stack Overflow. I have no idea how it actually works.
+    """
+
+    def __init__(self, data) -> None:
+        QAbstractTableModel.__init__(self)
+        self._data = data
+
+    def rowCount(self, parent=None) -> int:
+        """
+        does what it says
+
+        Args:
+            parent (_type_, optional): _description_. Defaults to None.
+
+        Returns:
+            int: number of rows in data
+        """
+        return self._data.shape[0]
+
+    def columnCount(self, parent=None) -> int:
+        """
+        does what it says
+
+        Args:
+            parent (_type_, optional): _description_. Defaults to None.
+
+        Returns:
+            int: number of columns in data
+        """
+        return self._data.shape[1]
+
+    def data(self, index, role=Qt.ItemDataRole.DisplayRole) -> str | None:
+        if index.isValid():
+            if role == Qt.ItemDataRole.DisplayRole:
+                return str(self._data.iloc[index.row(), index.column()])
+        return None
+
+    def headerData(self, col, orientation, role):
+        if orientation == Qt.Orientation.Horizontal and role == Qt.ItemDataRole.DisplayRole:
+            return self._data.columns[col]
+        return None
+

 from .app import App
-from .controls_chart import *
-from .equipment_usage import *
-from .functions import *
-from .gel_checker import *
-from .info_tab import *
-from .misc import *
-from .omni_search import *
-from .pop_ups import *
-from .submission_details import *
-from .submission_table import *
-from .submission_widget import *
-from .summary import *
-from .turnaround import *
-from .omni_add_edit import *
-from .omni_manager_pydant import *
+from .concentrations import Concentrations
+from .controls_chart import ControlsViewer
+from .date_type_picker import DateTypePicker
+from .equipment_usage import EquipmentUsage, RoleComboBox
+from .functions import select_open_file, select_save_file, save_pdf
+from .gel_checker import GelBox, ControlsForm
+from .info_tab import InfoPane
+from .misc import StartEndDatePicker, CheckableComboBox, Pagifier
+from .omni_add_edit import AddEdit, EditProperty
+from .omni_search import SearchBox, SearchResults, FieldSearch
+from .pop_ups import QuestionAsker, AlertPop, HTMLPop, ObjectSelector
+from .procedure_creation import ProcedureCreation
+from .sample_checker import SampleChecker
+from .submission_details import SubmissionDetails, SubmissionComment
+from .submission_table import SubmissionsTree, ClientSubmissionRunModel
+from .submission_widget import MyQComboBox, MyQDateEdit, SubmissionFormContainer, SubmissionFormWidget, ClientSubmissionFormWidget
+from .summary import Summary
+from .turnaround import TurnaroundMaker
--- a/src/submissions/frontend/widgets/app.py
+++ b/src/submissions/frontend/widgets/app.py
@@ -13,7 +13,7 @@ from PyQt6.QtGui import QAction
 from pathlib import Path
 from markdown import markdown
 from pandas import ExcelWriter
-from backend.db.models import ReagentLot
+# from backend.db.models import ReagentLot
 from tools import (
    check_if_app, Settings, Report, jinja_template_loading, check_authorization, page_size, is_power_user,
    under_development
@@ -24,7 +24,6 @@ from .pop_ups import HTMLPop
 from .misc import Pagifier
 from .submission_table import SubmissionsTree, ClientSubmissionRunModel
 from .submission_widget import SubmissionFormContainer
-# from .controls_chart import ControlsViewer
 from .summary import Summary
 from .turnaround import TurnaroundTime
 from .concentrations import Concentrations
@@ -181,7 +180,8 @@ class App(QMainWindow):

    @check_authorization
    def edit_reagent(self, *args, **kwargs):
-        dlg = SearchBox(parent=self, object_type=ReagentLot, extras=[dict(name='Role', field="reagentrole")])
+        from backend.db.models import ReagentLot
+        dlg = SearchBox(parent=self, object_type=ReagentLot, extras=ReagentLot.get_searchables())
        dlg.exec()

    def update_data(self):
--- a/src/submissions/frontend/widgets/date_type_picker.py
+++ b/src/submissions/frontend/widgets/date_type_picker.py
@@ -5,7 +5,6 @@ from PyQt6.QtWidgets import (
    QVBoxLayout, QDialog, QDialogButtonBox
 )
 from .misc import CheckableComboBox, StartEndDatePicker
-from backend.db.models.procedures import SubmissionType
 import logging

 logger = logging.getLogger(f"submissions.{__name__}")
@@ -14,6 +13,7 @@ logger = logging.getLogger(f"submissions.{__name__}")
 class DateTypePicker(QDialog):
    
    def __init__(self, parent):
+        from backend.db.models.procedures import SubmissionType
        super().__init__(parent)
        self.layout = QVBoxLayout()
        self.setFixedWidth(500)
--- a/src/submissions/frontend/widgets/omni_add_edit.py
+++ b/src/submissions/frontend/widgets/omni_add_edit.py
@@ -34,7 +34,6 @@ class AddEdit(QDialog):
        self.buttonBox.accepted.connect(self.accept)
        self.buttonBox.rejected.connect(self.reject)
        fields = {k: v for k, v in self.instance.omnigui_instance_dict.items() if "id" not in k}
-        logger.debug(f"Fields: {pformat(fields)}")
        # NOTE: Move 'name' to the front
        try:
            fields = {'name': fields.pop('name'), **fields}
--- a/src/submissions/frontend/widgets/omni_search.py
+++ b/src/submissions/frontend/widgets/omni_search.py
@@ -1,7 +1,6 @@
 """
 Search box that performs fuzzy search for various object types
 """
-from copy import deepcopy
 from pprint import pformat
 from typing import Tuple, Any, List, Generator
 from pandas import DataFrame
@@ -10,8 +9,8 @@ from PyQt6.QtWidgets import (
    QLabel, QVBoxLayout, QDialog,
    QTableView, QWidget, QLineEdit, QGridLayout, QComboBox, QDialogButtonBox
 )
-from .submission_table import pandasModel
-import logging
+from . import pandasModel
+import logging, sys

 logger = logging.getLogger(f"submissions.{__name__}")

@@ -164,10 +163,10 @@ class SearchResults(QTableView):
        self.context = kwargs
        self.parent = parent
        self.object_type = object_type
-        try:
-            self.extras = extras + [item for item in deepcopy(self.object_type.searchables)]
-        except AttributeError:
-            self.extras = extras
+        # try:
+        #     self.extras = extras + [item for item in deepcopy(self.object_type.searchables)]
+        # except AttributeError:
+        #     self.extras = extras

    def setData(self, df: DataFrame) -> None:
        """
@@ -176,10 +175,11 @@ class SearchResults(QTableView):

        self.data = df
        try:
-            self.columns_of_interest = [dict(name=item['field'], column=self.data.columns.get_loc(item['field'])) for
-                                        item in self.extras]
+            self.columns_of_interest = [dict(name=item, column=self.data.columns.get_loc(item)) for
+                                        item in self.object_type.get_searchables()]
        except KeyError:
            self.columns_of_interest = []
+        logger.debug(f"Columns of Interest: {pformat(self.columns_of_interest)}")
        try:
            self.data['id'] = self.data['id'].apply(str)
            self.data['id'] = self.data['id'].str.zfill(3)
@@ -204,10 +204,13 @@ class SearchResults(QTableView):
            None
        """
        context = {item['name']: x.sibling(x.row(), item['column']).data() for item in self.columns_of_interest}
+        logger.debug(f"Context: {pformat(context)}")
        try:
            object = self.object_type.query(**context)
-        except KeyError:
+        except KeyError as e:
+            logger.error(e)
            object = None
+        logger.debug(f"Object: {object}")
        try:
            object.edit_from_search(obj=self.parent, **context)
        except AttributeError as e:
--- a/src/submissions/frontend/widgets/procedure_creation.py
+++ b/src/submissions/frontend/widgets/procedure_creation.py
@@ -69,11 +69,13 @@ class ProcedureCreation(QDialog):
                                        equipment['name'] == relevant_procedure_item.name))
                equipmentrole['equipment'].insert(0, equipmentrole['equipment'].pop(
                    equipmentrole['equipment'].index(item_in_er_list)))
-        proceduretype_dict['equipment_section'] = EquipmentUsage.construct_html(procedure=self.procedure, child=True)
+        # proceduretype_dict['equipment_section'] = EquipmentUsage.construct_html(procedure=self.procedure, child=True)
        proceduretype_dict['equipment'] = [sanitize_object_for_json(object) for object in proceduretype_dict['equipment']]
        self.update_equipment = EquipmentUsage.update_equipment
        regex = re.compile(r".*R\d$")
        proceduretype_dict['previous'] = [""] + [item.name for item in self.run.procedure if item.proceduretype == self.proceduretype and not bool(regex.match(item.name))]
+        logger.debug(f"Procedure:\n{pformat(self.procedure.__dict__)}")
+        logger.debug(f"ProcedureType:\n{pformat(proceduretype_dict)}")
        html = render_details_template(
            template_name="procedure_creation",
            js_in=["procedure_form", "grid_drag", "context_menu"],
--- a/src/submissions/frontend/widgets/submission_table.py
+++ b/src/submissions/frontend/widgets/submission_table.py
@@ -1,234 +1,18 @@
 """
 Contains widgets specific to the procedure summary and procedure details.
 """
-
-import sys, logging, re
+import sys, logging
 from pprint import pformat
-from PyQt6.QtWidgets import QTableView, QMenu, QTreeView, QStyledItemDelegate, QStyle, QStyleOptionViewItem, \
-    QHeaderView, QAbstractItemView, QWidget, QTreeWidgetItemIterator
-from PyQt6.QtCore import Qt, QAbstractTableModel, QSortFilterProxyModel, pyqtSlot, QModelIndex
-from PyQt6.QtGui import QAction, QCursor, QStandardItemModel, QStandardItem, QIcon, QColor, QContextMenuEvent
-from typing import Dict, List
-# from backend import Procedure
-from backend.db.models.submissions import Run, ClientSubmission
-from backend.db.models.procedures import  Procedure
-from tools import Report, Result, report_result, get_application_from_parent
-from .functions import select_open_file
+from PyQt6.QtWidgets import QMenu, QTreeView, QAbstractItemView
+from PyQt6.QtCore import QModelIndex
+from PyQt6.QtGui import QAction, QCursor, QStandardItemModel, QStandardItem, QContextMenuEvent
+from typing import List
+from backend.db.models import Run, ClientSubmission, Procedure
+from tools import get_application_from_parent

 logger = logging.getLogger(f"submissions.{__name__}")


-class pandasModel(QAbstractTableModel):
-    """
-    pandas model for inserting summary sheet into gui
-    NOTE: Copied from Stack Overflow. I have no idea how it actually works.
-    """
-
-    def __init__(self, data) -> None:
-        QAbstractTableModel.__init__(self)
-        self._data = data
-
-    def rowCount(self, parent=None) -> int:
-        """
-        does what it says
-
-        Args:
-            parent (_type_, optional): _description_. Defaults to None.
-
-        Returns:
-            int: number of rows in data
-        """
-        return self._data.shape[0]
-
-    def columnCount(self, parent=None) -> int:
-        """
-        does what it says
-
-        Args:
-            parent (_type_, optional): _description_. Defaults to None.
-
-        Returns:
-            int: number of columns in data
-        """
-        return self._data.shape[1]
-
-    def data(self, index, role=Qt.ItemDataRole.DisplayRole) -> str | None:
-        if index.isValid():
-            if role == Qt.ItemDataRole.DisplayRole:
-                return str(self._data.iloc[index.row(), index.column()])
-        return None
-
-    def headerData(self, col, orientation, role):
-        if orientation == Qt.Orientation.Horizontal and role == Qt.ItemDataRole.DisplayRole:
-            return self._data.columns[col]
-        return None
-
-
-class SubmissionsSheet(QTableView):
-    """
-    presents procedure summary to user in tab1
-    """
-
-    def __init__(self, parent) -> None:
-        super().__init__(parent)
-        self.app = self.parent()
-        self.report = Report()
-        try:
-            page_size = self.app.page_size
-        except AttributeError:
-            page_size = 250
-        self.set_data(page=1, page_size=page_size)
-        self.resizeColumnsToContents()
-        self.resizeRowsToContents()
-        self.setSortingEnabled(True)
-        self.doubleClicked.connect(lambda x: Run.query(id=x.sibling(x.row(), 0).data()).show_details(self))
-        # NOTE: Have to procedure native query here because mine just returns results?
-        self.total_count = Run.__database_session__.query(Run).count()
-
-    def set_data(self, page: int = 1, page_size: int = 250) -> None:
-        """
-        sets data in model
-        """
-        self.data = Run.submissions_to_df(page=page, page_size=page_size)
-        try:
-            self.data['Id'] = self.data['Id'].apply(str)
-            self.data['Id'] = self.data['Id'].str.zfill(4)
-        except KeyError as e:
-            logger.error(f"Could not alter id to string due to {e}")
-        proxyModel = QSortFilterProxyModel()
-        proxyModel.setSourceModel(pandasModel(self.data))
-        self.setModel(proxyModel)
-
-    def contextMenuEvent(self, event):
-        """
-        Creates actions for right click menu events.
-
-        Args:
-            event (_type_): the item of interest
-        """
-        # NOTE: Get current row index
-        id = self.selectionModel().currentIndex()
-        # NOTE: Convert to data in id column (i.e. column 0)
-        id = id.sibling(id.row(), 0).data()
-        submission = Run.query(id=id)
-        self.menu = QMenu(self)
-        self.con_actions = submission.custom_context_events()
-        for k in self.con_actions.keys():
-            action = QAction(k, self)
-            action.triggered.connect(lambda _, action_name=k: self.triggered_action(action_name=action_name))
-            self.menu.addAction(action)
-        # NOTE: add other required actions
-        self.menu.popup(QCursor.pos())
-
-    def triggered_action(self, action_name: str):
-        """
-        Calls the triggered action from the context menu
-
-        Args:
-            action_name (str): name of the action from the menu
-        """
-        func = self.con_actions[action_name]
-        func(obj=self)
-
-    @report_result
-    def link_extractions(self):
-        """
-        Pull extraction logs into the db
-        """
-        report = Report()
-        result = self.link_extractions_function()
-        report.add_result(result)
-        return report
-
-    def link_extractions_function(self):
-        """
-        Link extractions from runlogs to imported procedure
-
-        Args:
-            obj (QMainWindow): original app window
-
-        Returns:
-            Tuple[QMainWindow, dict]: Collection of new main app window and result dict
-        """
-        report = Report()
-        fname = select_open_file(self, file_extension="csv")
-        with open(fname.__str__(), 'r') as f:
-            # NOTE: split csv on commas
-            runs = [col.strip().split(",") for col in f.readlines()]
-        count = 0
-        for run in runs:
-            new_run = dict(
-                start_time=run[0].strip(),
-                rsl_plate_number=run[1].strip(),
-                sample_count=run[2].strip(),
-                status=run[3].strip(),
-                experiment_name=run[4].strip(),
-                end_time=run[5].strip()
-            )
-            # NOTE: elution columns are item 6 in the comma split list to the end
-            for ii in range(6, len(run)):
-                new_run[f"column{str(ii - 5)}_vol"] = run[ii]
-            # NOTE: Lookup imported procedure
-            sub = Run.query(name=new_run['name'])
-            # NOTE: If no such procedure exists, move onto the next procedure
-            if sub is None:
-                continue
-            try:
-                count += 1
-            except AttributeError:
-                continue
-            sub.set_attribute('extraction_info', new_run)
-            sub.save()
-        report.add_result(Result(msg=f"We added {count} logs to the database.", status='Information'))
-        return report
-
-    @report_result
-    def link_pcr(self):
-        """
-        Pull pcr logs into the db
-        """
-        report = Report()
-        result = self.link_pcr_function()
-        report.add_result(result)
-        return report
-
-    def link_pcr_function(self):
-        """
-        Link PCR data from procedure logs to an imported procedure
-
-        Args:
-            obj (QMainWindow): original app window
-
-        Returns:
-            Tuple[QMainWindow, dict]: Collection of new main app window and result dict
-        """
-        report = Report()
-        fname = select_open_file(self, file_extension="csv")
-        with open(fname.__str__(), 'r') as f:
-            # NOTE: split csv rows on comma
-            runs = [col.strip().split(",") for col in f.readlines()]
-        count = 0
-        for run in runs:
-            new_run = dict(
-                start_time=run[0].strip(),
-                rsl_plate_number=run[1].strip(),
-                biomek_status=run[2].strip(),
-                quant_status=run[3].strip(),
-                experiment_name=run[4].strip(),
-                end_time=run[5].strip()
-            )
-            # NOTE: lookup imported procedure
-            sub = Run.query(rsl_number=new_run['name'])
-            # NOTE: if imported procedure doesn't exist move on to next procedure
-            if sub is None:
-                continue
-            sub.set_attribute('pcr_info', new_run)
-            # NOTE: check if pcr_info already exists
-            sub.save()
-        report.add_result(Result(msg=f"We added {count} logs to the database.", status='Information'))
-        return report
-
-
 class SubmissionsTree(QTreeView):
    """
    https://stackoverflow.com/questions/54385437/how-can-i-make-a-table-that-can-collapse-its-rows-into-categories-in-qt
@@ -372,10 +156,6 @@ class SubmissionsTree(QTreeView):

 class ClientSubmissionRunModel(QStandardItemModel):

-
-    # def __init__(self, parent=None):
-    #     super(ClientSubmissionRunModel, self).__init__(parent)
-    #
    def add_child(self, parent: QStandardItem, child:dict):
        item = QStandardItem(child['name'])
        item.setData(dict(item_type=child['item_type'], query_str=child['query_str']), 1)
--- a/src/submissions/frontend/widgets/submission_widget.py
+++ b/src/submissions/frontend/widgets/submission_widget.py
@@ -11,12 +11,12 @@ from .functions import select_open_file, select_save_file
 from pathlib import Path
 from tools import Report, Result, check_not_nan, main_form_style, report_result, get_application_from_parent
 from backend.validators import PydReagent, PydClientSubmission, PydSample
-from backend.db import (
+from backend.db.models import (
    ClientLab, SubmissionType, Reagent, ReagentLot,
    ReagentRole, ProcedureTypeReagentRoleAssociation, Run, ClientSubmission
 )
 from pprint import pformat
-from .pop_ups import QuestionAsker, AlertPop
+from .pop_ups import QuestionAsker
 from .omni_add_edit import AddEdit
 from typing import List, Tuple
 from datetime import date
--- a/src/submissions/frontend/widgets/summary.py
+++ b/src/submissions/frontend/widgets/summary.py
@@ -3,7 +3,7 @@ Pane to hold information e.g. cost summary.
 """
 from .info_tab import InfoPane
 from PyQt6.QtWidgets import QWidget, QLabel, QPushButton
-from backend.db import ClientLab
+from backend.db.models import ClientLab
 from backend.excel.reports import ReportMaker
 from .misc import CheckableComboBox
 import logging
--- a/src/submissions/frontend/widgets/turnaround.py
+++ b/src/submissions/frontend/widgets/turnaround.py
@@ -4,7 +4,7 @@ Pane showing turnaround time summary.
 from PyQt6.QtWidgets import QWidget, QPushButton, QComboBox, QLabel
 from .info_tab import InfoPane
 from backend.excel.reports import TurnaroundMaker
-from backend.db import SubmissionType
+from backend.db.models import SubmissionType
 from frontend.visualizations.turnaround_chart import TurnaroundChart
 import logging