Added ability to not import reagents on first import.

CHANGELOG.md

@@ -1,3 +1,11 @@
+## 202412.03
+
+- Automated truncating of object names longer than 64 chars going into _auditlog
+- Writer will now blank out the lookup table before writing to ensure removal of extraneous help info.
+- Added support for running startup and teardown scripts.
+- Created startup script to pull irida controls from secondary database.
+- Added ability to not import reagents on first import.
+
 ## 202412.02
 
 - Addition of turnaround time tracking

src/submissions/__main__.py

@@ -1,17 +1,50 @@
 import sys, os
 from tools import ctx, setup_logger, check_if_app
+from backend import scripts
 
 # environment variable must be set to enable qtwebengine in network path
 if check_if_app():
     os.environ['QTWEBENGINE_DISABLE_SANDBOX'] = "1"
+
 # setup custom logger
 logger = setup_logger(verbosity=3)
-# create settings object
 
 from PyQt6.QtWidgets import QApplication
 from frontend.widgets.app import App
 
+
+def run_startup():
+    try:
+        startup_scripts = ctx.startup_scripts
+    except AttributeError as e:
+        logger.error(f"Couldn't get startup scripts due to {e}")
+        return
+    for script in startup_scripts:
+        try:
+            func = getattr(scripts, script)
+        except AttributeError as e:
+            logger.error(f"Couldn't run startup script {script} due to {e}")
+            continue
+        func(ctx)
+
+
+def run_teardown():
+    try:
+        teardown_scripts = ctx.teardown_scripts
+    except AttributeError as e:
+        logger.error(f"Couldn't get teardown scripts due to {e}")
+        return
+    for script in teardown_scripts:
+        try:
+            func = getattr(scripts, script)
+        except AttributeError as e:
+            logger.error(f"Couldn't run teardown script {script} due to {e}")
+            continue
+        func(ctx)
+
 if __name__ == '__main__':
+    run_startup()
     app = QApplication(['', '--no-sandbox'])
     ex = App(ctx=ctx)
-    sys.exit(app.exec())
+    app.exec()
+    sys.exit(run_teardown())

src/submissions/backend/db/__init__.py

@@ -37,10 +37,11 @@ from .models import *
 
 
 def update_log(mapper, connection, target):
-    logger.debug("\n\nBefore update\n\n")
+    # logger.debug("\n\nBefore update\n\n")
     state = inspect(target)
     # logger.debug(state)
-    update = dict(user=getuser(), time=datetime.now(), object=str(state.object), changes=[])
+    object_name = state.object.truncated_name()
+    update = dict(user=getuser(), time=datetime.now(), object=object_name, changes=[])
     # logger.debug(update)
     for attr in state.attrs:
         hist = attr.load_history()
@@ -49,8 +50,10 @@ def update_log(mapper, connection, target):
         if attr.key == "custom":
             continue
         added = [str(item) for item in hist.added]
-        if attr.key in ['submission_sample_associations', 'submission_reagent_associations']:
+        if attr.key in ['artic_technician', 'submission_sample_associations', 'submission_reagent_associations',
-            added = ['Numbers truncated for space purposes.']
+                        'submission_equipment_associations', 'submission_tips_associations', 'contact_id', 'gel_info',
+                        'gel_controls', 'source_plates']:
+            continue
         deleted = [str(item) for item in hist.deleted]
         change = dict(field=attr.key, added=added, deleted=deleted)
         # logger.debug(f"Adding: {pformat(change)}")

src/submissions/backend/db/models/__init__.py

@@ -25,6 +25,16 @@ logger = logging.getLogger(f"submissions.{__name__}")
 class LogMixin(Base):
     __abstract__ = True
 
+    def truncated_name(self):
+        name = str(self)
+        if len(name) > 64:
+            name = name.replace("<", "").replace(">", "")
+        if len(name) > 64:
+            name = name.replace("agent", "")
+        if len(name) > 64:
+            name = f"...{name[-61:]}"
+        return name
+
 
 class BaseClass(Base):
     """

src/submissions/backend/db/models/controls.py

@@ -539,7 +539,7 @@ class IridaControl(Control):
         except AttributeError:
             consolidate = False
         report = Report()
-        logger.debug(f"settings: {pformat(chart_settings)}")
+        # logger.debug(f"settings: {pformat(chart_settings)}")
         controls = cls.query(subtype=chart_settings['sub_type'], start_date=chart_settings['start_date'],
                              end_date=chart_settings['end_date'])
         # logger.debug(f"Controls found: {controls}")

src/submissions/backend/db/models/kits.py

@@ -427,9 +427,10 @@ class Reagent(BaseClass, LogMixin):
 
     def __repr__(self):
         if self.name:
-            return f"<Reagent({self.name}-{self.lot})>"
+            name = f"<Reagent({self.name}-{self.lot})>"
         else:
-            return f"<Reagent({self.role.name}-{self.lot})>"
+            name = f"<Reagent({self.role.name}-{self.lot})>"
+        return name
 
     def to_sub_dict(self, extraction_kit: KitType = None, full_data: bool = False, **kwargs) -> dict:
         """
@@ -1347,7 +1348,7 @@ class SubmissionReagentAssociation(BaseClass):
         return PydReagent(**self.to_sub_dict(extraction_kit=extraction_kit))
 
 
-class Equipment(BaseClass):
+class Equipment(BaseClass, LogMixin):
     """
     A concrete instance of equipment
     """
@@ -1851,7 +1852,7 @@ class TipRole(BaseClass):
         super().save()
 
 
-class Tips(BaseClass):
+class Tips(BaseClass, LogMixin):
     """
     A concrete instance of tips.
     """

src/submissions/backend/db/models/submissions.py

@@ -174,7 +174,8 @@ class BasicSubmission(BaseClass, LogMixin):
                             'platemap', 'export_map', 'equipment', 'tips', 'custom'],
             # NOTE: Fields not placed in ui form
             form_ignore=['reagents', 'ctx', 'id', 'cost', 'extraction_info', 'signed_by', 'comment', 'namer',
-                         'submission_object', "tips", 'contact_phone', 'custom', 'cost_centre'] + recover,
+                         'submission_object', "tips", 'contact_phone', 'custom', 'cost_centre', 'completed_date',
+                         'controls'] + recover,
             # NOTE: Fields not placed in ui form to be moved to pydantic
             form_recover=recover
         ))

src/submissions/backend/excel/writer.py

@@ -3,6 +3,7 @@ contains writer objects for pushing values to submission sheet templates.
 """
 import logging
 from copy import copy
+from datetime import date
 from operator import itemgetter
 from pprint import pformat
 from typing import List, Generator, Tuple
@@ -214,6 +215,10 @@ class ReagentWriter(object):
         Returns:
             List[dict]: merged dictionary
         """
+        filled_roles = [item['role'] for item in reagent_list]
+        for map_obj in reagent_map.keys():
+            if map_obj not in filled_roles:
+                reagent_list.append(dict(name="Not Applicable", role=map_obj, lot="Not Applicable", expiry="Not Applicable"))
         for reagent in reagent_list:
             try:
                 mp_info = reagent_map[reagent['role']]
@@ -268,6 +273,7 @@ class SampleWriter(object):
         # NOTE: exclude any samples without a submission rank.
         samples = [item for item in self.reconcile_map(sample_list) if item['submission_rank'] > 0]
         self.samples = sorted(samples, key=itemgetter('submission_rank'))
+        self.blank_lookup_table()
 
     def reconcile_map(self, sample_list: list) -> Generator[dict, None, None]:
         """
@@ -291,6 +297,16 @@ class SampleWriter(object):
                     new[k] = v
                 yield new
 
+    def blank_lookup_table(self):
+        """
+        Blanks out columns in the lookup table to ensure help values are removed before writing.
+        """
+        sheet = self.xl[self.sample_map['sheet']]
+        for row in range(self.sample_map['start_row'], self.sample_map['end_row'] + 1):
+            for column in self.sample_map['sample_columns'].values():
+                if sheet.cell(row, column).data_type != 'f':
+                    sheet.cell(row=row, column=column, value="")
+
     def write_samples(self) -> Workbook:
         """
         Performs writing operations.

src/submissions/backend/scripts/__init__.py

@@ -0,0 +1,7 @@
+from .irida import import_irida
+
+def hello(ctx):
+    print("\n\nHello!\n\n")
+
+def goodbye(ctx):
+    print("\n\nGoodbye\n\n")

src/submissions/backend/scripts/irida.py

@@ -0,0 +1,56 @@
+import logging, sqlite3, json
+from pprint import pformat, pprint
+from datetime import datetime
+from tools import Settings
+from backend import BasicSample
+from backend.db import IridaControl, ControlType
+
+logger = logging.getLogger(f"submissions.{__name__}")
+
+def import_irida(ctx:Settings):
+    """
+    Grabs Irida controls from secondary database.
+
+    Args:
+        ctx (Settings): Settings inherited from app.
+
+    """
+    ct = ControlType.query(name="Irida Control")
+    existing_controls = [item.name for item in IridaControl.query()]
+    prm_list = ", ".join([f"'{thing}'" for thing in existing_controls])
+    ctrl_db_path = ctx.directory_path.joinpath("submissions_parser_output", "submissions.db")
+    # print(f"Incoming settings: {pformat(ctx)}")
+    try:
+        conn = sqlite3.connect(ctrl_db_path)
+    except AttributeError as e:
+        print(f"Error, could not import from irida due to {e}")
+        return
+    sql = f"SELECT name, submitted_date, submission_id, contains, matches, kraken, subtype, refseq_version, " \
+          f"kraken2_version, kraken2_db_version, sample_id FROM _iridacontrol INNER JOIN _control on _control.id " \
+          f"= _iridacontrol.id WHERE _control.name NOT IN ({prm_list})"
+    cursor = conn.execute(sql)
+    records = [dict(name=row[0], submitted_date=row[1], submission_id=row[2], contains=row[3], matches=row[4], kraken=row[5],
+                    subtype=row[6], refseq_version=row[7], kraken2_version=row[8], kraken2_db_version=row[9],
+                    sample_id=row[10]) for row in cursor]
+    # incoming_controls = set(item['name'] for item in records)
+    # relevant = list(incoming_controls - existing_controls)
+    for record in records:
+        instance = IridaControl.query(name=record['name'])
+        if instance:
+            logger.warning(f"Irida Control {instance.name} already exists, skipping.")
+            continue
+        record['contains'] = json.loads(record['contains'])
+        assert isinstance(record['contains'], dict)
+        record['matches'] = json.loads(record['matches'])
+        assert isinstance(record['matches'], dict)
+        record['kraken'] = json.loads(record['kraken'])
+        assert isinstance(record['kraken'], dict)
+        record['submitted_date'] = datetime.strptime(record['submitted_date'], "%Y-%m-%d %H:%M:%S.%f")
+        assert isinstance(record['submitted_date'], datetime)
+        instance = IridaControl(controltype=ct, **record)
+        sample = BasicSample.query(submitter_id=instance.name)
+        if sample:
+            instance.sample = sample
+            instance.submission = sample.submissions[0]
+        # pprint(instance.__dict__)
+        instance.save()

src/submissions/frontend/widgets/app.py

@@ -27,7 +27,7 @@ from .turnaround import TurnaroundTime
 from .omni_search import SearchBox
 
 logger = logging.getLogger(f'submissions.{__name__}')
-logger.info("Hello, I am a logger")
+# logger.info("Hello, I am a logger")
 
 
 class App(QMainWindow):

src/submissions/frontend/widgets/submission_details.py

@@ -64,9 +64,6 @@ class SubmissionDetails(QDialog):
                 self.reagent_details(reagent=sub)
         self.webview.page().setWebChannel(self.channel)
 
-    # def back_function(self):
-    #     self.webview.back()
-
     def activate_export(self):
         title = self.webview.title()
         self.setWindowTitle(title)

src/submissions/frontend/widgets/submission_widget.py

@@ -3,9 +3,9 @@ Contains all submission related frontend functions
 '''
 from PyQt6.QtWidgets import (
     QWidget, QPushButton, QVBoxLayout,
-    QComboBox, QDateEdit, QLineEdit, QLabel
+    QComboBox, QDateEdit, QLineEdit, QLabel, QCheckBox, QBoxLayout, QHBoxLayout, QGridLayout
 )
-from PyQt6.QtCore import pyqtSignal, Qt
+from PyQt6.QtCore import pyqtSignal, Qt, QSignalBlocker
 from . import select_open_file, select_save_file
 import logging
 from pathlib import Path
@@ -228,9 +228,26 @@ class SubmissionFormWidget(QWidget):
             # if k == "extraction_kit":
             if k in self.__class__.update_reagent_fields:
                 add_widget.input.currentTextChanged.connect(self.scrape_reagents)
+                self.disabler = self.DisableReagents(self)
+                self.disabler.checkbox.setChecked(True)
+                self.layout.addWidget(self.disabler)
+                self.disabler.checkbox.checkStateChanged.connect(self.disable_reagents)
         self.setStyleSheet(main_form_style)
         self.scrape_reagents(self.extraction_kit)
 
+    def disable_reagents(self):
+        for reagent in self.findChildren(self.ReagentFormWidget):
+            # if self.disabler.checkbox.isChecked():
+            #     # reagent.setVisible(True)
+            #     # with QSignalBlocker(self.disabler.checkbox) as b:
+            #     reagent.flip_check()
+            # else:
+            #     # reagent.setVisible(False)
+            #     # with QSignalBlocker(self.disabler.checkbox) as b:
+            #     reagent.check.setChecked(False)
+            reagent.flip_check(self.disabler.checkbox.isChecked())
+
+
     def create_widget(self, key: str, value: dict | PydReagent, submission_type: str | SubmissionType | None = None,
                       extraction_kit: str | None = None, sub_obj: BasicSubmission | None = None,
                       disable: bool = False) -> "self.InfoItem":
@@ -350,8 +367,9 @@ class SubmissionFormWidget(QWidget):
         report.add_result(result)
         # logger.debug(f"Submission: {pformat(self.pyd)}")
         # logger.debug("Checking kit integrity...")
-        _, result = self.pyd.check_kit_integrity()
+        if self.disabler.checkbox.isChecked():
-        report.add_result(result)
+            _, result = self.pyd.check_kit_integrity()
+            report.add_result(result)
         if len(result.results) > 0:
             return
         # logger.debug(f"PYD before transformation into SQL:\n\n{self.pyd}\n\n")
@@ -665,11 +683,15 @@ class SubmissionFormWidget(QWidget):
             self.app = self.parent().parent().parent().parent().parent().parent().parent().parent()
             self.reagent = reagent
             self.extraction_kit = extraction_kit
-            layout = QVBoxLayout()
+            layout = QGridLayout()
+            self.check = QCheckBox()
+            self.check.setChecked(True)
+            self.check.checkStateChanged.connect(self.disable)
+            layout.addWidget(self.check, 0, 0, 1, 1)
             self.label = self.ReagentParsedLabel(reagent=reagent)
-            layout.addWidget(self.label)
+            layout.addWidget(self.label, 0, 1, 1, 9)
             self.lot = self.ReagentLot(scrollWidget=parent, reagent=reagent, extraction_kit=extraction_kit)
-            layout.addWidget(self.lot)
+            layout.addWidget(self.lot, 1, 0, 1, 10)
             # NOTE: Remove spacing between reagents
             layout.setContentsMargins(0, 0, 0, 0)
             self.setLayout(layout)
@@ -678,6 +700,20 @@ class SubmissionFormWidget(QWidget):
             # NOTE: If changed set self.missing to True and update self.label
             self.lot.currentTextChanged.connect(self.updated)
 
+        def flip_check(self, checked:bool):
+            with QSignalBlocker(self.check) as b:
+                self.check.setChecked(checked)
+                self.lot.setEnabled(checked)
+                self.label.setEnabled(checked)
+
+        def disable(self):
+            self.lot.setEnabled(self.check.isChecked())
+            self.label.setEnabled(self.check.isChecked())
+            if not any([item.lot.isEnabled() for item in self.parent().findChildren(self.__class__)]):
+                self.parent().disabler.checkbox.setChecked(False)
+            else:
+                self.parent().disabler.checkbox.setChecked(True)
+
         def parse_form(self) -> Tuple[PydReagent | None, Report]:
             """
             Pulls form info into PydReagent
@@ -686,6 +722,8 @@ class SubmissionFormWidget(QWidget):
                 Tuple[PydReagent, dict]: PydReagent and Report(?)
             """
             report = Report()
+            if not self.lot.isEnabled():
+                return None, report
             lot = self.lot.currentText()
             # logger.debug(f"Using this lot for the reagent {self.reagent}: {lot}")
             wanted_reagent = Reagent.query(lot=lot, role=self.reagent.role)
@@ -786,3 +824,15 @@ class SubmissionFormWidget(QWidget):
                 self.setObjectName(f"lot_{reagent.role}")
                 self.addItems(relevant_reagents)
                 self.setToolTip(f"Enter lot number for the reagent used for {reagent.role}")
+
+    class DisableReagents(QWidget):
+
+        def __init__(self, parent: QWidget):
+            super().__init__(parent)
+            self.app = self.parent().parent().parent().parent().parent().parent().parent().parent()
+            layout = QHBoxLayout()
+            self.label = QLabel("Import Reagents")
+            self.checkbox = QCheckBox()
+            layout.addWidget(self.label)
+            layout.addWidget(self.checkbox)
+            self.setLayout(layout)

src/submissions/templates/basicsubmission_details.html

@@ -17,12 +17,12 @@
         {% if sub['custom'] %}{% for key, value in sub['custom'].items() %}
             &nbsp;&nbsp;&nbsp;&nbsp;<b>{{ key | replace("_", " ") | title }}: </b>{{ value }}<br>
         {% endfor %}{% endif %}</p>
-
+        {% if sub['reagents'] %}
         <h3><u>Reagents:</u></h3>
         <p>{% for item in sub['reagents'] %}
             &nbsp;&nbsp;&nbsp;&nbsp;<b>{{ item['role'] }}</b>: <a class="data-link reagent" id="{{ item['lot'] }}">{{ item['lot'] }} (EXP: {{ item['expiry'] }})</a><br>
         {% endfor %}</p>
-
+        {% endif %}
         {% if sub['equipment'] %}
         <h3><u>Equipment:</u></h3>
         <p>{% for item in sub['equipment'] %}

src/submissions/tools/__init__.py

@@ -418,13 +418,13 @@ class Settings(BaseSettings, extra="allow"):
 
         super().__init__(*args, **kwargs)
         self.set_from_db()
-        pprint(f"User settings:\n{self.__dict__}")
+        # pprint(f"User settings:\n{self.__dict__}")
 
     def set_from_db(self):
         if 'pytest' in sys.modules:
             output = dict(power_users=['lwark', 'styson', 'ruwang'])
         else:
-            print(f"Hello from database settings getter.")
+            # print(f"Hello from database settings getter.")
             # print(self.__dict__)
             session = self.database_session
             metadata = MetaData()