Debugging scripts import hell.

CHANGELOG.md

@@ -1,6 +1,11 @@
+# 202412.05
+
+- Switched startup/teardown scripts to decorator registration.
+
 # 202412.04
 
 - Update of wastewater to allow for duplex PCR primers.
+- Addition of expiry check after kit integrity check.
 
 ## 202412.03
 

TODO.md

@@ -31,7 +31,6 @@
 - [x] Create platemap image from html for export to pdf.
 - [x] Move plate map maker to submission.
 - [x] Finish Equipment Parser (add in regex to id asset_number)
-- [ ] Complete info_map in the SubmissionTypeCreator widget.
 - [x] Update Artic and add in equipment listings... *sigh*.
 - [x] Fix WastewaterAssociations not in Session error.
     - Done... I think?

src/config.yml

@@ -4,4 +4,3 @@ database_schema: null
 database_user: null
 database_password: null
 database_name: null
-logging_enabled: false

src/submissions/__init__.py

@@ -14,7 +14,7 @@ def get_week_of_month() -> int:
     Gets the current week number of the month.
 
     Returns:
-        int:
+        int: 1 if first week of month, etc.
     """
     for ii, week in enumerate(calendar.monthcalendar(date.today().year, date.today().month)):
         if day in week:

src/submissions/__main__.py

@@ -1,6 +1,7 @@
 import sys, os
-from tools import ctx, setup_logger, check_if_app
+from tools import ctx, setup_logger, check_if_app, timer
 from threading import Thread
+
 # environment variable must be set to enable qtwebengine in network path
 if check_if_app():
     os.environ['QTWEBENGINE_DISABLE_SANDBOX'] = "1"
@@ -8,46 +9,27 @@ if check_if_app():
 # setup custom logger
 logger = setup_logger(verbosity=3)
 
-# from backend.scripts import modules
+# from backend import scripts
-from backend import scripts
 from PyQt6.QtWidgets import QApplication
 from frontend.widgets.app import App
 
 
+@timer
 def run_startup():
-    try:
+    for script in ctx.startup_scripts.values():
-        startup_scripts = ctx.startup_scripts
+        logger.info(f"Running startup script: {script.__name__}")
-    except AttributeError as e:
+        thread = Thread(target=script, args=(ctx,))
-        logger.error(f"Couldn't get startup scripts due to {e}")
-        return
-    for script in startup_scripts:
-        try:
-            func = getattr(scripts, script)
-        except AttributeError as e:
-            logger.error(f"Couldn't run startup script {script} due to {e}")
-            continue
-        logger.info(f"Running startup script: {func.__name__}")
-        thread = Thread(target=func.script, args=(ctx, ))
         thread.start()
 
 
+@timer
 def run_teardown():
-    try:
+    for script in ctx.teardown_scripts.values():
-        teardown_scripts = ctx.teardown_scripts
+        logger.info(f"Running teardown script: {script.__name__}")
-    except AttributeError as e:
+        thread = Thread(target=script, args=(ctx,))
-        logger.error(f"Couldn't get teardown scripts due to {e}")
-        return
-    for script in teardown_scripts:
-        try:
-            func = getattr(scripts, script)
-            # func = modules[script]
-        except AttributeError as e:
-            logger.error(f"Couldn't run teardown script {script} due to {e}")
-            continue
-        logger.info(f"Running teardown script: {func.__name__}")
-        thread = Thread(target=func.script, args=(ctx,))
         thread.start()
 
+
 if __name__ == '__main__':
     run_startup()
     app = QApplication(['', '--no-sandbox'])

src/submissions/backend/db/__init__.py

@@ -21,10 +21,10 @@ def set_sqlite_pragma(dbapi_connection, connection_record):
     if ctx.database_schema == "sqlite":
         execution_phrase = "PRAGMA foreign_keys=ON"
     else:
-        print("Nothing to execute, returning")
+        # print("Nothing to execute, returning")
         cursor.close()
         return
-    print(f"Executing {execution_phrase} in sql.")
+    print(f"Executing '{execution_phrase}' in sql.")
     cursor.execute(execution_phrase)
     cursor.close()
 
@@ -34,7 +34,7 @@ from .models import *
 
 def update_log(mapper, connection, target):
     state = inspect(target)
-    object_name = state.object.truncated_name()
+    object_name = state.object.truncated_name
     update = dict(user=getuser(), time=datetime.now(), object=object_name, changes=[])
     for attr in state.attrs:
         hist = attr.load_history()

src/submissions/backend/db/models/__init__.py

@@ -24,6 +24,7 @@ logger = logging.getLogger(f"submissions.{__name__}")
 class LogMixin(Base):
     __abstract__ = True
 
+    @property
     def truncated_name(self):
         name = str(self)
         if len(name) > 64:

src/submissions/backend/db/models/controls.py

@@ -368,7 +368,7 @@ class IridaControl(Control):
                            polymorphic_load="inline",
                            inherit_condition=(id == Control.id))
 
-    @validates("sub_type")
+    @validates("subtype")
     def enforce_subtype_literals(self, key: str, value: str) -> str:
         """
         Validates sub_type field with acceptable values

src/submissions/backend/db/models/kits.py

@@ -738,7 +738,13 @@ class SubmissionType(BaseClass):
         return f"<SubmissionType({self.name})>"
 
     @classmethod
-    def retrieve_template_file(cls):
+    def retrieve_template_file(cls) -> bytes:
+        """
+        Grabs the default excel template file.
+
+        Returns:
+            bytes: The excel sheet.
+        """
         submission_type = cls.query(name="Bacterial Culture")
         return submission_type.template_file
 

src/submissions/backend/excel/reports.py

@@ -145,6 +145,7 @@ class ReportMaker(object):
             if cell.row > 1:
                 cell.style = 'Currency'
 
+
 class TurnaroundMaker(ReportArchetype):
 
     def __init__(self, start_date: date, end_date: date, submission_type:str):

src/submissions/backend/scripts/__init__.py

@@ -1,8 +0,0 @@
-from pathlib import Path
-import importlib
-
-p = Path(__file__).parent.absolute()
-subs = [item.stem for item in p.glob("*.py") if "__" not in item.stem]
-modules = {}
-for sub in subs:
-    importlib.import_module(f"backend.scripts.{sub}")

src/submissions/backend/validators/pydant.py

@@ -901,7 +901,7 @@ class PydSubmission(BaseModel, extra='allow'):
         return render
 
     # @report_result
-    def check_kit_integrity(self, extraction_kit: str | dict | None = None, exempt:List[PydReagent]=[]) -> Tuple[
+    def check_kit_integrity(self, extraction_kit: str | dict | None = None, exempt: List[PydReagent] = []) -> Tuple[
         List[PydReagent], Report]:
         """
         Ensures all reagents expected in kit are listed in Submission
@@ -929,16 +929,40 @@ class PydSubmission(BaseModel, extra='allow'):
         missing_reagents = [rt for rt in ext_kit_rtypes if rt.role not in missing_check and rt.role not in exempt]
         # logger.debug(f"Missing reagents: {missing_reagents}")
         missing_reagents += [rt for rt in output_reagents if rt.missing]
+        logger.debug(pformat(missing_reagents))
         output_reagents += [rt for rt in missing_reagents if rt not in output_reagents]
         # NOTE: if lists are equal return no problem
         if len(missing_reagents) == 0:
             result = None
         else:
             result = Result(
-                    msg=f"The excel sheet you are importing is missing some reagents expected by the kit.\n\nIt looks like you are missing: {[item.role.upper() for item in missing_reagents]}\n\nAlternatively, you may have set the wrong extraction kit.\n\nThe program will populate lists using existing reagents.\n\nPlease make sure you check the lots carefully!",
+                msg=f"The excel sheet you are importing is missing some reagents expected by the kit.\n\nIt looks like you are missing: {[item.role.upper() for item in missing_reagents]}\n\nAlternatively, you may have set the wrong extraction kit.\n\nThe program will populate lists using existing reagents.\n\nPlease make sure you check the lots carefully!",
-                    status="Warning")
+                status="Warning")
         report.add_result(result)
-        return output_reagents, report
+        return output_reagents, report, missing_reagents
+
+    def check_reagent_expiries(self, exempt: List[PydReagent]=[]):
+        report = Report()
+        expired = []
+        for reagent in self.reagents:
+            if reagent not in exempt:
+                role_expiry = ReagentRole.query(name=reagent.role).eol_ext
+                try:
+                    dt = datetime.combine(reagent.expiry, datetime.min.time())
+                except TypeError:
+                    continue
+                if datetime.now() > dt + role_expiry:
+                    expired.append(f"{reagent.role}, {reagent.lot}: {reagent.expiry} + {role_expiry.days}")
+        if expired:
+            output = '\n'.join(expired)
+            result = Result(status="Warning",
+                            msg = f"The following reagents are expired:\n\n{output}"
+                            )
+            report.add_result(result)
+        return report
+
+
+
 
     def export_csv(self, filename: Path | str):
         try:

src/submissions/frontend/visualizations/pcr_charts.py

@@ -32,5 +32,3 @@ class PCRFigure(CustomFigure):
             scatter = px.scatter()
         self.add_traces(scatter.data)
         self.update_traces(marker={'size': 15})
-
-

src/submissions/frontend/widgets/app.py

@@ -15,7 +15,7 @@ from __init__ import project_path
 from backend import SubmissionType, Reagent, BasicSample
 from tools import check_if_app, Settings, Report, jinja_template_loading, check_authorization, page_size
 from .functions import select_save_file, select_open_file
-from datetime import date
+# from datetime import date
 from .pop_ups import HTMLPop, AlertPop
 from .misc import Pagifier
 import logging, webbrowser, sys, shutil

src/submissions/frontend/widgets/omni_search.py

@@ -1,5 +1,5 @@
 """
-Search box that performs fuzzy search for samples
+Search box that performs fuzzy search for various object types
 """
 from pprint import pformat
 from typing import Tuple, Any, List

src/submissions/frontend/widgets/submission_details.py

@@ -174,6 +174,7 @@ class SubmissionDetails(QDialog):
         fname = select_save_file(obj=self, default_name=self.export_plate, extension="pdf")
         save_pdf(obj=self.webview, filename=fname)
 
+
 class SubmissionComment(QDialog):
     """
     a window for adding comment text to a submission

src/submissions/frontend/widgets/submission_widget.py

@@ -283,11 +283,13 @@ class SubmissionFormWidget(QWidget):
         for reagent in old_reagents:
             if isinstance(reagent, self.ReagentFormWidget) or isinstance(reagent, QPushButton):
                 reagent.setParent(None)
-        reagents, integrity_report = self.pyd.check_kit_integrity(extraction_kit=self.extraction_kit)
+        reagents, integrity_report, missing_reagents = self.pyd.check_kit_integrity(extraction_kit=self.extraction_kit)
+        expiry_report = self.pyd.check_reagent_expiries(exempt=missing_reagents)
         for reagent in reagents:
             add_widget = self.ReagentFormWidget(parent=self, reagent=reagent, extraction_kit=self.extraction_kit)
             self.layout.addWidget(add_widget)
         report.add_result(integrity_report)
+        report.add_result(expiry_report)
         if hasattr(self.pyd, "csv"):
             export_csv_btn = QPushButton("Export CSV")
             export_csv_btn.setObjectName("export_csv_btn")
@@ -338,13 +340,11 @@ class SubmissionFormWidget(QWidget):
         report = Report()
         result = self.parse_form()
         report.add_result(result)
-        # allow = not all([item.lot.isEnabled() for item in self.findChildren(self.ReagentFormWidget)])
         exempt = [item.reagent.role for item in self.findChildren(self.ReagentFormWidget) if not item.lot.isEnabled()]
-        # if allow:
-        #     logger.warning(f"Some reagents are disabled, allowing incomplete kit.")
         if self.disabler.checkbox.isChecked():
-            _, result = self.pyd.check_kit_integrity(exempt=exempt)
+            _, result, _ = self.pyd.check_kit_integrity(exempt=exempt)
             report.add_result(result)
+            # result = self.pyd.check_reagent_expiries(exempt=exempt)
         if len(result.results) > 0:
             return report
         base_submission, result = self.pyd.to_sql()

src/submissions/tools/__init__.py

@@ -2,11 +2,12 @@
 Contains miscellaenous functions used by both frontend and backend.
 '''
 from __future__ import annotations
+
+import importlib
+import time
 from datetime import date, datetime, timedelta
 from json import JSONDecodeError
-from pprint import pprint
+import logging, re, yaml, sys, os, stat, platform, getpass, inspect, json, numpy as np, pandas as pd
-import numpy as np
-import logging, re, yaml, sys, os, stat, platform, getpass, inspect, json, pandas as pd
 from dateutil.easter import easter
 from jinja2 import Environment, FileSystemLoader
 from logging import handlers
@@ -22,6 +23,7 @@ from tkinter import Tk  # NOTE: This is for choosing database path before app is
 from tkinter.filedialog import askdirectory
 from sqlalchemy.exc import IntegrityError as sqlalcIntegrityError
 from pytz import timezone as tz
+from functools import wraps
 
 timezone = tz("America/Winnipeg")
 
@@ -44,6 +46,7 @@ LOGDIR = main_aux_dir.joinpath("logs")
 row_map = {1: "A", 2: "B", 3: "C", 4: "D", 5: "E", 6: "F", 7: "G", 8: "H"}
 row_keys = {v: k for k, v in row_map.items()}
 
+# NOTE: Sets background for uneditable comboboxes and date edits.
 main_form_style = '''
                         QComboBox:!editable, QDateEdit {
                             background-color:light gray;
@@ -53,6 +56,7 @@ main_form_style = '''
 
 page_size = 250
 
+
 def divide_chunks(input_list: list, chunk_count: int):
     """
     Divides a list into {chunk_count} equal parts
@@ -417,6 +421,7 @@ class Settings(BaseSettings, extra="allow"):
 
         super().__init__(*args, **kwargs)
         self.set_from_db()
+        # self.set_startup_teardown()
         # pprint(f"User settings:\n{self.__dict__}")
 
     def set_from_db(self):
@@ -448,6 +453,15 @@ class Settings(BaseSettings, extra="allow"):
             if not hasattr(self, k):
                 self.__setattr__(k, v)
 
+    def set_scripts(self):
+        """
+        Imports all functions from "scripts" folder which will run their @registers, adding them to ctx scripts
+        """
+        p = Path(__file__).parent.joinpath("scripts").absolute()
+        subs = [item.stem for item in p.glob("*.py") if "__" not in item.stem]
+        for sub in subs:
+            importlib.import_module(f"tools.scripts.{sub}")
+
     @classmethod
     def get_alembic_db_path(cls, alembic_path, mode=Literal['path', 'schema', 'user', 'pass']) -> Path | str:
         c = ConfigParser()
@@ -514,6 +528,7 @@ def get_config(settings_path: Path | str | None = None) -> Settings:
     def join(loader, node):
         seq = loader.construct_sequence(node)
         return ''.join([str(i) for i in seq])
+
     # NOTE: register the tag handler
     yaml.add_constructor('!join', join)
     # NOTE: make directories
@@ -738,6 +753,7 @@ def setup_lookup(func):
         func (_type_): wrapped function
     """
 
+    @wraps(func)
     def wrapper(*args, **kwargs):
         sanitized_kwargs = {}
         for k, v in locals()['kwargs'].items():
@@ -881,7 +897,7 @@ def yaml_regex_creator(loader, node):
     return f"(?P<{name}>RSL(?:-|_)?{abbr}(?:-|_)?20\d{2}-?\d{2}-?\d{2}(?:(_|-)?\d?([^_0123456789\sA-QS-Z]|$)?R?\d?)?)"
 
 
-def super_splitter(ins_str:str, substring:str, idx:int) -> str:
+def super_splitter(ins_str: str, substring: str, idx: int) -> str:
     """
 
     Args:
@@ -898,9 +914,6 @@ def super_splitter(ins_str:str, substring:str, idx:int) -> str:
         return ins_str
 
 
-ctx = get_config(None)
-
-
 def is_power_user() -> bool:
     """
     Checks if user is in list of power users
@@ -930,8 +943,11 @@ def check_authorization(func):
         else:
             logger.error(f"User {getpass.getuser()} is not authorized for this function.")
             report = Report()
-            report.add_result(Result(owner=func.__str__(), code=1, msg="This user does not have permission for this function.", status="warning"))
+            report.add_result(
+                Result(owner=func.__str__(), code=1, msg="This user does not have permission for this function.",
+                       status="warning"))
             return report
+
     return wrapper
 
 
@@ -946,6 +962,8 @@ def report_result(func):
         __type__: Output from decorated function
 
     """
+
+    @wraps(func)
     def wrapper(*args, **kwargs):
         logger.info(f"Report result being called by {func.__name__}")
         output = func(*args, **kwargs)
@@ -980,16 +998,17 @@ def report_result(func):
         else:
             true_output = None
         return true_output
+
     return wrapper
 
 
-def create_holidays_for_year(year: int|None=None) -> List[date]:
+def create_holidays_for_year(year: int | None = None) -> List[date]:
-    def find_nth_monday(year, month, occurence: int | None=None, day: int|None=None):
+    def find_nth_monday(year, month, occurence: int | None = None, day: int | None = None):
         if not occurence:
             occurence = 1
         if not day:
             day = occurence * 7
-        max_days = (date(2012, month+1, 1) - date(2012, month, 1)).days
+        max_days = (date(2012, month + 1, 1) - date(2012, month, 1)).days
         if day > max_days:
             day = max_days
         try:
@@ -999,12 +1018,13 @@ def create_holidays_for_year(year: int|None=None) -> List[date]:
         offset = -d.weekday()  # weekday == 0 means Monday
         output = d + timedelta(offset)
         return output.date()
+
     if not year:
         year = date.today().year
     # Includes New Year's day for next year.
-    holidays = [date(year, 1, 1), date(year, 7,1), date(year, 9, 30),
+    holidays = [date(year, 1, 1), date(year, 7, 1), date(year, 9, 30),
                 date(year, 11, 11), date(year, 12, 25), date(year, 12, 26),
-                date(year+1, 1, 1)]
+                date(year + 1, 1, 1)]
     # Labour Day
     holidays.append(find_nth_monday(year, 9))
     # Thanksgiving
@@ -1015,3 +1035,39 @@ def create_holidays_for_year(year: int|None=None) -> List[date]:
     holidays.append(easter(year) - timedelta(days=2))
     holidays.append(easter(year) + timedelta(days=1))
     return sorted(holidays)
+
+
+def timer(func):
+    """
+    Performs timing of wrapped function
+
+    Args:
+        func (__function__): incoming function
+
+    """
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        start_time = time.perf_counter()
+        value = func(*args, **kwargs)
+        end_time = time.perf_counter()
+        run_time = end_time - start_time
+        logger.debug(f"Finished {func.__name__}() in {run_time:.4f} secs")
+        return value
+
+    return wrapper
+
+
+ctx = get_config(None)
+
+
+def register_script(func):
+    """Register a function as a plug-in"""
+    if func.__name__ in ctx.startup_scripts.keys():
+        ctx.startup_scripts[func.__name__] = func
+    if func.__name__ in ctx.teardown_scripts.keys():
+        ctx.teardown_scripts[func.__name__] = func
+    return func
+
+
+ctx.set_scripts()

src/submissions/tools/scripts/backup_database.py

@@ -1,16 +1,16 @@
 """
 script meant to copy database data to new file. Currently for Sqlite only
 """
-import logging, shutil
+import logging, shutil, pyodbc
 from datetime import date
 from pathlib import Path
 from tools import Settings
-import pyodbc
+from .. import register_script
 
 logger = logging.getLogger(f"submissions.{__name__}")
 
-
+@register_script
-def script(ctx: Settings):
+def backup_database(ctx: Settings):
     """
     Copies the database into the backup directory the first time it is opened every month.
     """

src/submissions/tools/scripts/goodbye.py

@@ -1,5 +1,9 @@
 """
 Test script for teardown_scripts
 """
-def script(ctx):
+
+from .. import register_script
+
+@register_script
+def goodbye(ctx):
     print("\n\nGoodbye. Thank you for using Robotics Submission Tracker.\n\n")

src/submissions/tools/scripts/hello.py

@@ -1,5 +1,8 @@
 """
 Test script for startup_scripts
 """
-def script(ctx):
+from .. import register_script
+
+@register_script
+def hello(ctx):
     print("\n\nHello! Welcome to Robotics Submission Tracker.\n\n")

src/submissions/tools/scripts/import_irida.py

@@ -2,25 +2,25 @@ import logging, sqlite3, json
 from pprint import pformat, pprint
 from datetime import datetime
 from tools import Settings
-from backend import BasicSample
+
-from backend.db import IridaControl, ControlType
 from sqlalchemy.orm import Session
+from .. import register_script
 
 logger = logging.getLogger(f"submissions.{__name__}")
 
-
+@register_script
-def script(ctx: Settings):
+def import_irida(ctx: Settings):
     """
     Grabs Irida controls from secondary database.
 
     Args:
         ctx (Settings): Settings inherited from app.
     """
+    from backend import BasicSample
+    from backend.db import IridaControl, ControlType
     # NOTE: Because the main session will be busy in another thread, this requires a new session.
     new_session = Session(ctx.database_session.get_bind())
-    # ct = ControlType.query(name="Irida Control")
     ct = new_session.query(ControlType).filter(ControlType.name == "Irida Control").first()
-    # existing_controls = [item.name for item in IridaControl.query()]
     existing_controls = [item.name for item in new_session.query(IridaControl)]
     prm_list = ", ".join([f"'{thing}'" for thing in existing_controls])
     ctrl_db_path = ctx.directory_path.joinpath("submissions_parser_output", "submissions.db")