landon/Submissions-App
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Debugging scripts import hell.

Browse Source
This commit is contained in:
lwark
2024-12-30 08:37:41 -06:00
parent 5fd36308b2
commit 0808b54264
22 changed files with 156 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGELOG.md
View File

@@ -1,6 +1,11 @@
# 202412.05
- Switched startup/teardown scripts to decorator registration.
# 202412.04
- Update of wastewater to allow for duplex PCR primers.
- Addition of expiry check after kit integrity check.
## 202412.03

1
TODO.md
View File

@@ -31,7 +31,6 @@
- [x] Create platemap image from html for export to pdf.
- [x] Move plate map maker to submission.
- [x] Finish Equipment Parser (add in regex to id asset_number)
- [ ] Complete info_map in the SubmissionTypeCreator widget.
- [x] Update Artic and add in equipment listings... *sigh*.
- [x] Fix WastewaterAssociations not in Session error.
- Done... I think?

1
src/config.yml
View File

@@ -4,4 +4,3 @@ database_schema: null
database_user: null
database_password: null
database_name: null
logging_enabled: false

2
src/submissions/__init__.py
View File

@@ -14,7 +14,7 @@ def get_week_of_month() -> int:
Gets the current week number of the month.
Returns:
int:
int: 1 if first week of month, etc.
"""
for ii, week in enumerate(calendar.monthcalendar(date.today().year, date.today().month)):
if day in week:

42
src/submissions/__main__.py
View File

@@ -1,6 +1,7 @@
import sys, os
from tools import ctx, setup_logger, check_if_app
from tools import ctx, setup_logger, check_if_app, timer
from threading import Thread
# environment variable must be set to enable qtwebengine in network path
if check_if_app():
os.environ['QTWEBENGINE_DISABLE_SANDBOX'] = "1"
@@ -8,46 +9,27 @@ if check_if_app():
# setup custom logger
logger = setup_logger(verbosity=3)
# from backend.scripts import modules
from backend import scripts
# from backend import scripts
from PyQt6.QtWidgets import QApplication
from frontend.widgets.app import App
@timer
def run_startup():
try:
startup_scripts = ctx.startup_scripts
except AttributeError as e:
logger.error(f"Couldn't get startup scripts due to {e}")
return
for script in startup_scripts:
try:
func = getattr(scripts, script)
except AttributeError as e:
logger.error(f"Couldn't run startup script {script} due to {e}")
continue
logger.info(f"Running startup script: {func.__name__}")
thread = Thread(target=func.script, args=(ctx, ))
for script in ctx.startup_scripts.values():
logger.info(f"Running startup script: {script.__name__}")
thread = Thread(target=script, args=(ctx,))
thread.start()
@timer
def run_teardown():
try:
teardown_scripts = ctx.teardown_scripts
except AttributeError as e:
logger.error(f"Couldn't get teardown scripts due to {e}")
return
for script in teardown_scripts:
try:
func = getattr(scripts, script)
# func = modules[script]
except AttributeError as e:
logger.error(f"Couldn't run teardown script {script} due to {e}")
continue
logger.info(f"Running teardown script: {func.__name__}")
thread = Thread(target=func.script, args=(ctx,))
for script in ctx.teardown_scripts.values():
logger.info(f"Running teardown script: {script.__name__}")
thread = Thread(target=script, args=(ctx,))
thread.start()
if __name__ == '__main__':
run_startup()
app = QApplication(['', '--no-sandbox'])

6
src/submissions/backend/db/__init__.py
View File

@@ -21,10 +21,10 @@ def set_sqlite_pragma(dbapi_connection, connection_record):
if ctx.database_schema == "sqlite":
execution_phrase = "PRAGMA foreign_keys=ON"
else:
print("Nothing to execute, returning")
# print("Nothing to execute, returning")
cursor.close()
return
print(f"Executing {execution_phrase} in sql.")
print(f"Executing '{execution_phrase}' in sql.")
cursor.execute(execution_phrase)
cursor.close()
@@ -34,7 +34,7 @@ from .models import *
def update_log(mapper, connection, target):
state = inspect(target)
object_name = state.object.truncated_name()
object_name = state.object.truncated_name
update = dict(user=getuser(), time=datetime.now(), object=object_name, changes=[])
for attr in state.attrs:
hist = attr.load_history()

1
src/submissions/backend/db/models/__init__.py
View File

@@ -24,6 +24,7 @@ logger = logging.getLogger(f"submissions.{__name__}")
class LogMixin(Base):
__abstract__ = True
@property
def truncated_name(self):
name = str(self)
if len(name) > 64:

2
src/submissions/backend/db/models/controls.py
View File

@@ -368,7 +368,7 @@ class IridaControl(Control):
polymorphic_load="inline",
inherit_condition=(id == Control.id))
@validates("sub_type")
@validates("subtype")
def enforce_subtype_literals(self, key: str, value: str) -> str:
"""
Validates sub_type field with acceptable values

8
src/submissions/backend/db/models/kits.py
View File

@@ -738,7 +738,13 @@ class SubmissionType(BaseClass):
return f"<SubmissionType({self.name})>"
@classmethod
def retrieve_template_file(cls):
def retrieve_template_file(cls) -> bytes:
"""
Grabs the default excel template file.
Returns:
bytes: The excel sheet.
"""
submission_type = cls.query(name="Bacterial Culture")
return submission_type.template_file

1
src/submissions/backend/excel/reports.py
View File

@@ -145,6 +145,7 @@ class ReportMaker(object):
if cell.row > 1:
cell.style = 'Currency'
class TurnaroundMaker(ReportArchetype):
def __init__(self, start_date: date, end_date: date, submission_type:str):

8
src/submissions/backend/scripts/__init__.py
View File

@@ -1,8 +0,0 @@
from pathlib import Path
import importlib
p = Path(__file__).parent.absolute()
subs = [item.stem for item in p.glob("*.py") if "__" not in item.stem]
modules = {}
for sub in subs:
importlib.import_module(f"backend.scripts.{sub}")

28
src/submissions/backend/validators/pydant.py
View File

@@ -901,7 +901,7 @@ class PydSubmission(BaseModel, extra='allow'):
return render
# @report_result
def check_kit_integrity(self, extraction_kit: str | dict | None = None, exempt:List[PydReagent]=[]) -> Tuple[
def check_kit_integrity(self, extraction_kit: str | dict | None = None, exempt: List[PydReagent] = []) -> Tuple[
List[PydReagent], Report]:
"""
Ensures all reagents expected in kit are listed in Submission
@@ -929,6 +929,7 @@ class PydSubmission(BaseModel, extra='allow'):
missing_reagents = [rt for rt in ext_kit_rtypes if rt.role not in missing_check and rt.role not in exempt]
# logger.debug(f"Missing reagents: {missing_reagents}")
missing_reagents += [rt for rt in output_reagents if rt.missing]
logger.debug(pformat(missing_reagents))
output_reagents += [rt for rt in missing_reagents if rt not in output_reagents]
# NOTE: if lists are equal return no problem
if len(missing_reagents) == 0:
@@ -938,7 +939,30 @@ class PydSubmission(BaseModel, extra='allow'):
msg=f"The excel sheet you are importing is missing some reagents expected by the kit.\n\nIt looks like you are missing: {[item.role.upper() for item in missing_reagents]}\n\nAlternatively, you may have set the wrong extraction kit.\n\nThe program will populate lists using existing reagents.\n\nPlease make sure you check the lots carefully!",
status="Warning")
report.add_result(result)
return output_reagents, report
return output_reagents, report, missing_reagents
def check_reagent_expiries(self, exempt: List[PydReagent]=[]):
report = Report()
expired = []
for reagent in self.reagents:
if reagent not in exempt:
role_expiry = ReagentRole.query(name=reagent.role).eol_ext
try:
dt = datetime.combine(reagent.expiry, datetime.min.time())
except TypeError:
continue
if datetime.now() > dt + role_expiry:
expired.append(f"{reagent.role}, {reagent.lot}: {reagent.expiry} + {role_expiry.days}")
if expired:
output = '\n'.join(expired)
result = Result(status="Warning",
msg = f"The following reagents are expired:\n\n{output}"
)
report.add_result(result)
return report
def export_csv(self, filename: Path | str):
try:

2
src/submissions/frontend/visualizations/pcr_charts.py
View File

@@ -32,5 +32,3 @@ class PCRFigure(CustomFigure):
scatter = px.scatter()
self.add_traces(scatter.data)
self.update_traces(marker={'size': 15})

2
src/submissions/frontend/widgets/app.py
View File

@@ -15,7 +15,7 @@ from __init__ import project_path
from backend import SubmissionType, Reagent, BasicSample
from tools import check_if_app, Settings, Report, jinja_template_loading, check_authorization, page_size
from .functions import select_save_file, select_open_file
from datetime import date
# from datetime import date
from .pop_ups import HTMLPop, AlertPop
from .misc import Pagifier
import logging, webbrowser, sys, shutil

2
src/submissions/frontend/widgets/omni_search.py
View File

@@ -1,5 +1,5 @@
"""
Search box that performs fuzzy search for samples
Search box that performs fuzzy search for various object types
"""
from pprint import pformat
from typing import Tuple, Any, List

1
src/submissions/frontend/widgets/submission_details.py
View File

@@ -174,6 +174,7 @@ class SubmissionDetails(QDialog):
fname = select_save_file(obj=self, default_name=self.export_plate, extension="pdf")
save_pdf(obj=self.webview, filename=fname)
class SubmissionComment(QDialog):
"""
a window for adding comment text to a submission

10
src/submissions/frontend/widgets/submission_widget.py
View File

@@ -283,11 +283,13 @@ class SubmissionFormWidget(QWidget):
for reagent in old_reagents:
if isinstance(reagent, self.ReagentFormWidget) or isinstance(reagent, QPushButton):
reagent.setParent(None)
reagents, integrity_report = self.pyd.check_kit_integrity(extraction_kit=self.extraction_kit)
reagents, integrity_report, missing_reagents = self.pyd.check_kit_integrity(extraction_kit=self.extraction_kit)
expiry_report = self.pyd.check_reagent_expiries(exempt=missing_reagents)
for reagent in reagents:
add_widget = self.ReagentFormWidget(parent=self, reagent=reagent, extraction_kit=self.extraction_kit)
self.layout.addWidget(add_widget)
report.add_result(integrity_report)
report.add_result(expiry_report)
if hasattr(self.pyd, "csv"):
export_csv_btn = QPushButton("Export CSV")
export_csv_btn.setObjectName("export_csv_btn")
@@ -338,13 +340,11 @@ class SubmissionFormWidget(QWidget):
report = Report()
result = self.parse_form()
report.add_result(result)
# allow = not all([item.lot.isEnabled() for item in self.findChildren(self.ReagentFormWidget)])
exempt = [item.reagent.role for item in self.findChildren(self.ReagentFormWidget) if not item.lot.isEnabled()]
# if allow:
# logger.warning(f"Some reagents are disabled, allowing incomplete kit.")
if self.disabler.checkbox.isChecked():
_, result = self.pyd.check_kit_integrity(exempt=exempt)
_, result, _ = self.pyd.check_kit_integrity(exempt=exempt)
report.add_result(result)
# result = self.pyd.check_reagent_expiries(exempt=exempt)
if len(result.results) > 0:
return report
base_submission, result = self.pyd.to_sql()

82
src/submissions/tools/__init__.py
View File

@@ -2,11 +2,12 @@
Contains miscellaenous functions used by both frontend and backend.
'''
from __future__ import annotations
import importlib
import time
from datetime import date, datetime, timedelta
from json import JSONDecodeError
from pprint import pprint
import numpy as np
import logging, re, yaml, sys, os, stat, platform, getpass, inspect, json, pandas as pd
import logging, re, yaml, sys, os, stat, platform, getpass, inspect, json, numpy as np, pandas as pd
from dateutil.easter import easter
from jinja2 import Environment, FileSystemLoader
from logging import handlers
@@ -22,6 +23,7 @@ from tkinter import Tk # NOTE: This is for choosing database path before app is
from tkinter.filedialog import askdirectory
from sqlalchemy.exc import IntegrityError as sqlalcIntegrityError
from pytz import timezone as tz
from functools import wraps
timezone = tz("America/Winnipeg")
@@ -44,6 +46,7 @@ LOGDIR = main_aux_dir.joinpath("logs")
row_map = {1: "A", 2: "B", 3: "C", 4: "D", 5: "E", 6: "F", 7: "G", 8: "H"}
row_keys = {v: k for k, v in row_map.items()}
# NOTE: Sets background for uneditable comboboxes and date edits.
main_form_style = '''
QComboBox:!editable, QDateEdit {
background-color:light gray;
@@ -53,6 +56,7 @@ main_form_style = '''
page_size = 250
def divide_chunks(input_list: list, chunk_count: int):
"""
Divides a list into {chunk_count} equal parts
@@ -417,6 +421,7 @@ class Settings(BaseSettings, extra="allow"):
super().__init__(*args, **kwargs)
self.set_from_db()
# self.set_startup_teardown()
# pprint(f"User settings:\n{self.__dict__}")
def set_from_db(self):
@@ -448,6 +453,15 @@ class Settings(BaseSettings, extra="allow"):
if not hasattr(self, k):
self.__setattr__(k, v)
def set_scripts(self):
"""
Imports all functions from "scripts" folder which will run their @registers, adding them to ctx scripts
"""
p = Path(__file__).parent.joinpath("scripts").absolute()
subs = [item.stem for item in p.glob("*.py") if "__" not in item.stem]
for sub in subs:
importlib.import_module(f"tools.scripts.{sub}")
@classmethod
def get_alembic_db_path(cls, alembic_path, mode=Literal['path', 'schema', 'user', 'pass']) -> Path | str:
c = ConfigParser()
@@ -514,6 +528,7 @@ def get_config(settings_path: Path | str | None = None) -> Settings:
def join(loader, node):
seq = loader.construct_sequence(node)
return ''.join([str(i) for i in seq])
# NOTE: register the tag handler
yaml.add_constructor('!join', join)
# NOTE: make directories
@@ -738,6 +753,7 @@ def setup_lookup(func):
func (_type_): wrapped function
"""
@wraps(func)
def wrapper(*args, **kwargs):
sanitized_kwargs = {}
for k, v in locals()['kwargs'].items():
@@ -881,7 +897,7 @@ def yaml_regex_creator(loader, node):
return f"(?P<{name}>RSL(?:-|_)?{abbr}(?:-|_)?20\d{2}-?\d{2}-?\d{2}(?:(_|-)?\d?([^_0123456789\sA-QS-Z]|$)?R?\d?)?)"
def super_splitter(ins_str:str, substring:str, idx:int) -> str:
def super_splitter(ins_str: str, substring: str, idx: int) -> str:
"""
Args:
@@ -898,9 +914,6 @@ def super_splitter(ins_str:str, substring:str, idx:int) -> str:
return ins_str
ctx = get_config(None)
def is_power_user() -> bool:
"""
Checks if user is in list of power users
@@ -930,8 +943,11 @@ def check_authorization(func):
else:
logger.error(f"User {getpass.getuser()} is not authorized for this function.")
report = Report()
report.add_result(Result(owner=func.__str__(), code=1, msg="This user does not have permission for this function.", status="warning"))
report.add_result(
Result(owner=func.__str__(), code=1, msg="This user does not have permission for this function.",
status="warning"))
return report
return wrapper
@@ -946,6 +962,8 @@ def report_result(func):
__type__: Output from decorated function
"""
@wraps(func)
def wrapper(*args, **kwargs):
logger.info(f"Report result being called by {func.__name__}")
output = func(*args, **kwargs)
@@ -980,16 +998,17 @@ def report_result(func):
else:
true_output = None
return true_output
return wrapper
def create_holidays_for_year(year: int|None=None) -> List[date]:
def find_nth_monday(year, month, occurence: int | None=None, day: int|None=None):
def create_holidays_for_year(year: int | None = None) -> List[date]:
def find_nth_monday(year, month, occurence: int | None = None, day: int | None = None):
if not occurence:
occurence = 1
if not day:
day = occurence * 7
max_days = (date(2012, month+1, 1) - date(2012, month, 1)).days
max_days = (date(2012, month + 1, 1) - date(2012, month, 1)).days
if day > max_days:
day = max_days
try:
@@ -999,12 +1018,13 @@ def create_holidays_for_year(year: int|None=None) -> List[date]:
offset = -d.weekday() # weekday == 0 means Monday
output = d + timedelta(offset)
return output.date()
if not year:
year = date.today().year
# Includes New Year's day for next year.
holidays = [date(year, 1, 1), date(year, 7,1), date(year, 9, 30),
holidays = [date(year, 1, 1), date(year, 7, 1), date(year, 9, 30),
date(year, 11, 11), date(year, 12, 25), date(year, 12, 26),
date(year+1, 1, 1)]
date(year + 1, 1, 1)]
# Labour Day
holidays.append(find_nth_monday(year, 9))
# Thanksgiving
@@ -1015,3 +1035,39 @@ def create_holidays_for_year(year: int|None=None) -> List[date]:
holidays.append(easter(year) - timedelta(days=2))
holidays.append(easter(year) + timedelta(days=1))
return sorted(holidays)
def timer(func):
"""
Performs timing of wrapped function
Args:
func (__function__): incoming function
"""
@wraps(func)
def wrapper(*args, **kwargs):
start_time = time.perf_counter()
value = func(*args, **kwargs)
end_time = time.perf_counter()
run_time = end_time - start_time
logger.debug(f"Finished {func.__name__}() in {run_time:.4f} secs")
return value
return wrapper
ctx = get_config(None)
def register_script(func):
"""Register a function as a plug-in"""
if func.__name__ in ctx.startup_scripts.keys():
ctx.startup_scripts[func.__name__] = func
if func.__name__ in ctx.teardown_scripts.keys():
ctx.teardown_scripts[func.__name__] = func
return func
ctx.set_scripts()

8
src/submissions/backend/scripts/backup_database.py → src/submissions/tools/scripts/backup_database.py
View File

@@ -1,16 +1,16 @@
"""
script meant to copy database data to new file. Currently for Sqlite only
"""
import logging, shutil
import logging, shutil, pyodbc
from datetime import date
from pathlib import Path
from tools import Settings
import pyodbc
from .. import register_script
logger = logging.getLogger(f"submissions.{__name__}")
def script(ctx: Settings):
@register_script
def backup_database(ctx: Settings):
"""
Copies the database into the backup directory the first time it is opened every month.
"""

6
src/submissions/backend/scripts/goodbye.py → src/submissions/tools/scripts/goodbye.py
View File

@@ -1,5 +1,9 @@
"""
Test script for teardown_scripts
"""
def script(ctx):
from .. import register_script
@register_script
def goodbye(ctx):
print("\n\nGoodbye. Thank you for using Robotics Submission Tracker.\n\n")

5
src/submissions/backend/scripts/hello.py → src/submissions/tools/scripts/hello.py
View File

@@ -1,5 +1,8 @@
"""
Test script for startup_scripts
"""
def script(ctx):
from .. import register_script
@register_script
def hello(ctx):
print("\n\nHello! Welcome to Robotics Submission Tracker.\n\n")

12
src/submissions/backend/scripts/import_irida.py → src/submissions/tools/scripts/import_irida.py
View File

@@ -2,25 +2,25 @@ import logging, sqlite3, json
from pprint import pformat, pprint
from datetime import datetime
from tools import Settings
from backend import BasicSample
from backend.db import IridaControl, ControlType
from sqlalchemy.orm import Session
from .. import register_script
logger = logging.getLogger(f"submissions.{__name__}")
def script(ctx: Settings):
@register_script
def import_irida(ctx: Settings):
"""
Grabs Irida controls from secondary database.
Args:
ctx (Settings): Settings inherited from app.
"""
from backend import BasicSample
from backend.db import IridaControl, ControlType
# NOTE: Because the main session will be busy in another thread, this requires a new session.
new_session = Session(ctx.database_session.get_bind())
# ct = ControlType.query(name="Irida Control")
ct = new_session.query(ControlType).filter(ControlType.name == "Irida Control").first()
# existing_controls = [item.name for item in IridaControl.query()]
existing_controls = [item.name for item in new_session.query(IridaControl)]
prm_list = ", ".join([f"'{thing}'" for thing in existing_controls])
ctrl_db_path = ctx.directory_path.joinpath("submissions_parser_output", "submissions.db")