landon/Submissions-App
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Upgrade of WastewaterArtic parsers/writers for flexibility.

Browse Source
This commit is contained in:
lwark
2024-07-09 13:09:27 -05:00
parent e589420815
commit 54e1e55804
11 changed files with 122 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGELOG.md
View File

@@ -1,3 +1,8 @@
## 202407.02
- HTML template for 'About'.
- More flexible custom parsers/writers due to custom info items.
## 202407.01
- Better documentation.

5
README.md
View File

@@ -1,7 +1,7 @@
## Startup:
1. Open the app using the shortcut in the Submissions folder. For example: L:\\Robotics Laboratory Support\\Submissions\\submissions_v122b.exe - Shortcut.lnk (Version may have changed).
a. Ignore the large black window of fast scrolling text, it is there for debugging purposes.
b. The 'Submissions' tab should be open by default.
1. Ignore the large black window of fast scrolling text, it is there for debugging purposes.
2. The 'Submissions' tab should be open by default.
## Logging in New Run:
*should fit 90% of usage cases*
@@ -32,6 +32,7 @@
## Importing PCR results (Wastewater only):
This is meant to import .xslx files created from the Design & Analysis Software
1. Click on 'File' -> 'Import PCR Results'.
2. Use the file dialog to locate the .xlsx file you want to import.
3. Click 'Okay'.

2
src/submissions/__init__.py
View File

@@ -15,10 +15,12 @@ def get_week_of_month() -> int:
if day in week:
return ii + 1
# Automatically completes project info for help menu and compiling.
__project__ = "submissions"
__version__ = f"{year}{str(month).zfill(2)}.{get_week_of_month()}b"
__author__ = {"name":"Landon Wark", "email":"Landon.Wark@phac-aspc.gc.ca"}
__copyright__ = f"2022-{date.today().year}, Government of Canada"
__github__ = "https://github.com/landowark/submissions"
project_path = Path(__file__).parents[2].absolute()

5
src/submissions/backend/db/models/kits.py
View File

@@ -672,8 +672,8 @@ class SubmissionType(BaseClass):
Returns:
dict: Map of locations
"""
info = self.info_map
# logger.debug(f"Info map: {info}")
info = {k:v for k,v in self.info_map.items() if k != "custom"}
logger.debug(f"Info map: {info}")
output = {}
match mode:
case "read":
@@ -681,6 +681,7 @@ class SubmissionType(BaseClass):
case "write":
output = {k: v[mode] + v['read'] for k, v in info.items() if v[mode] or v['read']}
output = {k: v for k, v in output.items() if all([isinstance(item, dict) for item in v])}
output['custom'] = self.info_map['custom']
return output
def construct_sample_map(self) -> dict:

47
src/submissions/backend/db/models/submissions.py
View File

@@ -685,7 +685,7 @@ class BasicSubmission(BaseClass):
# Child class custom functions
@classmethod
def custom_info_parser(cls, input_dict: dict, xl: Workbook | None = None) -> dict:
def custom_info_parser(cls, input_dict: dict, xl: Workbook | None = None, custom_fields:dict={}) -> dict:
"""
Update submission dictionary with type specific information
@@ -731,7 +731,7 @@ class BasicSubmission(BaseClass):
return input_dict
@classmethod
def custom_info_writer(cls, input_excel: Workbook, info: dict | None = None, backup: bool = False) -> Workbook:
def custom_info_writer(cls, input_excel: Workbook, info: dict | None = None, backup: bool = False, custom_fields:dict={}) -> Workbook:
"""
Adds custom autofill methods for submission
@@ -1307,6 +1307,7 @@ class BacterialCulture(BasicSubmission):
row = idx.index.to_list()[0]
return row + 1
class Wastewater(BasicSubmission):
"""
derivative submission type from BasicSubmission
@@ -1345,7 +1346,7 @@ class Wastewater(BasicSubmission):
return output
@classmethod
def custom_info_parser(cls, input_dict: dict, xl: Workbook | None = None) -> dict:
def custom_info_parser(cls, input_dict: dict, xl: Workbook | None = None, custom_fields:dict={}) -> dict:
"""
Update submission dictionary with type specific information. Extends parent
@@ -1550,7 +1551,6 @@ class Wastewater(BasicSubmission):
return input_dict
class WastewaterArtic(BasicSubmission):
"""
derivative submission type for artic wastewater
@@ -1597,7 +1597,7 @@ class WastewaterArtic(BasicSubmission):
return output
@classmethod
def custom_info_parser(cls, input_dict: dict, xl: Workbook | None = None) -> dict:
def custom_info_parser(cls, input_dict: dict, xl: Workbook | None = None, custom_fields:dict={}) -> dict:
"""
Update submission dictionary with type specific information
@@ -1608,16 +1608,20 @@ class WastewaterArtic(BasicSubmission):
Returns:
dict: Updated sample dictionary
"""
# TODO: Clean up and move range start/stops to db somehow.
input_dict = super().custom_info_parser(input_dict)
ws = xl['Egel results']
data = [ws.cell(row=ii, column=jj) for jj in range(15, 27) for ii in range(10, 18)]
egel_section = custom_fields['egel_results']
ws = xl[egel_section['sheet']]
data = [ws.cell(row=ii, column=jj) for jj in range(egel_section['start_column'], egel_section['end_column']) for ii in range(egel_section['start_row'], egel_section['end_row'])]
data = [cell for cell in data if cell.value is not None and "NTC" in cell.value]
input_dict['gel_controls'] = [
dict(sample_id=cell.value, location=f"{row_map[cell.row - 9]}{str(cell.column - 14).zfill(2)}") for cell in
data]
ws = xl['First Strand List']
data = [dict(plate=ws.cell(row=ii, column=3).value, starting_sample=ws.cell(row=ii, column=4).value) for ii in
range(8, 11)]
# NOTE: Get source plate information
source_plates_section = custom_fields['source_plates']
ws = xl[source_plates_section['sheet']]
data = [dict(plate=ws.cell(row=ii, column=source_plates_section['plate_column']).value, starting_sample=ws.cell(row=ii, column=source_plates_section['starting_sample_column']).value) for ii in
range(source_plates_section['start_row'], source_plates_section['end_row'])]
for datum in data:
if datum['plate'] in ["None", None, ""]:
continue
@@ -1808,7 +1812,7 @@ class WastewaterArtic(BasicSubmission):
return input_dict
@classmethod
def custom_info_writer(cls, input_excel: Workbook, info: dict | None = None, backup: bool = False) -> Workbook:
def custom_info_writer(cls, input_excel: Workbook, info: dict | None = None, backup: bool = False, custom_fields:dict={}) -> Workbook:
"""
Adds custom autofill methods for submission. Extends Parent
@@ -1822,30 +1826,33 @@ class WastewaterArtic(BasicSubmission):
"""
input_excel = super().custom_info_writer(input_excel, info, backup)
# logger.debug(f"Info:\n{pformat(info)}")
# logger.debug(f"Custom fields:\n{pformat(custom_fields)}")
# NOTE: check for source plate information
source_plates_section = custom_fields['source_plates']
if check_key_or_attr(key='source_plates', interest=info, check_none=True):
worksheet = input_excel['First Strand List']
start_row = 8
worksheet = input_excel[source_plates_section['sheet']]
start_row = source_plates_section['start_row']
# NOTE: write source plates to First strand list
for iii, plate in enumerate(info['source_plates']['value']):
# logger.debug(f"Plate: {plate}")
row = start_row + iii
try:
worksheet.cell(row=row, column=3, value=plate['plate'])
worksheet.cell(row=row, column=source_plates_section['plate_column'], value=plate['plate'])
except TypeError:
pass
try:
worksheet.cell(row=row, column=4, value=plate['starting_sample'])
worksheet.cell(row=row, column=source_plates_section['starting_sample_column'], value=plate['starting_sample'])
except TypeError:
pass
# NOTE: check for gel information
egel_section = custom_fields['egel_results']
if check_key_or_attr(key='gel_info', interest=info, check_none=True):
# logger.debug(f"Gel info check passed.")
# NOTE: print json field gel results to Egel results
worksheet = input_excel['Egel results']
worksheet = input_excel[egel_section['sheet']]
# TODO: Move all this into a seperate function?
start_row = 21
start_column = 15
start_row = egel_section['start_row']
start_column = egel_section['start_column']
for row, ki in enumerate(info['gel_info']['value'], start=1):
# logger.debug(f"ki: {ki}")
# logger.debug(f"vi: {vi}")
@@ -1862,14 +1869,14 @@ class WastewaterArtic(BasicSubmission):
except AttributeError:
logger.error(f"Failed {kj['name']} with value {kj['value']} to row {row}, column {column}")
if check_key_or_attr(key='gel_image_path', interest=info, check_none=True):
worksheet = input_excel['Egel results']
worksheet = input_excel[egel_section['sheet']]
# logger.debug(f"We got an image: {info['gel_image']}")
with ZipFile(cls.__directory_path__.joinpath("submission_imgs.zip")) as zipped:
z = zipped.extract(info['gel_image_path']['value'], Path(TemporaryDirectory().name))
img = OpenpyxlImage(z)
img.height = 400 # insert image height in pixels as float or int (e.g. 305.5)
img.width = 600
img.anchor = 'B9'
img.anchor = egel_section['img_anchor']
worksheet.add_image(img)
return input_excel

6
src/submissions/backend/excel/parser.py
View File

@@ -212,12 +212,14 @@ class InfoParser(object):
"""
dicto = {}
# NOTE: This loop parses generic info
# logger.debug(f"Map: {self.map}")
logger.debug(f"Map: {self.map}")
for sheet in self.xl.sheetnames:
ws = self.xl[sheet]
relevant = []
for k, v in self.map.items():
# NOTE: If the value is hardcoded put it in the dictionary directly.
if k == "custom":
continue
if isinstance(v, str):
dicto[k] = dict(value=v, missing=False)
continue
@@ -265,7 +267,7 @@ class InfoParser(object):
except (KeyError, IndexError):
continue
# Return after running the parser components held in submission object.
return self.sub_object.custom_info_parser(input_dict=dicto, xl=self.xl)
return self.sub_object.custom_info_parser(input_dict=dicto, xl=self.xl, custom_fields=self.map['custom'])
class ReagentParser(object):

10
src/submissions/backend/excel/writer.py
View File

@@ -8,8 +8,6 @@ from pathlib import Path
# from pathlib import Path
from pprint import pformat
from typing import List
from collections import OrderedDict
from jinja2 import TemplateNotFound
from openpyxl import load_workbook, Workbook
from backend.db.models import SubmissionType, KitType, BasicSubmission
from backend.validators.pydant import PydSubmission
@@ -135,8 +133,8 @@ class InfoWriter(object):
self.submission_type = submission_type
self.sub_object = sub_object
self.xl = xl
info_map = submission_type.construct_info_map(mode='write')
self.info = self.reconcile_map(info_dict, info_map)
self.info_map = submission_type.construct_info_map(mode='write')
self.info = self.reconcile_map(info_dict, self.info_map)
# logger.debug(pformat(self.info))
def reconcile_map(self, info_dict: dict, info_map: dict) -> dict:
@@ -154,6 +152,8 @@ class InfoWriter(object):
for k, v in info_dict.items():
if v is None:
continue
if k == "custom":
continue
dicto = {}
try:
dicto['locations'] = info_map[k]
@@ -187,7 +187,7 @@ class InfoWriter(object):
logger.debug(f"Writing {k} to {loc['sheet']}, row: {loc['row']}, column: {loc['column']}")
sheet = self.xl[loc['sheet']]
sheet.cell(row=loc['row'], column=loc['column'], value=v['value'])
return self.sub_object.custom_info_writer(self.xl, info=self.info)
return self.sub_object.custom_info_writer(self.xl, info=self.info, custom_fields=self.info_map['custom'])
class ReagentWriter(object):

50
src/submissions/frontend/widgets/app.py
View File

@@ -8,9 +8,12 @@ from PyQt6.QtWidgets import (
)
from PyQt6.QtGui import QAction
from pathlib import Path
from tools import check_if_app, Settings, Report
from markdown import markdown
from tools import check_if_app, Settings, Report, jinja_template_loading
from datetime import date
from .pop_ups import AlertPop
from .pop_ups import AlertPop, HTMLPop
from .misc import LogParser
import logging, webbrowser, sys, shutil
from .submission_table import SubmissionsSheet
@@ -31,22 +34,22 @@ class App(QMainWindow):
self.ctx = ctx
self.last_dir = ctx.directory_path
self.report = Report()
# indicate version and connected database in title bar
# NOTE: indicate version and connected database in title bar
try:
self.title = f"Submissions App (v{ctx.package.__version__}) - {ctx.database_path}"
except (AttributeError, KeyError):
self.title = f"Submissions App"
# set initial app position and size
# NOTE: set initial app position and size
self.left = 0
self.top = 0
self.width = 1300
self.height = 1000
self.setWindowTitle(self.title)
self.setGeometry(self.left, self.top, self.width, self.height)
# insert tabs into main app
# NOTE: insert tabs into main app
self.table_widget = AddSubForm(self)
self.setCentralWidget(self.table_widget)
# run initial setups
# NOTE: run initial setups
self._createActions()
self._createMenuBar()
self._createToolBar()
@@ -126,8 +129,11 @@ class App(QMainWindow):
"""
Show the 'about' message
"""
output = f"Version: {self.ctx.package.__version__}\n\nAuthor: {self.ctx.package.__author__['name']} - {self.ctx.package.__author__['email']}\n\nCopyright: {self.ctx.package.__copyright__}"
about = AlertPop(message=output, status="Information")
j_env = jinja_template_loading()
template = j_env.get_template("project.html")
html = template.render(info=self.ctx.package.__dict__)
# logger.debug(html)
about = HTMLPop(html=html, title="About")
about.exec()
def openDocs(self):
@@ -143,11 +149,21 @@ class App(QMainWindow):
def openGithub(self):
"""
Opens the instructions html page
Opens the github page
"""
url = "https://github.com/landowark/submissions"
webbrowser.get('windows-default').open(url)
def openInstructions(self):
if check_if_app():
url = Path(sys._MEIPASS).joinpath("files", "README.md")
else:
url = Path("README.md")
with open(url, "r", encoding="utf-8") as f:
html = markdown(f.read())
instr = HTMLPop(html=html, title="Instructions")
instr.exec()
def result_reporter(self):
"""
@@ -199,35 +215,35 @@ class AddSubForm(QWidget):
# logger.debug(f"Initializating subform...")
super(QWidget, self).__init__(parent)
self.layout = QVBoxLayout(self)
# Initialize tab screen
# NOTE: Initialize tab screen
self.tabs = QTabWidget()
self.tab1 = QWidget()
self.tab2 = QWidget()
self.tab3 = QWidget()
self.tab4 = QWidget()
self.tabs.resize(300,200)
# Add tabs
# NOTE: Add tabs
self.tabs.addTab(self.tab1,"Submissions")
self.tabs.addTab(self.tab2,"Controls")
self.tabs.addTab(self.tab3, "Add SubmissionType")
self.tabs.addTab(self.tab4, "Add Kit")
# Create submission adder form
# NOTE: Create submission adder form
self.formwidget = SubmissionFormContainer(self)
self.formlayout = QVBoxLayout(self)
self.formwidget.setLayout(self.formlayout)
self.formwidget.setFixedWidth(300)
# Make scrollable interior for form
# NOTE: Make scrollable interior for form
self.interior = QScrollArea(self.tab1)
self.interior.setWidgetResizable(True)
self.interior.setFixedWidth(325)
self.interior.setWidget(self.formwidget)
# Create sheet to hold existing submissions
# NOTE: Create sheet to hold existing submissions
self.sheetwidget = QWidget(self)
self.sheetlayout = QVBoxLayout(self)
self.sheetwidget.setLayout(self.sheetlayout)
self.sub_wid = SubmissionsSheet(parent=parent)
self.sheetlayout.addWidget(self.sub_wid)
# Create layout of first tab to hold form and sheet
# NOTE: Create layout of first tab to hold form and sheet
self.tab1.layout = QHBoxLayout(self)
self.tab1.setLayout(self.tab1.layout)
self.tab1.layout.addWidget(self.interior)
@@ -236,7 +252,7 @@ class AddSubForm(QWidget):
self.controls_viewer = ControlsViewer(self)
self.tab2.layout.addWidget(self.controls_viewer)
self.tab2.setLayout(self.tab2.layout)
# create custom widget to add new tabs
# NOTE: create custom widget to add new tabs
ST_adder = SubmissionTypeAdder(self)
self.tab3.layout = QVBoxLayout(self)
self.tab3.layout.addWidget(ST_adder)
@@ -245,6 +261,6 @@ class AddSubForm(QWidget):
self.tab4.layout = QVBoxLayout(self)
self.tab4.layout.addWidget(kit_adder)
self.tab4.setLayout(self.tab4.layout)
# add tabs to main widget
# NOTE: add tabs to main widget
self.layout.addWidget(self.tabs)
self.setLayout(self.layout)

13
src/submissions/frontend/widgets/gel_checker.py
View File

@@ -3,7 +3,7 @@ Gel box for artic quality control
"""
from PyQt6.QtWidgets import (QWidget, QDialog, QGridLayout,
QLabel, QLineEdit, QDialogButtonBox,
QTextEdit
QTextEdit, QComboBox
)
import pyqtgraph as pg
from PyQt6.QtGui import QIcon
@@ -119,8 +119,11 @@ class ControlsForm(QWidget):
self.layout.addWidget(label, iii, 1, 1, 1)
for iii in range(3):
for jjj in range(3):
widge = QLineEdit()
widge.setText("Neg")
# widge = QLineEdit()
widge = QComboBox()
widge.addItems(['Neg', 'Pos'])
widge.setCurrentIndex(0)
widge.setEditable(True)
widge.setObjectName(f"{rows[iii]} : {columns[jjj]}")
self.layout.addWidget(widge, iii+1, jjj+2, 1, 1)
self.layout.addWidget(QLabel("Comments:"), 0,5,1,1)
@@ -137,13 +140,13 @@ class ControlsForm(QWidget):
List[dict]: output of values
"""
output = []
for le in self.findChildren(QLineEdit):
for le in self.findChildren(QComboBox):
label = [item.strip() for item in le.objectName().split(" : ")]
try:
dicto = [item for item in output if item['name']==label[0]][0]
except IndexError:
dicto = dict(name=label[0], values=[])
dicto['values'].append(dict(name=label[1], value=le.text()))
dicto['values'].append(dict(name=label[1], value=le.currentText()))
if label[0] not in [item['name'] for item in output]:
output.append(dicto)
# logger.debug(pformat(output))

21
src/submissions/frontend/widgets/pop_ups.py
View File

@@ -5,6 +5,8 @@ from PyQt6.QtWidgets import (
QLabel, QVBoxLayout, QDialog,
QDialogButtonBox, QMessageBox, QComboBox
)
from PyQt6.QtWebEngineWidgets import QWebEngineView
from PyQt6.QtCore import Qt
from tools import jinja_template_loading
import logging
from backend.db import models
@@ -19,7 +21,7 @@ class QuestionAsker(QDialog):
"""
dialog to ask yes/no questions
"""
def __init__(self, title:str, message:str) -> QDialog:
def __init__(self, title:str, message:str):
super().__init__()
self.setWindowTitle(title)
# NOTE: set yes/no buttons
@@ -39,7 +41,7 @@ class AlertPop(QMessageBox):
"""
Dialog to show an alert.
"""
def __init__(self, message:str, status:Literal['Information', 'Question', 'Warning', 'Critical'], owner:str|None=None) -> QMessageBox:
def __init__(self, message:str, status:Literal['Information', 'Question', 'Warning', 'Critical'], owner:str|None=None):
super().__init__()
# NOTE: select icon by string
icon = getattr(QMessageBox.Icon, status)
@@ -47,12 +49,25 @@ class AlertPop(QMessageBox):
self.setInformativeText(message)
self.setWindowTitle(f"{owner} - {status.title()}")
class HTMLPop(QDialog):
def __init__(self, html:str, owner:str|None=None, title:str="python"):
super().__init__()
self.webview = QWebEngineView(parent=self)
self.layout = QVBoxLayout()
self.setWindowTitle(title)
self.webview.setHtml(html)
self.webview.setMinimumSize(600, 500)
self.webview.setMaximumSize(600, 500)
self.layout.addWidget(self.webview)
class ObjectSelector(QDialog):
"""
dialog to input BaseClass type manually
"""
def __init__(self, title:str, message:str, obj_type:str|models.BaseClass) -> QDialog:
def __init__(self, title:str, message:str, obj_type:str|models.BaseClass):
super().__init__()
self.setWindowTitle(title)
self.widget = QComboBox()

14
src/submissions/templates/project.html Normal file
View File

@@ -0,0 +1,14 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<script src="qrc:///qtwebchannel/qwebchannel.js"></script>
<title>{{ title }}</title>
</head>
<body>
<p>Version: {{ info['__version__'] }}</p>
<p>Author: {{ info['__author__']['name'] }} - <a href="mailto:{{ info['__author__']['email' ] }}">{{ info['__author__']['email'] }}</a></p>
<p>Copyright: {{ info['__copyright__'] }}</p>
<p>Github: <a id="link" href="{{ info['__github__'] }}">{{ info['__github__'] }}</a></p>
</body>
</html>