diff --git a/src/submissions/CHANGELOG.md b/src/submissions/CHANGELOG.md index 6827ffd..3359abb 100644 --- a/src/submissions/CHANGELOG.md +++ b/src/submissions/CHANGELOG.md @@ -1,3 +1,8 @@ +**202303.04** + +- Completed partial imports that will add in missing reagents found in the kit indicated by the user. +- Added web documentation to the help menu. + **202303.03** - Increased robustness by utilizing PyQT6 widget names to pull data from forms instead of previously used label/input zip. diff --git a/src/submissions/README.md b/src/submissions/README.md index a317b26..1eea30d 100644 --- a/src/submissions/README.md +++ b/src/submissions/README.md @@ -1 +1,42 @@ -**This is the readme file** \ No newline at end of file +## Logging in New Run: +*should fit 90% of usage cases* + + 1. Ensure a properly formatted Submission Excel form has been filled out. (It will save you a few headaches) + a. All fields should be filled in to ensure proper lookups of reagents. + 2. Open the app using the shortcut in the Submissions folder. For example: "L:\Robotics Laboratory Support\Submissions\submissions_v122b.exe - Shortcut.lnk" (Version may have changed). + a. Ignore the large black window of fast scrolling text, it is there for debugging purposes. + b. The 'Submissions' tab should be open by default. + 3. Click on 'File' in the menu bar, followed by 'Import' and use the locate the form you definitely made sure was properly filled out in step 1. + 4. Click "Ok". + 5. Most of the fields in the form should be automatically filled in from the form area to the left of the screen. + 6. You may need to maximize the app to ensure you can see all the info. + 7. Any fields that are not automatically filled in can be filled in manually from the drop down menus. + 8. Once you are certain all the information is correct, click 'Submit' at the bottom of the form. + 9. Add in any reagents the app doesn't recognize. + 10. Once the new run shows up at the bottom of the Submissions, everything is fine. + 11. In case of any mistakes, the run can be overwritten by a reimport. + +## Check existing Run: + + 1. Details of existing runs can be checked by double clicking on the row of interest in the summary sheet on the right of the 'Submissions' tab. + 2. All information available on the run should be available in the resulting text window. This information can be exported by clicking 'Export PDF' at the top. + +## Generating a report: + + 1. Click on 'Reports' -> 'Make Report' in the menu bar. + 2. Select the start date and the end date you want for the report. Click 'ok'. + 3. Use the file dialog to select a location to save the report. + a. Both an excel sheet and a pdf should be generated containing summary information for submissions made by each client lab. + +## Checking Controls: + + 1. Controls for bacterial runs are now incorporated directly into the submissions database using webview. (Admittedly this performance is not as good as with a browser, so you will have to triage your data) + 2. Click on the "Controls" tab. + 3. Range of dates for controls can be selected from the date pickers at the top. + a. If start date is set after end date, the start date will default back to 3 months before end date. + b. Recommendation is to use less than 6 month date range keeping in mind that higher data density will affect performance (with kraken being the worst so far) + 4. Analysis type and subtype can be set using the drop down menus. (Only kraken has a subtype so far). + +## Adding new Kit: + + 1. Instructions to come. diff --git a/src/submissions/__main__.py b/src/submissions/__main__.py index 8c612e2..7ece35c 100644 --- a/src/submissions/__main__.py +++ b/src/submissions/__main__.py @@ -1,7 +1,7 @@ import sys from pathlib import Path import os -# must be set to enable qtwebengine in network path +# environment variable must be set to enable qtwebengine in network path if getattr(sys, 'frozen', False): os.environ['QTWEBENGINE_DISABLE_SANDBOX'] = "1" else : diff --git a/src/submissions/backend/db/functions.py b/src/submissions/backend/db/functions.py index 7d6b105..f934883 100644 --- a/src/submissions/backend/db/functions.py +++ b/src/submissions/backend/db/functions.py @@ -641,6 +641,16 @@ def lookup_submission_by_rsl_num(ctx:dict, rsl_num:str) -> models.BasicSubmissio def lookup_submissions_using_reagent(ctx:dict, reagent:models.Reagent) -> list[models.BasicSubmission]: + """ + Retrieves each submission using a specified reagent. + + Args: + ctx (dict): settings passed down from gui + reagent (models.Reagent): reagent object in question + + Returns: + list[models.BasicSubmission]: list of all submissions using specified reagent. + """ return ctx['database_session'].query(models.BasicSubmission).join(reagents_submissions).filter(reagents_submissions.c.reagent_id==reagent.id).all() diff --git a/src/submissions/backend/db/models/submissions.py b/src/submissions/backend/db/models/submissions.py index 8a0f270..e480b58 100644 --- a/src/submissions/backend/db/models/submissions.py +++ b/src/submissions/backend/db/models/submissions.py @@ -35,7 +35,7 @@ class BasicSubmission(Base): reagents = relationship("Reagent", back_populates="submissions", secondary=reagents_submissions) #: relationship to reagents reagents_id = Column(String, ForeignKey("_reagents.id", ondelete="SET NULL", name="fk_BS_reagents_id")) #: id of used reagents extraction_info = Column(JSON) #: unstructured output from the extraction table logger. - run_cost = Column(FLOAT(2)) #: total cost of running the plate. Set from kit costs at time of creation. + run_cost = Column(FLOAT(2)) #: total cost of running the plate. Set from constant and mutable kit costs at time of creation. uploaded_by = Column(String(32)) #: user name of person who submitted the submission to the database. # Allows for subclassing into ex. BacterialCulture, Wastewater, etc. diff --git a/src/submissions/backend/excel/parser.py b/src/submissions/backend/excel/parser.py index 9d40bff..5b82520 100644 --- a/src/submissions/backend/excel/parser.py +++ b/src/submissions/backend/excel/parser.py @@ -27,6 +27,7 @@ class SheetParser(object): # set attributes based on kwargs from gui ctx for kwarg in kwargs: setattr(self, f"_{kwarg}", kwargs[kwarg]) + # self.__dict__.update(kwargs) if filepath == None: logger.error(f"No filepath given.") self.xl = None @@ -38,12 +39,12 @@ class SheetParser(object): self.xl = None self.sub = OrderedDict() # make decision about type of sample we have - self.sub['submission_type'] = self._type_decider() + self.sub['submission_type'] = self.type_decider() # select proper parser based on sample type - parse_sub = getattr(self, f"_parse_{self.sub['submission_type'].lower()}") + parse_sub = getattr(self, f"parse_{self.sub['submission_type'].lower()}") parse_sub() - def _type_decider(self) -> str: + def type_decider(self) -> str: """ makes decisions about submission type based on structure of excel file @@ -60,7 +61,7 @@ class SheetParser(object): return "Unknown" - def _parse_unknown(self) -> None: + def parse_unknown(self) -> None: """ Dummy function to handle unknown excel structures """ @@ -68,7 +69,7 @@ class SheetParser(object): self.sub = None - def _parse_generic(self, sheet_name:str) -> pd.DataFrame: + def parse_generic(self, sheet_name:str) -> pd.DataFrame: """ Pulls information common to all submission types and passes on dataframe @@ -89,12 +90,12 @@ class SheetParser(object): return submission_info - def _parse_bacterial_culture(self) -> None: + def parse_bacterial_culture(self) -> None: """ pulls info specific to bacterial culture sample type """ - def _parse_reagents(df:pd.DataFrame) -> None: + def parse_reagents(df:pd.DataFrame) -> None: """ Pulls reagents from the bacterial sub-dataframe @@ -126,7 +127,7 @@ class SheetParser(object): else: expiry = date.today() self.sub[f"lot_{reagent_type}"] = {'lot':output_var, 'exp':expiry} - submission_info = self._parse_generic("Sample List") + submission_info = self.parse_generic("Sample List") # iloc is [row][column] and the first row is set as header row so -2 tech = str(submission_info.iloc[11][1]) if tech == "nan": @@ -139,7 +140,7 @@ class SheetParser(object): # must be prefixed with 'lot_' to be recognized by gui # Todo: find a more adaptable way to read reagents. reagent_range = submission_info.iloc[1:13, 4:8] - _parse_reagents(reagent_range) + parse_reagents(reagent_range) # get individual sample info sample_parser = SampleParser(submission_info.iloc[15:111]) sample_parse = getattr(sample_parser, f"parse_{self.sub['submission_type'].lower()}_samples") @@ -147,12 +148,12 @@ class SheetParser(object): self.sub['samples'] = sample_parse() - def _parse_wastewater(self) -> None: + def parse_wastewater(self) -> None: """ pulls info specific to wastewater sample type """ - def _parse_reagents(df:pd.DataFrame) -> None: + def parse_reagents(df:pd.DataFrame) -> None: """ Pulls reagents from the bacterial sub-dataframe @@ -180,7 +181,7 @@ class SheetParser(object): expiry = date.today() self.sub[f"lot_{output_key}"] = {'lot':output_var, 'exp':expiry} # parse submission sheet - submission_info = self._parse_generic("WW Submissions (ENTER HERE)") + submission_info = self.parse_generic("WW Submissions (ENTER HERE)") # parse enrichment sheet enrichment_info = self.xl.parse("Enrichment Worksheet", dtype=object) # set enrichment reagent range @@ -195,9 +196,9 @@ class SheetParser(object): pcr_reagent_range = qprc_info.iloc[0:5, 9:20] # compile technician info self.sub['technician'] = f"Enr: {enrichment_info.columns[2]}, Ext: {extraction_info.columns[2]}, PCR: {qprc_info.columns[2]}" - _parse_reagents(enr_reagent_range) - _parse_reagents(ext_reagent_range) - _parse_reagents(pcr_reagent_range) + parse_reagents(enr_reagent_range) + parse_reagents(ext_reagent_range) + parse_reagents(pcr_reagent_range) # parse samples sample_parser = SampleParser(submission_info.iloc[16:40]) sample_parse = getattr(sample_parser, f"parse_{self.sub['submission_type'].lower()}_samples") diff --git a/src/submissions/backend/excel/reports.py b/src/submissions/backend/excel/reports.py index 11c169a..3de176e 100644 --- a/src/submissions/backend/excel/reports.py +++ b/src/submissions/backend/excel/reports.py @@ -8,11 +8,13 @@ from datetime import date, timedelta import sys from pathlib import Path import re +from tools import check_if_app logger = logging.getLogger(f"submissions.{__name__}") # set path of templates depending on pyinstaller/raw python -if getattr(sys, 'frozen', False): +# if getattr(sys, 'frozen', False): +if check_if_app(): loader_path = Path(sys._MEIPASS).joinpath("files", "templates") else: loader_path = Path(__file__).parents[2].joinpath('templates').absolute().__str__() diff --git a/src/submissions/configure/__init__.py b/src/submissions/configure/__init__.py index 35d460e..366a9eb 100644 --- a/src/submissions/configure/__init__.py +++ b/src/submissions/configure/__init__.py @@ -8,6 +8,7 @@ from logging import handlers from pathlib import Path from sqlalchemy.orm import Session from sqlalchemy import create_engine +from tools import check_if_app logger = logging.getLogger(f"submissions.{__name__}") @@ -102,7 +103,8 @@ def get_config(settings_path: Path|str|None=None) -> dict: settings_path = Path.home().joinpath(".submissions", "config.yml") # finally look in the local config else: - if getattr(sys, 'frozen', False): + # if getattr(sys, 'frozen', False): + if check_if_app(): settings_path = Path(sys._MEIPASS).joinpath("files", "config.yml") else: settings_path = package_dir.joinpath('config.yml') @@ -173,7 +175,7 @@ def setup_logger(verbosity:int=3): Set logger levels using settings. Args: - verbose (bool, optional): _description_. Defaults to False. + verbosit (int, optional): Level of verbosity desired 3 is highest. Defaults to 3. Returns: logger: logger object diff --git a/src/submissions/frontend/__init__.py b/src/submissions/frontend/__init__.py index 439fee1..15fec0e 100644 --- a/src/submissions/frontend/__init__.py +++ b/src/submissions/frontend/__init__.py @@ -3,6 +3,7 @@ Operations for all user interactions. ''' import json import re +import sys from PyQt6.QtWidgets import ( QMainWindow, QLabel, QToolBar, QTabWidget, QWidget, QVBoxLayout, @@ -32,7 +33,7 @@ from backend.db import (construct_submission_info, lookup_reagent, ) from backend.db import lookup_kittype_by_name from .functions import extract_form_info -from tools import check_not_nan, check_kit_integrity +from tools import check_not_nan, check_kit_integrity, check_if_app # from backend.excel.reports import from frontend.custom_widgets import SubmissionsSheet, AlertPop, QuestionAsker, AddReagentForm, ReportDatePicker, KitAdder, ControlsDatePicker, ImportReagent import logging @@ -40,6 +41,7 @@ import difflib from getpass import getuser from datetime import date from frontend.visualizations import create_charts +import webbrowser logger = logging.getLogger(f'submissions.{__name__}') logger.info("Hello, I am a logger") @@ -49,6 +51,7 @@ class App(QMainWindow): def __init__(self, ctx: dict = {}): super().__init__() self.ctx = ctx + # indicate version and database connected in title bar try: self.title = f"Submissions App (v{ctx['package'].__version__}) - {ctx['database']}" except AttributeError: @@ -84,6 +87,7 @@ class App(QMainWindow): maintenanceMenu = menuBar.addMenu("&Monthly") helpMenu = menuBar.addMenu("&Help") helpMenu.addAction(self.helpAction) + helpMenu.addAction(self.docsAction) fileMenu.addAction(self.importAction) reportMenu.addAction(self.generateReportAction) maintenanceMenu.addAction(self.joinControlsAction) @@ -111,6 +115,7 @@ class App(QMainWindow): self.joinControlsAction = QAction("Link Controls") self.joinExtractionAction = QAction("Link Ext Logs") self.helpAction = QAction("&About", self) + self.docsAction = QAction("&Docs", self) def _connectActions(self): @@ -129,12 +134,20 @@ class App(QMainWindow): self.joinControlsAction.triggered.connect(self.linkControls) self.joinExtractionAction.triggered.connect(self.linkExtractions) self.helpAction.triggered.connect(self.showAbout) + self.docsAction.triggered.connect(self.openDocs) def showAbout(self): output = f"Version: {self.ctx['package'].__version__}\n\nAuthor: {self.ctx['package'].__author__['name']} - {self.ctx['package'].__author__['email']}\n\nCopyright: {self.ctx['package'].__copyright__}" about = AlertPop(message=output, status="information") about.exec() + def openDocs(self): + if check_if_app(): + url = Path(sys._MEIPASS).joinpath("files", "docs", "index.html") + else: + url = Path("docs\\build\\index.html").absolute() + logger.debug(f"Attempting to open {url}") + webbrowser.get('windows-default').open(f"file://{url.__str__()}") def importSubmission(self): @@ -365,24 +378,19 @@ class App(QMainWindow): # Custom two date picker for start & end dates dlg = ReportDatePicker() if dlg.exec(): - # labels, values = extract_form_info(dlg) - # info = {item[0]:item[1] for item in zip(labels, values)} info = extract_form_info(dlg) logger.debug(f"Report info: {info}") # find submissions based on date range subs = lookup_submissions_by_date_range(ctx=self.ctx, start_date=info['start_date'], end_date=info['end_date']) # convert each object to dict records = [item.report_dict() for item in subs] + # make dataframe from record dictionaries df = make_report_xlsx(records=records) html = make_report_html(df=df, start_date=info['start_date'], end_date=info['end_date']) - # make dataframe from record dictionaries - # df = make_report_xlsx(records=records) # setup filedialog to handle save location of report home_dir = Path(self.ctx["directory_path"]).joinpath(f"Submissions_Report_{info['start_date']}-{info['end_date']}.pdf").resolve().__str__() - # fname = Path(QFileDialog.getSaveFileName(self, "Save File", home_dir, filter=".xlsx")[0]) fname = Path(QFileDialog.getSaveFileName(self, "Save File", home_dir, filter=".pdf")[0]) # logger.debug(f"report output name: {fname}") - # df.to_excel(fname, engine='openpyxl') with open(fname, "w+b") as f: pisa.CreatePDF(html, dest=f) writer = pd.ExcelWriter(fname.with_suffix(".xlsx"), engine='openpyxl') @@ -398,14 +406,7 @@ class App(QMainWindow): worksheet.column_dimensions[get_column_letter(idx)].width = max_len except ValueError: pass - # set_column(idx, idx, max_len) # set column width - # colu = worksheet.column_dimensions["C"] - # style = NamedStyle(name="custom_currency", number_format='Currency') for cell in worksheet['D']: - # try: - # check = int(cell.row) - # except TypeError: - # continue if cell.row > 1: cell.style = 'Currency' writer.close() @@ -431,19 +432,11 @@ class App(QMainWindow): return # send to kit creator function result = create_kit_from_yaml(ctx=self.ctx, exp=exp) - # msg = QMessageBox() - # msg.setIcon(QMessageBox.critical) match result['code']: case 0: msg = AlertPop(message=result['message'], status='info') - # msg.setText("Kit added") - # msg.setInformativeText(result['message']) - # msg.setWindowTitle("Kit added") case 1: msg = AlertPop(message=result['message'], status='critical') - # msg.setText("Permission Error") - # msg.setInformativeText(result['message']) - # msg.setWindowTitle("Permission Error") msg.exec() @@ -467,19 +460,11 @@ class App(QMainWindow): return # send to kit creator function result = create_org_from_yaml(ctx=self.ctx, org=org) - # msg = QMessageBox() - # msg.setIcon(QMessageBox.critical) match result['code']: case 0: msg = AlertPop(message=result['message'], status='information') - # msg.setText("Organization added") - # msg.setInformativeText(result['message']) - # msg.setWindowTitle("Kit added") case 1: msg = AlertPop(message=result['message'], status='critical') - # msg.setText("Permission Error") - # msg.setInformativeText(result['message']) - # msg.setWindowTitle("Permission Error") msg.exec() @@ -537,20 +522,12 @@ class App(QMainWindow): controls = get_all_controls_by_type(ctx=self.ctx, con_type=self.con_type, start_date=self.start_date, end_date=self.end_date) # if no data found from query set fig to none for reporting in webview if controls == None: - # return fig = None else: - # data = [] - # for control in controls: - # # change each control to list of dicts - # # dicts = convert_control_by_mode(ctx=self.ctx, control=control, mode=self.mode) - # dicts = control.convert_by_mode(mode=self.mode) - # data.append(dicts) # change each control to list of dicts data = [control.convert_by_mode(mode=self.mode) for control in controls] # flatten data to one dimensional list data = [item for sublist in data for item in sublist] - # logger.debug(data) # send to dataframe creator df = convert_data_list_to_df(ctx=self.ctx, input=data, subtype=self.subtype) if self.subtype == None: @@ -567,16 +544,12 @@ class App(QMainWindow): else: html += "