diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a3336f..6d1e64c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 202307.01 + +- Moved parser to metadata based recognition of submission type. + ## 202306.03 - Improve WW plate mapping by using layout in submission forms rather than PCR. diff --git a/TODO.md b/TODO.md index d563989..ca56500 100644 --- a/TODO.md +++ b/TODO.md @@ -1,4 +1,5 @@ -- [ ] Move submission types from config.yml into database. +- [ ] Migrate the parser.sub dictionary to pydantic models. +- [x] Move type_decider to metadata based method rather than excel map. - [x] Solve bug for plate mapping when two samples of same name are in different rows. - Try importing "L:\Robotics Laboratory Support\Submissions\Wastewater\2023\2023-06-21\RSL-WW-20230621-1.xlsx" for example. - [x] Improve plate mapping by using layout in submission forms rather than PCR. diff --git a/requirements.txt b/requirements.txt index e017284..8c6ed38 100644 Binary files a/requirements.txt and b/requirements.txt differ diff --git a/src/submissions/__init__.py b/src/submissions/__init__.py index 67138a0..800c631 100644 --- a/src/submissions/__init__.py +++ b/src/submissions/__init__.py @@ -4,7 +4,7 @@ from pathlib import Path # Version of the realpython-reader package __project__ = "submissions" -__version__ = "202306.3b" +__version__ = "202307.1b" __author__ = {"name":"Landon Wark", "email":"Landon.Wark@phac-aspc.gc.ca"} __copyright__ = "2022-2023, Government of Canada" @@ -25,6 +25,7 @@ class bcolors: # set out the workflow I've imagined for creating new submission types. # First of all, you will need to write new parsing methods in backend.excel.parser to pull information out of the submission form # for the submission itself as well as for any samples you can pull out of that same workbook. +# The workbooks no longer need a sheet map, but they do need their submission type put in the categories metadata of the client excel template. # Second, you will have to update the model in backend.db.models.submissions and provide a new polymorph to the BasicSubmission object. # The BSO should hold the majority of the general info. # You can also update any of the parsers to pull out any custom info you need, like enforcing RSL plate numbers, scraping PCR results, etc. diff --git a/src/submissions/backend/excel/parser.py b/src/submissions/backend/excel/parser.py index f0cce13..4ff1f17 100644 --- a/src/submissions/backend/excel/parser.py +++ b/src/submissions/backend/excel/parser.py @@ -2,7 +2,6 @@ contains parser object for pulling values from client generated submission sheets. ''' from getpass import getuser -import math import pprint from typing import Tuple import pandas as pd @@ -44,6 +43,7 @@ class SheetParser(object): except ValueError as e: logger.error(f"Incorrect value: {e}") self.xl = None + # TODO: replace OrderedDict with pydantic BaseModel self.sub = OrderedDict() # make decision about type of sample we have self.sub['submission_type'] = self.type_decider() @@ -58,14 +58,22 @@ class SheetParser(object): Returns: str: submission type name """ - try: - for type in self.ctx['submission_types']: - if self.xl.sheet_names == self.ctx['submission_types'][type]['excel_map']: - return type.title() - return "Unknown" - except Exception as e: - logger.warning(f"We were unable to parse the submission type due to: {e}") - return "Unknown" + # Check metadata for category, return first category + if self.xl.book.properties.category != None: + categories = [item.strip().title() for item in self.xl.book.properties.category.split(";")] + return categories[0].replace(" ", "_") + else: + # This code is going to be depreciated once there is full adoption of the client sheets + # with updated metadata + try: + for type in self.ctx['submission_types']: + # This gets the *first* submission type that matches the sheet names in the workbook + if self.xl.sheet_names == self.ctx['submission_types'][type]['excel_map']: + return type.title() + return "Unknown" + except Exception as e: + logger.warning(f"We were unable to parse the submission type due to: {e}") + return "Unknown" def parse_unknown(self) -> None: diff --git a/src/submissions/backend/excel/reports.py b/src/submissions/backend/excel/reports.py index 32a30c4..e5d54c1 100644 --- a/src/submissions/backend/excel/reports.py +++ b/src/submissions/backend/excel/reports.py @@ -9,7 +9,6 @@ import sys from pathlib import Path import re from tools import check_if_app -import asyncio logger = logging.getLogger(f"submissions.{__name__}") diff --git a/src/submissions/frontend/main_window_functions.py b/src/submissions/frontend/main_window_functions.py index abf09ee..29bd489 100644 --- a/src/submissions/frontend/main_window_functions.py +++ b/src/submissions/frontend/main_window_functions.py @@ -18,7 +18,7 @@ from backend.db.models import * import logging from PyQt6.QtWidgets import ( QMainWindow, QLabel, QWidget, QPushButton, QFileDialog, - QLineEdit, QMessageBox, QComboBox, QDateEdit + QLineEdit, QComboBox, QDateEdit ) from .all_window_functions import extract_form_info, select_open_file, select_save_file from PyQt6.QtCore import QSignalBlocker @@ -26,16 +26,15 @@ from backend.db.functions import ( lookup_all_orgs, lookup_kittype_by_use, lookup_kittype_by_name, construct_submission_info, lookup_reagent, store_submission, lookup_submissions_by_date_range, create_kit_from_yaml, create_org_from_yaml, get_control_subtypes, get_all_controls_by_type, - lookup_all_submissions_by_type, get_all_controls, lookup_submission_by_rsl_num, update_ww_sample, hitpick_plate + lookup_all_submissions_by_type, get_all_controls, lookup_submission_by_rsl_num, update_ww_sample ) from backend.excel.parser import SheetParser, PCRParser from backend.excel.reports import make_report_html, make_report_xlsx, convert_data_list_to_df from tools import RSLNamer, check_not_nan, check_kit_integrity from .custom_widgets.pop_ups import AlertPop, QuestionAsker -from .custom_widgets import ReportDatePicker, ReagentTypeForm +from .custom_widgets import ReportDatePicker from .custom_widgets.misc import ImportReagent from .visualizations.control_charts import create_charts, construct_html -from .visualizations import make_plate_map logger = logging.getLogger(f"submissions.{__name__}")