From 0c81c74f700985636d62fde0684baecf7d2bea81 Mon Sep 17 00:00:00 2001 From: Landon Wark Date: Wed, 5 Jul 2023 13:15:42 -0500 Subject: [PATCH] Commit before bringing in pydantic --- CHANGELOG.md | 4 +++ TODO.md | 3 +- requirements.txt | Bin 2262 -> 3872 bytes src/submissions/__init__.py | 3 +- src/submissions/backend/excel/parser.py | 26 ++++++++++++------ src/submissions/backend/excel/reports.py | 1 - .../frontend/main_window_functions.py | 7 ++--- 7 files changed, 28 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a3336f..6d1e64c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 202307.01 + +- Moved parser to metadata based recognition of submission type. + ## 202306.03 - Improve WW plate mapping by using layout in submission forms rather than PCR. diff --git a/TODO.md b/TODO.md index d563989..ca56500 100644 --- a/TODO.md +++ b/TODO.md @@ -1,4 +1,5 @@ -- [ ] Move submission types from config.yml into database. +- [ ] Migrate the parser.sub dictionary to pydantic models. +- [x] Move type_decider to metadata based method rather than excel map. - [x] Solve bug for plate mapping when two samples of same name are in different rows. - Try importing "L:\Robotics Laboratory Support\Submissions\Wastewater\2023\2023-06-21\RSL-WW-20230621-1.xlsx" for example. - [x] Improve plate mapping by using layout in submission forms rather than PCR. diff --git a/requirements.txt b/requirements.txt index e017284da00777f44b0248c7a908928d41d2b28d..8c6ed38cdf78e229ddfca90dab2bef34bfa5a118 100644 GIT binary patch literal 3872 zcma)q{HmX$@}|7iXHbsI&*djc7n0pbZ(hcsF_83K zXrz2wW_mp;t8%@!F$gPZqb+S&%F8G`O-iHRJLym5JH4LD=27PpYxda4GFsPRKT#fs zZq?t&{y_NFVRb8CSOqCo2c6ZeyskTs!!neFkA40g9;b?ymwZ3y{w2!Ga;NwgWfD?2 z8U#~*I}Y}?(zw#c?~nAlT6r4?#*(K7*akgx#L>Y%g7G8Xn_7ipR*2A<*a6L7n;YBXY@19jo1SLH8B>>;oc%OW37p>ZIs)c z3H`#TqI(ztiBY#WT0TK4$ zuqltSun5M%OYCMnkzNIYJ6W@|BCr+snChpEc%7B>X^hwMgO8zpPvtLl$iA%+jhRl} zQ|ZrDQ56q}=i;R8G6YX25yw0@n924=8e}m9ub(^S_ibP}g?@`t`}%NN;q z#_#|aj&sp5g>CH8-IjdPi?6brUa@LqNY;|wiTQK=ebr$@!z>fkjr@|8J9#(u#C00+ zwUXRKtyB*+vZW-jPc1!)HFUD`=|4P>_1F5o*4j$w#zHz0Gq8~S9BQ=UpGlZF=st4* z26)XHBD3yobdEb5pF@LQdk7nFNF+wN(rf15LXqIXy$w%QekWwJPHst+?i2xWFuigH z!sM{40T=>*D<3(hlWAAUbB!h}F%iv)Bme%Ji>R{fmefs_jDSf^gl7?3a`;*JX38|# z+;%MUesI9iu6&3Xvf|u9b+Qlxd!UV)Y=j0bw%?czyml8R$GeD?dSq&nPtOsuap(n3 z4)vOtWlMS9XtKE5*e2P@wJL~-$^`97S-A>6sJ4E^AVF%}O3^S`$px{JCD%+p(9&Uwz>IMxSVchi$GJMi;z8sNL7rf?L;yqfo z;>|g$Q)SrWj=Fjb>%_u-%(&?d%x|ody=Jkd!vE z;f+PE9)j6=)unZ6f@$FGboidQeYsu-rn-wOp8ksqd8F1_@wpHq$!Pj`iScl%G{U>m z4|58S3;hk_nSW=c@A3J2Jl?>OTt-ug&AE4WMM`r8(K`DQ10FSGZ#xP7(ZgB+L-_qXw0DAQb@Odz_8$>H7u6Fr=7 zo>YfS7yn0dv5Zt;k{m|Xn@$Bxw46Msh_;~ zv288eSxaKZj`#T=cGJbx{vaQh;m0$|(L2)YEBkxg=WwXL$t{cy48DsT?G)vz+s{at HoXP)x1M@xh delta 119 zcmZ1=cTJG#|G$YEZWHH)O!i^yQ8i}JV=w?>Lk3<3E{0@=9EMCFnayAegce|NJqF{+ z4;d>b-(xD6T*a(2`5kk`I3hL|ajGy*w&2~u QY63EM@<(3h$whpV00D&~&j0`b diff --git a/src/submissions/__init__.py b/src/submissions/__init__.py index 67138a0..800c631 100644 --- a/src/submissions/__init__.py +++ b/src/submissions/__init__.py @@ -4,7 +4,7 @@ from pathlib import Path # Version of the realpython-reader package __project__ = "submissions" -__version__ = "202306.3b" +__version__ = "202307.1b" __author__ = {"name":"Landon Wark", "email":"Landon.Wark@phac-aspc.gc.ca"} __copyright__ = "2022-2023, Government of Canada" @@ -25,6 +25,7 @@ class bcolors: # set out the workflow I've imagined for creating new submission types. # First of all, you will need to write new parsing methods in backend.excel.parser to pull information out of the submission form # for the submission itself as well as for any samples you can pull out of that same workbook. +# The workbooks no longer need a sheet map, but they do need their submission type put in the categories metadata of the client excel template. # Second, you will have to update the model in backend.db.models.submissions and provide a new polymorph to the BasicSubmission object. # The BSO should hold the majority of the general info. # You can also update any of the parsers to pull out any custom info you need, like enforcing RSL plate numbers, scraping PCR results, etc. diff --git a/src/submissions/backend/excel/parser.py b/src/submissions/backend/excel/parser.py index f0cce13..4ff1f17 100644 --- a/src/submissions/backend/excel/parser.py +++ b/src/submissions/backend/excel/parser.py @@ -2,7 +2,6 @@ contains parser object for pulling values from client generated submission sheets. ''' from getpass import getuser -import math import pprint from typing import Tuple import pandas as pd @@ -44,6 +43,7 @@ class SheetParser(object): except ValueError as e: logger.error(f"Incorrect value: {e}") self.xl = None + # TODO: replace OrderedDict with pydantic BaseModel self.sub = OrderedDict() # make decision about type of sample we have self.sub['submission_type'] = self.type_decider() @@ -58,14 +58,22 @@ class SheetParser(object): Returns: str: submission type name """ - try: - for type in self.ctx['submission_types']: - if self.xl.sheet_names == self.ctx['submission_types'][type]['excel_map']: - return type.title() - return "Unknown" - except Exception as e: - logger.warning(f"We were unable to parse the submission type due to: {e}") - return "Unknown" + # Check metadata for category, return first category + if self.xl.book.properties.category != None: + categories = [item.strip().title() for item in self.xl.book.properties.category.split(";")] + return categories[0].replace(" ", "_") + else: + # This code is going to be depreciated once there is full adoption of the client sheets + # with updated metadata + try: + for type in self.ctx['submission_types']: + # This gets the *first* submission type that matches the sheet names in the workbook + if self.xl.sheet_names == self.ctx['submission_types'][type]['excel_map']: + return type.title() + return "Unknown" + except Exception as e: + logger.warning(f"We were unable to parse the submission type due to: {e}") + return "Unknown" def parse_unknown(self) -> None: diff --git a/src/submissions/backend/excel/reports.py b/src/submissions/backend/excel/reports.py index 32a30c4..e5d54c1 100644 --- a/src/submissions/backend/excel/reports.py +++ b/src/submissions/backend/excel/reports.py @@ -9,7 +9,6 @@ import sys from pathlib import Path import re from tools import check_if_app -import asyncio logger = logging.getLogger(f"submissions.{__name__}") diff --git a/src/submissions/frontend/main_window_functions.py b/src/submissions/frontend/main_window_functions.py index abf09ee..29bd489 100644 --- a/src/submissions/frontend/main_window_functions.py +++ b/src/submissions/frontend/main_window_functions.py @@ -18,7 +18,7 @@ from backend.db.models import * import logging from PyQt6.QtWidgets import ( QMainWindow, QLabel, QWidget, QPushButton, QFileDialog, - QLineEdit, QMessageBox, QComboBox, QDateEdit + QLineEdit, QComboBox, QDateEdit ) from .all_window_functions import extract_form_info, select_open_file, select_save_file from PyQt6.QtCore import QSignalBlocker @@ -26,16 +26,15 @@ from backend.db.functions import ( lookup_all_orgs, lookup_kittype_by_use, lookup_kittype_by_name, construct_submission_info, lookup_reagent, store_submission, lookup_submissions_by_date_range, create_kit_from_yaml, create_org_from_yaml, get_control_subtypes, get_all_controls_by_type, - lookup_all_submissions_by_type, get_all_controls, lookup_submission_by_rsl_num, update_ww_sample, hitpick_plate + lookup_all_submissions_by_type, get_all_controls, lookup_submission_by_rsl_num, update_ww_sample ) from backend.excel.parser import SheetParser, PCRParser from backend.excel.reports import make_report_html, make_report_xlsx, convert_data_list_to_df from tools import RSLNamer, check_not_nan, check_kit_integrity from .custom_widgets.pop_ups import AlertPop, QuestionAsker -from .custom_widgets import ReportDatePicker, ReagentTypeForm +from .custom_widgets import ReportDatePicker from .custom_widgets.misc import ImportReagent from .visualizations.control_charts import create_charts, construct_html -from .visualizations import make_plate_map logger = logging.getLogger(f"submissions.{__name__}")