From 7f0b7feb5d638419428c4344fa4d396723f223d1 Mon Sep 17 00:00:00 2001 From: lwark Date: Mon, 25 Nov 2024 13:34:02 -0600 Subject: [PATCH] Can now calculate turnaround time including holidays. --- CHANGELOG.md | 6 + src/submissions/backend/db/__init__.py | 26 +- src/submissions/backend/db/models/audit.py | 67 +++ src/submissions/backend/db/models/controls.py | 478 +++++++++--------- src/submissions/backend/db/models/kits.py | 19 +- .../backend/db/models/submissions.py | 102 +++- src/submissions/backend/excel/parser.py | 3 + src/submissions/backend/validators/pydant.py | 65 ++- src/submissions/frontend/widgets/misc.py | 2 +- .../frontend/widgets/submission_details.py | 1 + .../frontend/widgets/submission_widget.py | 22 +- .../templates/basicsubmission_details.html | 2 +- src/submissions/tools/__init__.py | 40 ++ 13 files changed, 533 insertions(+), 300 deletions(-) create mode 100644 src/submissions/backend/db/models/audit.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fa30cf6..7356506 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,12 @@ +## 202411.05 + +- Can now calculate turnaround time including holidays. + ## 202411.04 - Add reagent from scrape now limits roles to those found in kit to prevent confusion. +- Added audit logs to track changes. +- Added completed_date column to _basicsubmission to track turnaround time. ## 202411.01 diff --git a/src/submissions/backend/db/__init__.py b/src/submissions/backend/db/__init__.py index f73b413..214143b 100644 --- a/src/submissions/backend/db/__init__.py +++ b/src/submissions/backend/db/__init__.py @@ -41,9 +41,9 @@ from .models import * def update_log(mapper, connection, target): logger.debug("\n\nBefore update\n\n") state = inspect(target) - logger.debug(state) + # logger.debug(state) update = dict(user=getuser(), time=datetime.now(), object=str(state.object), changes=[]) - logger.debug(update) + # logger.debug(update) for attr in state.attrs: hist = attr.load_history() if not hist.has_changes(): @@ -51,24 +51,24 @@ def update_log(mapper, connection, target): added = [str(item) for item in hist.added] deleted = [str(item) for item in hist.deleted] change = dict(field=attr.key, added=added, deleted=deleted) - logger.debug(f"Adding: {pformat(change)}") - try: - update['changes'].append(change) - except Exception as e: - logger.error(f"Something went horribly wrong adding attr: {attr.key}: {e}") - continue - - logger.debug(f"Adding to audit logs: {pformat(update)}") + # logger.debug(f"Adding: {pformat(change)}") + if added != deleted: + try: + update['changes'].append(change) + except Exception as e: + logger.error(f"Something went wrong adding attr: {attr.key}: {e}") + continue + # logger.debug(f"Adding to audit logs: {pformat(update)}") if update['changes']: # Note: must use execute as the session will be busy at this point. # https://medium.com/@singh.surbhicse/creating-audit-table-to-log-insert-update-and-delete-changes-in-flask-sqlalchemy-f2ca53f7b02f table = AuditLog.__table__ - logger.debug(f"Adding to {table}") + # logger.debug(f"Adding to {table}") connection.execute(table.insert().values(**update)) # logger.debug("Here is where I would insert values, if I was able.") else: logger.info(f"No changes detected, not updating logs.") -# event.listen(LogMixin, 'after_update', update_log, propagate=True) -# event.listen(LogMixin, 'after_insert', update_log, propagate=True) +event.listen(LogMixin, 'after_update', update_log, propagate=True) +event.listen(LogMixin, 'after_insert', update_log, propagate=True) diff --git a/src/submissions/backend/db/models/audit.py b/src/submissions/backend/db/models/audit.py new file mode 100644 index 0000000..9f695bd --- /dev/null +++ b/src/submissions/backend/db/models/audit.py @@ -0,0 +1,67 @@ +from dateutil.parser import parse +from sqlalchemy.orm import declarative_base, DeclarativeMeta, Query +from . import BaseClass +from sqlalchemy import Column, INTEGER, String, JSON, TIMESTAMP, func +from datetime import date, datetime, timedelta +import logging + +logger = logging.getLogger(f"submissions.{__name__}") + +Base: DeclarativeMeta = declarative_base() + +class AuditLog(Base): + + __tablename__ = "_auditlog" + + id = Column(INTEGER, primary_key=True) #: primary key + user = Column(String(64)) + time = Column(TIMESTAMP) + object = Column(String(64)) + changes = Column(JSON) + + def __repr__(self): + return f"<{self.user} @ {self.time}>" + + @classmethod + def query(cls, start_date: date | str | int | None = None, end_date: date | str | int | None = None, ): + session = BaseClass.__database_session__ + query: Query = session.query(cls) + if start_date is not None and end_date is None: + logger.warning(f"Start date with no end date, using today.") + end_date = date.today() + if end_date is not None and start_date is None: + logger.warning(f"End date with no start date, using Jan 1, 2023") + start_date = session.query(cls, func.min(cls.time)).first()[1] + if start_date is not None: + # logger.debug(f"Querying with start date: {start_date} and end date: {end_date}") + match start_date: + case date(): + # logger.debug(f"Lookup BasicSubmission by start_date({start_date})") + start_date = start_date.strftime("%Y-%m-%d") + case int(): + # logger.debug(f"Lookup BasicSubmission by ordinal start_date {start_date}") + start_date = datetime.fromordinal( + datetime(1900, 1, 1).toordinal() + start_date - 2).date().strftime("%Y-%m-%d") + case _: + # logger.debug(f"Lookup BasicSubmission by parsed str start_date {start_date}") + start_date = parse(start_date).strftime("%Y-%m-%d") + match end_date: + case date() | datetime(): + # logger.debug(f"Lookup BasicSubmission by end_date({end_date})") + end_date = end_date + timedelta(days=1) + end_date = end_date.strftime("%Y-%m-%d") + case int(): + # logger.debug(f"Lookup BasicSubmission by ordinal end_date {end_date}") + end_date = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + end_date - 2).date() + timedelta(days=1) + end_date = end_date.strftime("%Y-%m-%d") + case _: + # logger.debug(f"Lookup BasicSubmission by parsed str end_date {end_date}") + end_date = parse(end_date) + timedelta(days=1) + end_date = end_date.strftime("%Y-%m-%d") + # logger.debug(f"Compensating for same date by using time") + if start_date == end_date: + start_date = datetime.strptime(start_date, "%Y-%m-%d").strftime("%Y-%m-%d %H:%M:%S.%f") + query = query.filter(cls.time == start_date) + else: + query = query.filter(cls.time.between(start_date, end_date)) + return query.all() diff --git a/src/submissions/backend/db/models/controls.py b/src/submissions/backend/db/models/controls.py index bf62552..972c19f 100644 --- a/src/submissions/backend/db/models/controls.py +++ b/src/submissions/backend/db/models/controls.py @@ -142,97 +142,97 @@ class Control(BaseClass): def __repr__(self) -> str: return f"<{self.controltype_name}({self.name})>" - # @classmethod - # @setup_lookup - # def query(cls, - # submission_type: str | None = None, - # subtype: str | None = None, - # start_date: date | str | int | None = None, - # end_date: date | str | int | None = None, - # control_name: str | None = None, - # limit: int = 0, **kwargs - # ) -> Control | List[Control]: - # """ - # Lookup control objects in the database based on a number of parameters. - # - # Args: - # submission_type (str | None, optional): Control archetype. Defaults to None. - # start_date (date | str | int | None, optional): Beginning date to search by. Defaults to 2023-01-01 if end_date not None. - # end_date (date | str | int | None, optional): End date to search by. Defaults to today if start_date not None. - # control_name (str | None, optional): Name of control. Defaults to None. - # limit (int, optional): Maximum number of results to return (0 = all). Defaults to 0. - # - # Returns: - # PCRControl|List[PCRControl]: Control object of interest. - # """ - # from backend.db import SubmissionType - # query: Query = cls.__database_session__.query(cls) - # match submission_type: - # case str(): - # from backend import BasicSubmission, SubmissionType - # # logger.debug(f"Lookup controls by SubmissionType str: {submission_type}") - # query = query.join(BasicSubmission).join(SubmissionType).filter(SubmissionType.name == submission_type) - # case SubmissionType(): - # from backend import BasicSubmission - # # logger.debug(f"Lookup controls by SubmissionType: {submission_type}") - # query = query.join(BasicSubmission).filter(BasicSubmission.submission_type_name == submission_type.name) - # case _: - # pass - # # NOTE: by control type - # match subtype: - # case str(): - # if cls.__name__ == "Control": - # raise ValueError(f"Cannot query base class Control with subtype.") - # elif cls.__name__ == "IridaControl": - # query = query.filter(cls.subtype == subtype) - # else: - # try: - # query = query.filter(cls.subtype == subtype) - # except AttributeError as e: - # logger.error(e) - # case _: - # pass - # # NOTE: by date range - # if start_date is not None and end_date is None: - # logger.warning(f"Start date with no end date, using today.") - # end_date = date.today() - # if end_date is not None and start_date is None: - # logger.warning(f"End date with no start date, using 90 days ago.") - # # start_date = date(2023, 1, 1) - # start_date = date.today() - timedelta(days=90) - # if start_date is not None: - # match start_date: - # case date(): - # # logger.debug(f"Lookup control by start date({start_date})") - # start_date = start_date.strftime("%Y-%m-%d") - # case int(): - # # logger.debug(f"Lookup control by ordinal start date {start_date}") - # start_date = datetime.fromordinal( - # datetime(1900, 1, 1).toordinal() + start_date - 2).date().strftime("%Y-%m-%d") - # case _: - # # logger.debug(f"Lookup control with parsed start date {start_date}") - # start_date = parse(start_date).strftime("%Y-%m-%d") - # match end_date: - # case date(): - # # logger.debug(f"Lookup control by end date({end_date})") - # end_date = end_date.strftime("%Y-%m-%d") - # case int(): - # # logger.debug(f"Lookup control by ordinal end date {end_date}") - # end_date = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + end_date - 2).date().strftime( - # "%Y-%m-%d") - # case _: - # # logger.debug(f"Lookup control with parsed end date {end_date}") - # end_date = parse(end_date).strftime("%Y-%m-%d") - # # logger.debug(f"Looking up BasicSubmissions from start date: {start_date} and end date: {end_date}") - # query = query.filter(cls.submitted_date.between(start_date, end_date)) - # match control_name: - # case str(): - # # logger.debug(f"Lookup control by name {control_name}") - # query = query.filter(cls.name.startswith(control_name)) - # limit = 1 - # case _: - # pass - # return cls.execute_query(query=query, limit=limit) + @classmethod + @setup_lookup + def query(cls, + submission_type: str | None = None, + subtype: str | None = None, + start_date: date | str | int | None = None, + end_date: date | str | int | None = None, + name: str | None = None, + limit: int = 0, **kwargs + ) -> Control | List[Control]: + """ + Lookup control objects in the database based on a number of parameters. + + Args: + submission_type (str | None, optional): Control archetype. Defaults to None. + start_date (date | str | int | None, optional): Beginning date to search by. Defaults to 2023-01-01 if end_date not None. + end_date (date | str | int | None, optional): End date to search by. Defaults to today if start_date not None. + name (str | None, optional): Name of control. Defaults to None. + limit (int, optional): Maximum number of results to return (0 = all). Defaults to 0. + + Returns: + Control|List[Control]: Control object of interest. + """ + from backend.db import SubmissionType + query: Query = cls.__database_session__.query(cls) + match submission_type: + case str(): + from backend import BasicSubmission, SubmissionType + # logger.debug(f"Lookup controls by SubmissionType str: {submission_type}") + query = query.join(BasicSubmission).join(SubmissionType).filter(SubmissionType.name == submission_type) + case SubmissionType(): + from backend import BasicSubmission + # logger.debug(f"Lookup controls by SubmissionType: {submission_type}") + query = query.join(BasicSubmission).filter(BasicSubmission.submission_type_name == submission_type.name) + case _: + pass + # NOTE: by control type + match subtype: + case str(): + if cls.__name__ == "Control": + raise ValueError(f"Cannot query base class Control with subtype.") + elif cls.__name__ == "IridaControl": + query = query.filter(cls.subtype == subtype) + else: + try: + query = query.filter(cls.subtype == subtype) + except AttributeError as e: + logger.error(e) + case _: + pass + # NOTE: by date range + if start_date is not None and end_date is None: + logger.warning(f"Start date with no end date, using today.") + end_date = date.today() + if end_date is not None and start_date is None: + logger.warning(f"End date with no start date, using 90 days ago.") + # start_date = date(2023, 1, 1) + start_date = date.today() - timedelta(days=90) + if start_date is not None: + match start_date: + case date(): + # logger.debug(f"Lookup control by start date({start_date})") + start_date = start_date.strftime("%Y-%m-%d") + case int(): + # logger.debug(f"Lookup control by ordinal start date {start_date}") + start_date = datetime.fromordinal( + datetime(1900, 1, 1).toordinal() + start_date - 2).date().strftime("%Y-%m-%d") + case _: + # logger.debug(f"Lookup control with parsed start date {start_date}") + start_date = parse(start_date).strftime("%Y-%m-%d") + match end_date: + case date(): + # logger.debug(f"Lookup control by end date({end_date})") + end_date = end_date.strftime("%Y-%m-%d") + case int(): + # logger.debug(f"Lookup control by ordinal end date {end_date}") + end_date = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + end_date - 2).date().strftime( + "%Y-%m-%d") + case _: + # logger.debug(f"Lookup control with parsed end date {end_date}") + end_date = parse(end_date).strftime("%Y-%m-%d") + # logger.debug(f"Looking up BasicSubmissions from start date: {start_date} and end date: {end_date}") + query = query.filter(cls.submitted_date.between(start_date, end_date)) + match name: + case str(): + # logger.debug(f"Lookup control by name {control_name}") + query = query.filter(cls.name.startswith(name)) + limit = 1 + case _: + pass + return cls.execute_query(query=query, limit=limit) @classmethod def find_polymorphic_subclass(cls, polymorphic_identity: str | ControlType | None = None, @@ -323,82 +323,82 @@ class PCRControl(Control): return dict(name=self.name, ct=self.ct, subtype=self.subtype, target=self.target, reagent_lot=self.reagent_lot, submitted_date=self.submitted_date.date()) - @classmethod - @setup_lookup - def query(cls, - submission_type: str | None = None, - start_date: date | str | int | None = None, - end_date: date | str | int | None = None, - control_name: str | None = None, - limit: int = 0 - ) -> Control | List[Control]: - """ - Lookup control objects in the database based on a number of parameters. - - Args: - submission_type (str | None, optional): Control archetype. Defaults to None. - start_date (date | str | int | None, optional): Beginning date to search by. Defaults to 2023-01-01 if end_date not None. - end_date (date | str | int | None, optional): End date to search by. Defaults to today if start_date not None. - control_name (str | None, optional): Name of control. Defaults to None. - limit (int, optional): Maximum number of results to return (0 = all). Defaults to 0. - - Returns: - PCRControl|List[PCRControl]: Control object of interest. - """ - from backend.db import SubmissionType - query: Query = cls.__database_session__.query(cls) - # NOTE: by date range - if start_date is not None and end_date is None: - logger.warning(f"Start date with no end date, using today.") - end_date = date.today() - if end_date is not None and start_date is None: - logger.warning(f"End date with no start date, using 90 days ago.") - # start_date = date(2023, 1, 1) - start_date = date.today() - timedelta(days=90) - if start_date is not None: - match start_date: - case date(): - # logger.debug(f"Lookup control by start date({start_date})") - start_date = start_date.strftime("%Y-%m-%d") - case int(): - # logger.debug(f"Lookup control by ordinal start date {start_date}") - start_date = datetime.fromordinal( - datetime(1900, 1, 1).toordinal() + start_date - 2).date().strftime("%Y-%m-%d") - case _: - # logger.debug(f"Lookup control with parsed start date {start_date}") - start_date = parse(start_date).strftime("%Y-%m-%d") - match end_date: - case date(): - # logger.debug(f"Lookup control by end date({end_date})") - end_date = end_date.strftime("%Y-%m-%d") - case int(): - # logger.debug(f"Lookup control by ordinal end date {end_date}") - end_date = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + end_date - 2).date().strftime( - "%Y-%m-%d") - case _: - # logger.debug(f"Lookup control with parsed end date {end_date}") - end_date = parse(end_date).strftime("%Y-%m-%d") - # logger.debug(f"Looking up BasicSubmissions from start date: {start_date} and end date: {end_date}") - query = query.filter(cls.submitted_date.between(start_date, end_date)) - match submission_type: - case str(): - from backend import BasicSubmission, SubmissionType - # logger.debug(f"Lookup controls by SubmissionType str: {submission_type}") - query = query.join(BasicSubmission).join(SubmissionType).filter(SubmissionType.name == submission_type) - case SubmissionType(): - from backend import BasicSubmission - # logger.debug(f"Lookup controls by SubmissionType: {submission_type}") - query = query.join(BasicSubmission).filter(BasicSubmission.submission_type_name==submission_type.name) - case _: - pass - match control_name: - case str(): - # logger.debug(f"Lookup control by name {control_name}") - query = query.filter(cls.name.startswith(control_name)) - limit = 1 - case _: - pass - return cls.execute_query(query=query, limit=limit) + # @classmethod + # @setup_lookup + # def query(cls, + # submission_type: str | None = None, + # start_date: date | str | int | None = None, + # end_date: date | str | int | None = None, + # name: str | None = None, + # limit: int = 0 + # ) -> Control | List[Control]: + # """ + # Lookup control objects in the database based on a number of parameters. + # + # Args: + # submission_type (str | None, optional): Control archetype. Defaults to None. + # start_date (date | str | int | None, optional): Beginning date to search by. Defaults to 2023-01-01 if end_date not None. + # end_date (date | str | int | None, optional): End date to search by. Defaults to today if start_date not None. + # control_name (str | None, optional): Name of control. Defaults to None. + # limit (int, optional): Maximum number of results to return (0 = all). Defaults to 0. + # + # Returns: + # PCRControl|List[PCRControl]: Control object of interest. + # """ + # from backend.db import SubmissionType + # query: Query = cls.__database_session__.query(cls) + # # NOTE: by date range + # if start_date is not None and end_date is None: + # logger.warning(f"Start date with no end date, using today.") + # end_date = date.today() + # if end_date is not None and start_date is None: + # logger.warning(f"End date with no start date, using 90 days ago.") + # # start_date = date(2023, 1, 1) + # start_date = date.today() - timedelta(days=90) + # if start_date is not None: + # match start_date: + # case date(): + # # logger.debug(f"Lookup control by start date({start_date})") + # start_date = start_date.strftime("%Y-%m-%d") + # case int(): + # # logger.debug(f"Lookup control by ordinal start date {start_date}") + # start_date = datetime.fromordinal( + # datetime(1900, 1, 1).toordinal() + start_date - 2).date().strftime("%Y-%m-%d") + # case _: + # # logger.debug(f"Lookup control with parsed start date {start_date}") + # start_date = parse(start_date).strftime("%Y-%m-%d") + # match end_date: + # case date(): + # # logger.debug(f"Lookup control by end date({end_date})") + # end_date = end_date.strftime("%Y-%m-%d") + # case int(): + # # logger.debug(f"Lookup control by ordinal end date {end_date}") + # end_date = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + end_date - 2).date().strftime( + # "%Y-%m-%d") + # case _: + # # logger.debug(f"Lookup control with parsed end date {end_date}") + # end_date = parse(end_date).strftime("%Y-%m-%d") + # # logger.debug(f"Looking up BasicSubmissions from start date: {start_date} and end date: {end_date}") + # query = query.filter(cls.submitted_date.between(start_date, end_date)) + # match submission_type: + # case str(): + # from backend import BasicSubmission, SubmissionType + # # logger.debug(f"Lookup controls by SubmissionType str: {submission_type}") + # query = query.join(BasicSubmission).join(SubmissionType).filter(SubmissionType.name == submission_type) + # case SubmissionType(): + # from backend import BasicSubmission + # # logger.debug(f"Lookup controls by SubmissionType: {submission_type}") + # query = query.join(BasicSubmission).filter(BasicSubmission.submission_type_name==submission_type.name) + # case _: + # pass + # match control_name: + # case str(): + # # logger.debug(f"Lookup control by name {control_name}") + # query = query.filter(cls.name.startswith(control_name)) + # limit = 1 + # case _: + # pass + # return cls.execute_query(query=query, limit=limit) @classmethod @report_result @@ -432,7 +432,7 @@ class PCRControl(Control): def to_pydantic(self): from backend.validators import PydPCRControl - return PydPCRControl(**self.to_sub_dict()) + return PydPCRControl(**self.to_sub_dict(), controltype_name=self.controltype_name, submission_id=self.submission_id) class IridaControl(Control): @@ -569,76 +569,76 @@ class IridaControl(Control): cols = [] return cols - @classmethod - @setup_lookup - def query(cls, - sub_type: str | None = None, - start_date: date | str | int | None = None, - end_date: date | str | int | None = None, - control_name: str | None = None, - limit: int = 0 - ) -> Control | List[Control]: - """ - Lookup control objects in the database based on a number of parameters. - - Args: - sub_type (models.ControlType | str | None, optional): Control archetype. Defaults to None. - start_date (date | str | int | None, optional): Beginning date to search by. Defaults to 2023-01-01 if end_date not None. - end_date (date | str | int | None, optional): End date to search by. Defaults to today if start_date not None. - control_name (str | None, optional): Name of control. Defaults to None. - limit (int, optional): Maximum number of results to return (0 = all). Defaults to 0. - - Returns: - models.Control|List[models.Control]: Control object of interest. - """ - query: Query = cls.__database_session__.query(cls) - # NOTE: by control type - match sub_type: - case str(): - query = query.filter(cls.subtype == sub_type) - case _: - pass - # NOTE: If one date exists, we need the other one to exist as well. - if start_date is not None and end_date is None: - logger.warning(f"Start date with no end date, using today.") - end_date = date.today() - if end_date is not None and start_date is None: - logger.warning(f"End date with no start date, using 90 days ago.") - # start_date = date(2023, 1, 1) - start_date = date.today() - timedelta(days=90) - if start_date is not None: - match start_date: - case date(): - # logger.debug(f"Lookup control by start date({start_date})") - start_date = start_date.strftime("%Y-%m-%d") - case int(): - # logger.debug(f"Lookup control by ordinal start date {start_date}") - start_date = datetime.fromordinal( - datetime(1900, 1, 1).toordinal() + start_date - 2).date().strftime("%Y-%m-%d") - case _: - # logger.debug(f"Lookup control with parsed start date {start_date}") - start_date = parse(start_date).strftime("%Y-%m-%d") - match end_date: - case date(): - # logger.debug(f"Lookup control by end date({end_date})") - end_date = end_date.strftime("%Y-%m-%d") - case int(): - # logger.debug(f"Lookup control by ordinal end date {end_date}") - end_date = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + end_date - 2).date().strftime( - "%Y-%m-%d") - case _: - # logger.debug(f"Lookup control with parsed end date {end_date}") - end_date = parse(end_date).strftime("%Y-%m-%d") - # logger.debug(f"Looking up BasicSubmissions from start date: {start_date} and end date: {end_date}") - query = query.filter(cls.submitted_date.between(start_date, end_date)) - match control_name: - case str(): - # logger.debug(f"Lookup control by name {control_name}") - query = query.filter(cls.name.startswith(control_name)) - limit = 1 - case _: - pass - return cls.execute_query(query=query, limit=limit) + # @classmethod + # @setup_lookup + # def query(cls, + # sub_type: str | None = None, + # start_date: date | str | int | None = None, + # end_date: date | str | int | None = None, + # control_name: str | None = None, + # limit: int = 0 + # ) -> Control | List[Control]: + # """ + # Lookup control objects in the database based on a number of parameters. + # + # Args: + # sub_type (models.ControlType | str | None, optional): Control archetype. Defaults to None. + # start_date (date | str | int | None, optional): Beginning date to search by. Defaults to 2023-01-01 if end_date not None. + # end_date (date | str | int | None, optional): End date to search by. Defaults to today if start_date not None. + # control_name (str | None, optional): Name of control. Defaults to None. + # limit (int, optional): Maximum number of results to return (0 = all). Defaults to 0. + # + # Returns: + # models.Control|List[models.Control]: Control object of interest. + # """ + # query: Query = cls.__database_session__.query(cls) + # # NOTE: by control type + # match sub_type: + # case str(): + # query = query.filter(cls.subtype == sub_type) + # case _: + # pass + # # NOTE: If one date exists, we need the other one to exist as well. + # if start_date is not None and end_date is None: + # logger.warning(f"Start date with no end date, using today.") + # end_date = date.today() + # if end_date is not None and start_date is None: + # logger.warning(f"End date with no start date, using 90 days ago.") + # # start_date = date(2023, 1, 1) + # start_date = date.today() - timedelta(days=90) + # if start_date is not None: + # match start_date: + # case date(): + # # logger.debug(f"Lookup control by start date({start_date})") + # start_date = start_date.strftime("%Y-%m-%d") + # case int(): + # # logger.debug(f"Lookup control by ordinal start date {start_date}") + # start_date = datetime.fromordinal( + # datetime(1900, 1, 1).toordinal() + start_date - 2).date().strftime("%Y-%m-%d") + # case _: + # # logger.debug(f"Lookup control with parsed start date {start_date}") + # start_date = parse(start_date).strftime("%Y-%m-%d") + # match end_date: + # case date(): + # # logger.debug(f"Lookup control by end date({end_date})") + # end_date = end_date.strftime("%Y-%m-%d") + # case int(): + # # logger.debug(f"Lookup control by ordinal end date {end_date}") + # end_date = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + end_date - 2).date().strftime( + # "%Y-%m-%d") + # case _: + # # logger.debug(f"Lookup control with parsed end date {end_date}") + # end_date = parse(end_date).strftime("%Y-%m-%d") + # # logger.debug(f"Looking up BasicSubmissions from start date: {start_date} and end date: {end_date}") + # query = query.filter(cls.submitted_date.between(start_date, end_date)) + # match control_name: + # case str(): + # # logger.debug(f"Lookup control by name {control_name}") + # query = query.filter(cls.name.startswith(control_name)) + # limit = 1 + # case _: + # pass + # return cls.execute_query(query=query, limit=limit) @classmethod def make_parent_buttons(cls, parent: QWidget) -> None: @@ -828,7 +828,7 @@ class IridaControl(Control): return df, previous_dates # NOTE: if date was changed, rerun with new date else: - logger.warning(f"Date check failed, running recursion") + # logger.warning(f"Date check failed, running recursion") df, previous_dates = cls.check_date(df, item, previous_dates) return df, previous_dates diff --git a/src/submissions/backend/db/models/kits.py b/src/submissions/backend/db/models/kits.py index c57f7b2..5f649d8 100644 --- a/src/submissions/backend/db/models/kits.py +++ b/src/submissions/backend/db/models/kits.py @@ -485,6 +485,8 @@ class Reagent(BaseClass): output['editable'] = ['lot', 'expiry'] return output + + def update_last_used(self, kit: KitType) -> Report: """ Updates last used reagent lot for ReagentType/KitType @@ -1282,7 +1284,8 @@ class SubmissionReagentAssociation(BaseClass): try: return f"<{self.submission.rsl_plate_num} & {self.reagent.lot}>" except AttributeError: - return f"" def __init__(self, reagent=None, submission=None): if isinstance(reagent, list): @@ -1347,6 +1350,9 @@ class SubmissionReagentAssociation(BaseClass): output['comments'] = self.comments return output + def to_pydantic(self, extraction_kit: KitType): + from backend.validators import PydReagent + return PydReagent(**self.to_sub_dict(extraction_kit=extraction_kit)) class Equipment(BaseClass): """ @@ -1394,6 +1400,8 @@ class Equipment(BaseClass): else: return {k: v for k, v in self.__dict__.items()} + + def get_processes(self, submission_type: SubmissionType, extraction_kit: str | KitType | None = None) -> List[str]: """ Get all processes associated with this Equipment for a given SubmissionType @@ -1682,6 +1690,7 @@ class SubmissionEquipmentAssociation(BaseClass): equipment = relationship(Equipment, back_populates="equipment_submission_associations") #: associated equipment + def __repr__(self) -> str: return f"" @@ -1706,6 +1715,10 @@ class SubmissionEquipmentAssociation(BaseClass): processes=[process], role=self.role, nickname=self.equipment.nickname) return output + def to_pydantic(self): + from backend.validators import PydEquipment + return PydEquipment(**self.to_sub_dict()) + @classmethod @setup_lookup def query(cls, equipment_id: int, submission_id: int, role: str | None = None, limit: int = 0, **kwargs) -> Any | \ @@ -1999,3 +2012,7 @@ class SubmissionTipsAssociation(BaseClass): query = query.filter(cls.submission_id == submission_id) query = query.filter(cls.role_name == role) return cls.execute_query(query=query, limit=limit, **kwargs) + + def to_pydantic(self): + from backend.validators import PydTips + return PydTips(name=self.tips.name, lot=self.tips.lot, role=self.role_name) diff --git a/src/submissions/backend/db/models/submissions.py b/src/submissions/backend/db/models/submissions.py index 98b2067..da0fd8d 100644 --- a/src/submissions/backend/db/models/submissions.py +++ b/src/submissions/backend/db/models/submissions.py @@ -2,30 +2,30 @@ Models for the main submission and sample types. """ from __future__ import annotations -import sys -import types -import zipfile +# import sys +# import types +# import zipfile from copy import deepcopy from getpass import getuser -import logging, uuid, tempfile, re, base64 +import logging, uuid, tempfile, re, base64, numpy as np, pandas as pd, types, sys from zipfile import ZipFile from tempfile import TemporaryDirectory, TemporaryFile from operator import itemgetter from pprint import pformat from . import BaseClass, Reagent, SubmissionType, KitType, Organization, Contact, LogMixin -from sqlalchemy import Column, String, TIMESTAMP, INTEGER, ForeignKey, JSON, FLOAT, case, event, inspect +from sqlalchemy import Column, String, TIMESTAMP, INTEGER, ForeignKey, JSON, FLOAT, case, event, inspect, func from sqlalchemy.orm import relationship, validates, Query from sqlalchemy.orm.attributes import flag_modified from sqlalchemy.ext.associationproxy import association_proxy from sqlalchemy.exc import OperationalError as AlcOperationalError, IntegrityError as AlcIntegrityError, StatementError, \ ArgumentError from sqlite3 import OperationalError as SQLOperationalError, IntegrityError as SQLIntegrityError -import pandas as pd +# import pandas as pd from openpyxl import Workbook from openpyxl.drawing.image import Image as OpenpyxlImage from tools import row_map, setup_lookup, jinja_template_loading, rreplace, row_keys, check_key_or_attr, Result, Report, \ - report_result -from datetime import datetime, date + report_result, create_holidays_for_year +from datetime import datetime, date, timedelta from typing import List, Any, Tuple, Literal, Generator from dateutil.parser import parse from pathlib import Path @@ -73,6 +73,7 @@ class BasicSubmission(BaseClass, LogMixin): custom = Column(JSON) controls = relationship("Control", back_populates="submission", uselist=True) #: A control sample added to submission + completed_date = Column(TIMESTAMP) submission_sample_associations = relationship( "SubmissionSampleAssociation", @@ -345,6 +346,8 @@ class BasicSubmission(BaseClass, LogMixin): tips = self.generate_associations(name="submission_tips_associations") cost_centre = self.cost_centre custom = self.custom + controls = [item.to_sub_dict() for item in self.controls] + else: reagents = None samples = None @@ -352,6 +355,7 @@ class BasicSubmission(BaseClass, LogMixin): tips = None cost_centre = None custom = None + controls = None # logger.debug("Getting comments") try: comments = self.comment @@ -381,6 +385,8 @@ class BasicSubmission(BaseClass, LogMixin): output["contact"] = contact output["contact_phone"] = contact_phone output["custom"] = custom + output["controls"] = controls + output["completed_date"] = self.completed_date return output def calculate_column_count(self) -> int: @@ -619,7 +625,7 @@ class BasicSubmission(BaseClass, LogMixin): Returns: PydSubmission: converted object. """ - from backend.validators import PydSubmission, PydSample, PydReagent, PydEquipment + from backend.validators import PydSubmission dicto = self.to_dict(full_data=True, backup=backup) # logger.debug("To dict complete") new_dict = {} @@ -628,24 +634,43 @@ class BasicSubmission(BaseClass, LogMixin): missing = value in ['', 'None', None] match key: case "reagents": - new_dict[key] = [PydReagent(**reagent) for reagent in value] + # new_dict[key] = [PydReagent(**reagent) for reagent in value] + field_value = [item.to_pydantic(extraction_kit=self.extraction_kit) for item in self.submission_reagent_associations] case "samples": - new_dict[key] = [PydSample(**{k.lower().replace(" ", "_"): v for k, v in sample.items()}) for sample - in dicto['samples']] + field_value = [item.to_pydantic() for item in self.submission_sample_associations] case "equipment": + field_value = [item.to_pydantic() for item in self.submission_equipment_associations] + case "controls": try: - new_dict[key] = [PydEquipment(**equipment) for equipment in dicto['equipment']] + field_value = [item.to_pydantic() for item in self.__getattribute__(key)] except TypeError as e: - logger.error(f"Possible no equipment error: {e}") + logger.error(f"Error converting {key} to pydantic :{e}") + continue + case "tips": + field_value = [item.to_pydantic() for item in self.submission_tips_associations] + case "submission_type" | "contact": + field_value = dict(value=self.__getattribute__(key).name, missing=missing) case "plate_number": - new_dict['rsl_plate_num'] = dict(value=value, missing=missing) + key = 'rsl_plate_num' + field_value = dict(value=self.rsl_plate_num, missing=missing) + # continue case "submitter_plate_number": - new_dict['submitter_plate_num'] = dict(value=value, missing=missing) + # new_dict['submitter_plate_num'] = dict(value=self.submitter_plate_num, missing=missing) + # continue + key = "submitter_plate_num" + field_value = dict(value=self.submitter_plate_num, missing=missing) case "id": - pass + continue case _: - logger.debug(f"Setting dict {key} to {value}") - new_dict[key.lower().replace(" ", "_")] = dict(value=value, missing=missing) + try: + key = key.lower().replace(" ", "_") + field_value = dict(value=self.__getattribute__(key), missing=missing) + # new_dict[key.lower().replace(" ", "_")] = dict(value=self.__getattribute__(key), missing=missing) + except AttributeError: + logger.error(f"{key} is not available in {self}") + continue + logger.debug(f"Setting dict {key}") + new_dict[key] = field_value # logger.debug(f"{key} complete after {time()-start}") new_dict['filepath'] = Path(tempfile.TemporaryFile().name) # logger.debug("Done converting fields.") @@ -1021,6 +1046,7 @@ class BasicSubmission(BaseClass, LogMixin): Tuple(dict, Template): (Updated dictionary, Template to be rendered) """ base_dict['excluded'] = cls.get_default_info('details_ignore') + base_dict['excluded'] += ['controls'] env = jinja_template_loading() temp_name = f"{cls.__name__.lower()}_details.html" # logger.debug(f"Returning template: {temp_name}") @@ -1081,11 +1107,11 @@ class BasicSubmission(BaseClass, LogMixin): end_date = date.today() if end_date is not None and start_date is None: logger.warning(f"End date with no start date, using Jan 1, 2023") - start_date = date(2023, 1, 1) + start_date = cls.__database_session__.query(cls, func.min(cls.submitted_date)).first()[1] if start_date is not None: # logger.debug(f"Querying with start date: {start_date} and end date: {end_date}") match start_date: - case date(): + case date() | datetime(): # logger.debug(f"Lookup BasicSubmission by start_date({start_date})") start_date = start_date.strftime("%Y-%m-%d") case int(): @@ -1098,14 +1124,18 @@ class BasicSubmission(BaseClass, LogMixin): match end_date: case date() | datetime(): # logger.debug(f"Lookup BasicSubmission by end_date({end_date})") + end_date = end_date + timedelta(days=1) end_date = end_date.strftime("%Y-%m-%d") case int(): # logger.debug(f"Lookup BasicSubmission by ordinal end_date {end_date}") - end_date = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + end_date - 2).date().strftime( + end_date = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + end_date - 2).date() + timedelta( + days=1) + end_date = end_date.strftime( "%Y-%m-%d") case _: # logger.debug(f"Lookup BasicSubmission by parsed str end_date {end_date}") - end_date = parse(end_date).strftime("%Y-%m-%d") + end_date = parse(end_date) + timedelta(days=1) + end_date = end_date.strftime("%Y-%m-%d") # logger.debug(f"Compensating for same date by using time") if start_date == end_date: start_date = datetime.strptime(start_date, "%Y-%m-%d").strftime("%Y-%m-%d %H:%M:%S.%f") @@ -1339,10 +1369,22 @@ class BasicSubmission(BaseClass, LogMixin): writer = pyd.to_writer() writer.xl.save(filename=fname.with_suffix(".xlsx")) + def get_turnaround_time(self): + completed = self.completed_date or datetime.now() + return self.calculate_turnaround(start_date=self.submitted_date.date(), end_date=completed.date()) + + @classmethod + def calculate_turnaround(cls, start_date:date|None=None, end_date:date|None=None) -> int|None: + try: + delta = np.busday_count(start_date, end_date, holidays=create_holidays_for_year(start_date.year)) + except ValueError: + return None + return delta + 1 + # Below are the custom submission types -class BacterialCulture(BasicSubmission, LogMixin): +class BacterialCulture(BasicSubmission): """ derivative submission type from BasicSubmission """ @@ -1429,7 +1471,7 @@ class BacterialCulture(BasicSubmission, LogMixin): return input_dict -class Wastewater(BasicSubmission, LogMixin): +class Wastewater(BasicSubmission): """ derivative submission type from BasicSubmission """ @@ -1868,7 +1910,7 @@ class WastewaterArtic(BasicSubmission): pass try: input_dict['source_plate_number'] = int(input_dict['source_plate_number']) - except ValueError: + except (ValueError, KeyError): input_dict['source_plate_number'] = 0 # NOTE: Because generate_sample_object needs the submitter_id and the artic has the "({origin well})" # at the end, this has to be done here. No moving to sqlalchemy object :( @@ -2276,6 +2318,10 @@ class BasicSample(BaseClass): # logger.debug(f"Done converting {self} after {time()-start}") return sample + def to_pydantic(self): + from backend.validators import PydSample + return PydSample(**self.to_sub_dict()) + def set_attribute(self, name: str, value): """ Custom attribute setter (depreciated over built-in __setattr__) @@ -2733,6 +2779,10 @@ class SubmissionSampleAssociation(BaseClass): sample['submission_rank'] = self.submission_rank return sample + def to_pydantic(self): + from backend.validators import PydSample + return PydSample(**self.to_sub_dict()) + def to_hitpick(self) -> dict | None: """ Outputs a dictionary usable for html plate maps. diff --git a/src/submissions/backend/excel/parser.py b/src/submissions/backend/excel/parser.py index 90d91dd..66c6b60 100644 --- a/src/submissions/backend/excel/parser.py +++ b/src/submissions/backend/excel/parser.py @@ -227,6 +227,9 @@ class InfoParser(object): case "submission_type": value, missing = is_missing(value) value = value.title() + case "submitted_date": + value, missing = is_missing(value) + logger.debug(f"Parsed submitted date: {value}") # NOTE: is field a JSON? case thing if thing in self.sub_object.jsons(): value, missing = is_missing(value) diff --git a/src/submissions/backend/validators/pydant.py b/src/submissions/backend/validators/pydant.py index 0a0a44f..1a87c24 100644 --- a/src/submissions/backend/validators/pydant.py +++ b/src/submissions/backend/validators/pydant.py @@ -807,22 +807,28 @@ class PydSubmission(BaseModel, extra='allow'): # logger.debug(f"Setting {key} to {value}") match key: case "reagents": - if report.results[0].code == 1: - instance.submission_reagent_associations = [] + # if report.results[0].code == 1: + # instance.submission_reagent_associations = [] # logger.debug(f"Looking through {self.reagents}") for reagent in self.reagents: reagent, assoc, _ = reagent.toSQL(submission=instance) # logger.debug(f"Association: {assoc}") if assoc is not None: # and assoc not in instance.submission_reagent_associations: - instance.submission_reagent_associations.append(assoc) + if assoc not in instance.submission_reagent_associations: + instance.submission_reagent_associations.append(assoc) + else: + logger.warning(f"Reagent association {assoc} is already present in {instance}") case "samples": for sample in self.samples: sample, associations, _ = sample.toSQL(submission=instance) # logger.debug(f"Sample SQL object to be added to submission: {sample.__dict__}") - logger.debug(associations) + # logger.debug(associations) for assoc in associations: - if assoc is not None and assoc not in instance.submission_sample_associations: - instance.submission_sample_associations.append(assoc) + if assoc is not None: + if assoc not in instance.submission_sample_associations: + instance.submission_sample_associations.append(assoc) + else: + logger.warning(f"Sample association {assoc} is already present in {instance}") case "equipment": # logger.debug(f"Equipment: {pformat(self.equipment)}") for equip in self.equipment: @@ -841,8 +847,11 @@ class PydSubmission(BaseModel, extra='allow'): association = tips.to_sql(submission=instance) except AttributeError: continue - if association is not None and association not in instance.submission_tips_associations: - instance.submission_tips_associations.append(association) + if association is not None: + if association not in instance.submission_tips_associations: + instance.submission_tips_associations.append(association) + else: + logger.warning(f"Tips association {association} is already present in {instance}") case item if item in instance.timestamps(): logger.warning(f"Incoming timestamp key: {item}, with value: {value}") # value = value.replace(tzinfo=timezone) @@ -871,14 +880,20 @@ class PydSubmission(BaseModel, extra='allow'): instance.set_attribute(key=key, value=value) case _: try: - instance.set_attribute(key=key, value=value) - # instance.update({key:value}) - except AttributeError as e: - logger.error(f"Could not set attribute: {key} to {value} due to: \n\n {e}") + check = instance.__getattribute__(key) != value + except AttributeError: continue - except KeyError: - continue - print(f"\n\n{instance}\n\n") + if check: + try: + instance.set_attribute(key=key, value=value) + # instance.update({key:value}) + except AttributeError as e: + logger.error(f"Could not set attribute: {key} to {value} due to: \n\n {e}") + continue + except KeyError: + continue + else: + logger.warning(f"{key} already == {value} so no updating.") try: # logger.debug(f"Calculating costs for procedure...") instance.calculate_base_cost() @@ -1119,6 +1134,16 @@ class PydPCRControl(BaseModel): submission_id: int controltype_name: str + def to_sql(self): + instance = PCRControl.query(name=self.name) + if not instance: + instance = PCRControl() + for key in self.model_fields: + field_value = self.__getattribute__(key) + if instance.__getattribute__(key) != field_value: + instance.__setattr__(key, field_value) + return instance + class PydIridaControl(BaseModel, extra='ignore'): name: str @@ -1133,3 +1158,13 @@ class PydIridaControl(BaseModel, extra='ignore'): submitted_date: datetime #: Date submitted to Robotics submission_id: int controltype_name: str + + def to_sql(self): + instance = IridaControl.query(name=self.name) + if not instance: + instance = IridaControl() + for key in self.model_fields: + field_value = self.__getattribute__(key) + if instance.__getattribute__(key) != field_value: + instance.__setattr__(key, field_value) + return instance diff --git a/src/submissions/frontend/widgets/misc.py b/src/submissions/frontend/widgets/misc.py index 8432e8b..177594e 100644 --- a/src/submissions/frontend/widgets/misc.py +++ b/src/submissions/frontend/widgets/misc.py @@ -110,7 +110,7 @@ class AddReagentForm(QDialog): """ # logger.debug(self.type_input.currentText()) self.name_input.clear() - lookup = Reagent.query(reagent_role=self.type_input.currentText()) + lookup = Reagent.query(role=self.type_input.currentText()) self.name_input.addItems(list(set([item.name for item in lookup]))) diff --git a/src/submissions/frontend/widgets/submission_details.py b/src/submissions/frontend/widgets/submission_details.py index c7af9fc..faebc85 100644 --- a/src/submissions/frontend/widgets/submission_details.py +++ b/src/submissions/frontend/widgets/submission_details.py @@ -176,6 +176,7 @@ class SubmissionDetails(QDialog): if isinstance(submission, str): submission = BasicSubmission.query(rsl_plate_num=submission) submission.signed_by = getuser() + submission.completed = datetime.now().date() submission.save() self.submission_details(submission=self.rsl_plate_num) diff --git a/src/submissions/frontend/widgets/submission_widget.py b/src/submissions/frontend/widgets/submission_widget.py index 46ca734..a074669 100644 --- a/src/submissions/frontend/widgets/submission_widget.py +++ b/src/submissions/frontend/widgets/submission_widget.py @@ -542,10 +542,25 @@ class SubmissionFormWidget(QWidget): # NOTE: lookup organizations suitable for submitting_lab (ctx: self.InfoItem.SubmissionFormWidget.SubmissionFormContainer.AddSubForm ) labs = [item.name for item in Organization.query()] # NOTE: try to set closest match to top of list + # try: + # labs = difflib.get_close_matches(value, labs, len(labs), 0) + # except (TypeError, ValueError): + # pass + if isinstance(value, dict): + value = value['value'] + if isinstance(value, Organization): + value = value.name try: - labs = difflib.get_close_matches(value, labs, len(labs), 0) - except (TypeError, ValueError): - pass + looked_up_lab = Organization.query(name=value, limit=1) + except AttributeError: + looked_up_lab = None + logger.debug(f"\n\nLooked up lab: {looked_up_lab}") + if looked_up_lab: + try: + labs.remove(str(looked_up_lab.name)) + except ValueError as e: + logger.error(f"Error reordering labs: {e}") + labs.insert(0, str(looked_up_lab.name)) # NOTE: set combobox values to lookedup values add_widget.addItems(labs) add_widget.setToolTip("Select submitting lab.") @@ -760,7 +775,6 @@ class SubmissionFormWidget(QWidget): if looked_up_reg: try: relevant_reagents.remove(str(looked_up_reg.lot)) - except ValueError as e: logger.error(f"Error reordering relevant reagents: {e}") relevant_reagents.insert(0, str(looked_up_reg.lot)) diff --git a/src/submissions/templates/basicsubmission_details.html b/src/submissions/templates/basicsubmission_details.html index 5cd7c22..6d642ef 100644 --- a/src/submissions/templates/basicsubmission_details.html +++ b/src/submissions/templates/basicsubmission_details.html @@ -71,7 +71,7 @@ {% endif %} {% endblock %} {% block signing_button %} - {% if permission %} + {% if permission and not sub['signed_by'] %} {% endif %} {% endblock %} diff --git a/src/submissions/tools/__init__.py b/src/submissions/tools/__init__.py index 497a813..8abba64 100644 --- a/src/submissions/tools/__init__.py +++ b/src/submissions/tools/__init__.py @@ -5,10 +5,12 @@ from __future__ import annotations import json import pprint +from datetime import date, datetime, timedelta from json import JSONDecodeError import numpy as np import logging, re, yaml, sys, os, stat, platform, getpass, inspect import pandas as pd +from dateutil.easter import easter from jinja2 import Environment, FileSystemLoader from logging import handlers from pathlib import Path @@ -990,3 +992,41 @@ def report_result(func): return wrapper +def create_holidays_for_year(year: int|None=None) -> List[date]: + def find_nth_monday(year, month, occurence: int | None=None, day: int|None=None): + if not occurence: + occurence = 1 + if not day: + day = occurence * 7 + max_days = (date(2012, month+1, 1) - date(2012, month, 1)).days + if day > max_days: + day = max_days + try: + d = datetime(year, int(month), day=day) + except ValueError: + return + offset = -d.weekday() # weekday == 0 means Monday + output = d + timedelta(offset) + return output.date() + if not year: + year = date.today().year + # Includes New Year's day for next year. + holidays = [date(year, 1, 1), date(year, 7,1), date(year, 9, 30), + date(year, 11, 11), date(year, 12, 25), date(year, 12, 26), + date(year+1, 1, 1)] + # August Civic + # holidays.append(find_nth_monday(year, 8)) + # Labour Day + holidays.append(find_nth_monday(year, 9)) + # Thanksgiving + holidays.append(find_nth_monday(year, 10, occurence=2)) + # Victoria Day + holidays.append(find_nth_monday(year, 5, day=25)) + # Easter, etc + holidays.append(easter(year) - timedelta(days=2)) + holidays.append(easter(year) + timedelta(days=1)) + return sorted(holidays) + + + +