diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ad3de3..3b17e65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# 202504.02 + +- Refined query-by-date to use start/end of day times to improve accuracy. + # 202504.01 - Added in method to backup submissions to xlsx (partly). diff --git a/TODO.md b/TODO.md index 2d7948d..997b1f7 100644 --- a/TODO.md +++ b/TODO.md @@ -1,3 +1,6 @@ +- [x] Apply below fix to all other date-based queries. +- [x] Fix Control graph chart bug that excludes today's controls. +- [x] Convert logger to a custom class. - [x] Change "Manage Organizations" to the Pydantic version. - [x] Can my "to_dict", "to_sub_dict", "to_pydantic" methods be rewritten as properties? - [ ] Stop displacing date on Irida controls and just do what Turnaround time does. diff --git a/src/submissions/__main__.py b/src/submissions/__main__.py index d8e2a82..7232a1b 100644 --- a/src/submissions/__main__.py +++ b/src/submissions/__main__.py @@ -1,12 +1,15 @@ +import logging import sys, os -from tools import ctx, setup_logger, check_if_app +from tools import ctx, check_if_app, CustomLogger # NOTE: environment variable must be set to enable qtwebengine in network path if check_if_app(): os.environ['QTWEBENGINE_DISABLE_SANDBOX'] = "1" # NOTE: setup custom logger -logger = setup_logger(verbosity=3) +logging.setLoggerClass(CustomLogger) +# logger = logging.getLogger("submissions") +# logger = setup_logger(verbosity=3) from PyQt6.QtWidgets import QApplication from frontend.widgets.app import App diff --git a/src/submissions/backend/db/models/controls.py b/src/submissions/backend/db/models/controls.py index e6b1ae0..b4f7836 100644 --- a/src/submissions/backend/db/models/controls.py +++ b/src/submissions/backend/db/models/controls.py @@ -12,7 +12,8 @@ from sqlalchemy.orm import relationship, Query, validates import logging, re from operator import itemgetter from . import BaseClass -from tools import setup_lookup, report_result, Result, Report, Settings, get_unique_values_in_df_column, super_splitter +from tools import setup_lookup, report_result, Result, Report, Settings, get_unique_values_in_df_column, super_splitter, \ + rectify_query_date from datetime import date, datetime, timedelta from typing import List, Literal, Tuple, Generator from dateutil.parser import parse @@ -149,8 +150,8 @@ class Control(BaseClass): def query(cls, submissiontype: str | None = None, subtype: str | None = None, - start_date: date | str | int | None = None, - end_date: date | str | int | None = None, + start_date: date | datetime | str | int | None = None, + end_date: date | datetime | str | int | None = None, name: str | None = None, limit: int = 0, **kwargs ) -> Control | List[Control]: @@ -201,22 +202,30 @@ class Control(BaseClass): logger.warning(f"End date with no start date, using 90 days ago.") start_date = date.today() - timedelta(days=90) if start_date is not None: - match start_date: - case date(): - start_date = start_date.strftime("%Y-%m-%d") - case int(): - start_date = datetime.fromordinal( - datetime(1900, 1, 1).toordinal() + start_date - 2).date().strftime("%Y-%m-%d") - case _: - start_date = parse(start_date).strftime("%Y-%m-%d") - match end_date: - case date(): - end_date = end_date.strftime("%Y-%m-%d") - case int(): - end_date = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + end_date - 2).date().strftime( - "%Y-%m-%d") - case _: - end_date = parse(end_date).strftime("%Y-%m-%d") + # match start_date: + # case datetime(): + # start_date = start_date.strftime("%Y-%m-%d %H:%M:%S") + # case date(): + # start_date = datetime.combine(start_date, datetime.min.time()) + # start_date = start_date.strftime("%Y-%m-%d %H:%M:%S") + # case int(): + # start_date = datetime.fromordinal( + # datetime(1900, 1, 1).toordinal() + start_date - 2).date().strftime("%Y-%m-%d %H:%M:%S") + # case _: + # start_date = parse(start_date).strftime("%Y-%m-%d %H:%M:%S") + start_date = rectify_query_date(start_date) + end_date = rectify_query_date(end_date, eod=True) + # match end_date: + # case datetime(): + # end_date = end_date.strftime("%Y-%m-%d %H:%M:%S") + # case date(): + # end_date = datetime.combine(end_date, datetime.max.time()) + # end_date = end_date.strftime("%Y-%m-%d %H:%M:%S") + # case int(): + # end_date = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + end_date - 2).date().strftime( + # "%Y-%m-%d %H:%M:%S") + # case _: + # end_date = parse(end_date).strftime("%Y-%m-%d %H:%M:%S") query = query.filter(cls.submitted_date.between(start_date, end_date)) match name: case str(): diff --git a/src/submissions/backend/db/models/submissions.py b/src/submissions/backend/db/models/submissions.py index 215db6b..5aede85 100644 --- a/src/submissions/backend/db/models/submissions.py +++ b/src/submissions/backend/db/models/submissions.py @@ -27,7 +27,7 @@ from sqlite3 import OperationalError as SQLOperationalError, IntegrityError as S from openpyxl import Workbook from openpyxl.drawing.image import Image as OpenpyxlImage from tools import row_map, setup_lookup, jinja_template_loading, rreplace, row_keys, check_key_or_attr, Result, Report, \ - report_result, create_holidays_for_year, check_dictionary_inclusion_equality + report_result, create_holidays_for_year, check_dictionary_inclusion_equality, rectify_query_date from datetime import datetime, date, timedelta from typing import List, Any, Tuple, Literal, Generator, Type from dateutil.parser import parse @@ -1152,35 +1152,37 @@ class BasicSubmission(BaseClass, LogMixin): start_date = cls.__database_session__.query(cls, func.min(cls.submitted_date)).first()[1] logger.warning(f"End date with no start date, using first submission date: {start_date}") if start_date is not None: - match start_date: - case date(): - pass - case datetime(): - start_date = start_date.date() - case int(): - start_date = datetime.fromordinal( - datetime(1900, 1, 1).toordinal() + start_date - 2).date() - case _: - start_date = parse(start_date).date() - # start_date = start_date.strftime("%Y-%m-%d") - match end_date: - case date(): - pass - case datetime(): - end_date = end_date # + timedelta(days=1) - # pass - case int(): - end_date = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + end_date - 2).date() # \ - # + timedelta(days=1) - case _: - end_date = parse(end_date).date() # + timedelta(days=1) - # end_date = end_date.strftime("%Y-%m-%d") - start_date = datetime.combine(start_date, datetime.min.time()).strftime("%Y-%m-%d %H:%M:%S.%f") - end_date = datetime.combine(end_date, datetime.max.time()).strftime("%Y-%m-%d %H:%M:%S.%f") - # if start_date == end_date: - # start_date = start_date.strftime("%Y-%m-%d %H:%M:%S.%f") - # query = query.filter(model.submitted_date == start_date) - # else: + # match start_date: + # case date(): + # pass + # case datetime(): + # start_date = start_date.date() + # case int(): + # start_date = datetime.fromordinal( + # datetime(1900, 1, 1).toordinal() + start_date - 2).date() + # case _: + # start_date = parse(start_date).date() + # # start_date = start_date.strftime("%Y-%m-%d") + # match end_date: + # case date(): + # pass + # case datetime(): + # end_date = end_date # + timedelta(days=1) + # # pass + # case int(): + # end_date = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + end_date - 2).date() # \ + # # + timedelta(days=1) + # case _: + # end_date = parse(end_date).date() # + timedelta(days=1) + # # end_date = end_date.strftime("%Y-%m-%d") + # start_date = datetime.combine(start_date, datetime.min.time()).strftime("%Y-%m-%d %H:%M:%S.%f") + # end_date = datetime.combine(end_date, datetime.max.time()).strftime("%Y-%m-%d %H:%M:%S.%f") + # # if start_date == end_date: + # # start_date = start_date.strftime("%Y-%m-%d %H:%M:%S.%f") + # # query = query.filter(model.submitted_date == start_date) + # # else: + start_date = rectify_query_date(start_date) + end_date = rectify_query_date(end_date, eod=True) query = query.filter(model.submitted_date.between(start_date, end_date)) # NOTE: by reagent (for some reason) match reagent: diff --git a/src/submissions/frontend/widgets/controls_chart.py b/src/submissions/frontend/widgets/controls_chart.py index 3a52a5d..c61de74 100644 --- a/src/submissions/frontend/widgets/controls_chart.py +++ b/src/submissions/frontend/widgets/controls_chart.py @@ -108,6 +108,7 @@ class ControlsViewer(InfoPane): parent=self, months=months ) + logger.debug(f"Chart settings: {chart_settings}") self.fig = self.archetype.instance_class.make_chart(chart_settings=chart_settings, parent=self, ctx=self.app.ctx) self.report_obj = ChartReportMaker(df=self.fig.df, sheet_name=self.archetype.name) if issubclass(self.fig.__class__, CustomFigure): diff --git a/src/submissions/frontend/widgets/info_tab.py b/src/submissions/frontend/widgets/info_tab.py index 1ae3fa1..6b626dc 100644 --- a/src/submissions/frontend/widgets/info_tab.py +++ b/src/submissions/frontend/widgets/info_tab.py @@ -1,7 +1,7 @@ """ A pane to show info e.g. cost reports and turnaround times. """ -from datetime import date +from datetime import date, datetime from PyQt6.QtCore import QSignalBlocker from PyQt6.QtWebEngineWidgets import QWebEngineView from PyQt6.QtWidgets import QWidget, QGridLayout @@ -32,8 +32,11 @@ class InfoPane(QWidget): @report_result def update_data(self, *args, **kwargs): report = Report() + # self.start_date = self.datepicker.start_date.date().toPyDate() + # self.end_date = self.datepicker.end_date.date().toPyDate() self.start_date = self.datepicker.start_date.date().toPyDate() self.end_date = self.datepicker.end_date.date().toPyDate() + logger.debug(f"Start date: {self.start_date}, End date: {self.end_date}") if self.datepicker.start_date.date() > self.datepicker.end_date.date(): lastmonth = self.datepicker.end_date.date().addDays(-31) msg = f"Start date after end date is not allowed! Setting to {lastmonth.toString()}." diff --git a/src/submissions/tools/__init__.py b/src/submissions/tools/__init__.py index 2951627..ed5857c 100644 --- a/src/submissions/tools/__init__.py +++ b/src/submissions/tools/__init__.py @@ -8,8 +8,9 @@ from json import JSONDecodeError from threading import Thread from inspect import getmembers, isfunction, stack from dateutil.easter import easter +from dateutil.parser import parse from jinja2 import Environment, FileSystemLoader -from logging import handlers +from logging import handlers, Logger from pathlib import Path from sqlalchemy.orm import Session, InstrumentedAttribute from sqlalchemy import create_engine, text, MetaData @@ -294,6 +295,7 @@ class GroupWriteRotatingFileHandler(handlers.RotatingFileHandler): class CustomFormatter(logging.Formatter): + class bcolors: HEADER = '\033[95m' OKBLUE = '\033[94m' @@ -339,6 +341,42 @@ class StreamToLogger(object): self.logger.log(self.log_level, line.rstrip()) +class CustomLogger(Logger): + + def __init__(self, name: str = "submissions", level=logging.DEBUG): + super().__init__(name, level) + self.extra_info = None + ch = logging.StreamHandler(stream=sys.stdout) + ch.name = "Stream" + ch.setLevel(self.level) + # NOTE: create formatter and add it to the handlers + ch.setFormatter(CustomFormatter()) + # NOTE: add the handlers to the logger + self.addHandler(ch) + sys.excepthook = self.handle_exception + + def info(self, msg, *args, xtra=None, **kwargs): + extra_info = xtra if xtra is not None else self.extra_info + super().info(msg, *args, extra=extra_info, **kwargs) + + @classmethod + def handle_exception(cls, exc_type, exc_value, exc_traceback): + """ + System won't halt after error, except KeyboardInterrupt + + Args: + exc_value (): + exc_traceback (): + + Returns: + + """ + if issubclass(exc_type, KeyboardInterrupt): + sys.__excepthook__(exc_type, exc_value, exc_traceback) + return + logger.critical("Uncaught exception", exc_info=(exc_type, exc_value, exc_traceback)) + + def setup_logger(verbosity: int = 3): """ Set logger levels using settings. @@ -858,6 +896,35 @@ def check_dictionary_inclusion_equality(listo: List[dict] | dict, dicto: dict) - raise TypeError(f"Unsupported variable: {type(listo)}") +def rectify_query_date(input_date, eod: bool = False) -> str: + """ + Converts input into a datetime string for querying purposes + + Args: + eod (bool, optional): Whether to use max time to indicate end of day. + input_date (): + + Returns: + datetime: properly formated datetime + """ + match input_date: + case datetime(): + output_date = input_date.strftime("%Y-%m-%d %H:%M:%S") + case date(): + if eod: + addition_time = datetime.max.time() + else: + addition_time = datetime.min.time() + output_date = datetime.combine(input_date, addition_time) + output_date = output_date.strftime("%Y-%m-%d %H:%M:%S") + case int(): + output_date = datetime.fromordinal( + datetime(1900, 1, 1).toordinal() + input_date - 2).date().strftime("%Y-%m-%d %H:%M:%S") + case _: + output_date = parse(input_date).strftime("%Y-%m-%d %H:%M:%S") + return output_date + + class classproperty(property): def __get__(self, owner_self, owner_cls): return self.fget(owner_cls)