Refined query-by-date to use start/end of day times to improve accuracy.

CHANGELOG.md

@@ -1,3 +1,7 @@
+# 202504.02
+
+- Refined query-by-date to use start/end of day times to improve accuracy.
+
 # 202504.01
 
 - Added in method to backup submissions to xlsx (partly).

TODO.md

@@ -1,3 +1,6 @@
+- [x] Apply below fix to all other date-based queries.
+- [x] Fix Control graph chart bug that excludes today's controls.
+- [x] Convert logger to a custom class.
 - [x] Change "Manage Organizations" to the Pydantic version.
 - [x] Can my "to_dict", "to_sub_dict", "to_pydantic" methods be rewritten as properties?
 - [ ] Stop displacing date on Irida controls and just do what Turnaround time does.

src/submissions/__main__.py

@@ -1,12 +1,15 @@
+import logging
 import sys, os
-from tools import ctx, setup_logger, check_if_app
+from tools import ctx, check_if_app, CustomLogger
 
 # NOTE: environment variable must be set to enable qtwebengine in network path
 if check_if_app():
     os.environ['QTWEBENGINE_DISABLE_SANDBOX'] = "1"
 
 # NOTE: setup custom logger
-logger = setup_logger(verbosity=3)
+logging.setLoggerClass(CustomLogger)
+# logger = logging.getLogger("submissions")
+# logger = setup_logger(verbosity=3)
 
 from PyQt6.QtWidgets import QApplication
 from frontend.widgets.app import App

src/submissions/backend/db/models/controls.py

@@ -12,7 +12,8 @@ from sqlalchemy.orm import relationship, Query, validates
 import logging, re
 from operator import itemgetter
 from . import BaseClass
-from tools import setup_lookup, report_result, Result, Report, Settings, get_unique_values_in_df_column, super_splitter
+from tools import setup_lookup, report_result, Result, Report, Settings, get_unique_values_in_df_column, super_splitter, \
+    rectify_query_date
 from datetime import date, datetime, timedelta
 from typing import List, Literal, Tuple, Generator
 from dateutil.parser import parse
@@ -149,8 +150,8 @@ class Control(BaseClass):
     def query(cls,
               submissiontype: str | None = None,
               subtype: str | None = None,
-              start_date: date | str | int | None = None,
+              start_date: date | datetime | str | int | None = None,
-              end_date: date | str | int | None = None,
+              end_date: date | datetime | str | int | None = None,
               name: str | None = None,
               limit: int = 0, **kwargs
               ) -> Control | List[Control]:
@@ -201,22 +202,30 @@ class Control(BaseClass):
             logger.warning(f"End date with no start date, using 90 days ago.")
             start_date = date.today() - timedelta(days=90)
         if start_date is not None:
-            match start_date:
+            # match start_date:
-                case date():
+            #     case datetime():
-                    start_date = start_date.strftime("%Y-%m-%d")
+            #         start_date = start_date.strftime("%Y-%m-%d %H:%M:%S")
-                case int():
+            #     case date():
-                    start_date = datetime.fromordinal(
+            #         start_date = datetime.combine(start_date, datetime.min.time())
-                        datetime(1900, 1, 1).toordinal() + start_date - 2).date().strftime("%Y-%m-%d")
+            #         start_date = start_date.strftime("%Y-%m-%d %H:%M:%S")
-                case _:
+            #     case int():
-                    start_date = parse(start_date).strftime("%Y-%m-%d")
+            #         start_date = datetime.fromordinal(
-            match end_date:
+            #             datetime(1900, 1, 1).toordinal() + start_date - 2).date().strftime("%Y-%m-%d %H:%M:%S")
-                case date():
+            #     case _:
-                    end_date = end_date.strftime("%Y-%m-%d")
+            #         start_date = parse(start_date).strftime("%Y-%m-%d %H:%M:%S")
-                case int():
+            start_date = rectify_query_date(start_date)
-                    end_date = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + end_date - 2).date().strftime(
+            end_date = rectify_query_date(end_date, eod=True)
-                        "%Y-%m-%d")
+            # match end_date:
-                case _:
+            #     case datetime():
-                    end_date = parse(end_date).strftime("%Y-%m-%d")
+            #         end_date = end_date.strftime("%Y-%m-%d %H:%M:%S")
+            #     case date():
+            #         end_date = datetime.combine(end_date, datetime.max.time())
+            #         end_date = end_date.strftime("%Y-%m-%d %H:%M:%S")
+            #     case int():
+            #         end_date = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + end_date - 2).date().strftime(
+            #             "%Y-%m-%d %H:%M:%S")
+            #     case _:
+            #         end_date = parse(end_date).strftime("%Y-%m-%d %H:%M:%S")
             query = query.filter(cls.submitted_date.between(start_date, end_date))
         match name:
             case str():

src/submissions/backend/db/models/submissions.py

@@ -27,7 +27,7 @@ from sqlite3 import OperationalError as SQLOperationalError, IntegrityError as S
 from openpyxl import Workbook
 from openpyxl.drawing.image import Image as OpenpyxlImage
 from tools import row_map, setup_lookup, jinja_template_loading, rreplace, row_keys, check_key_or_attr, Result, Report, \
-    report_result, create_holidays_for_year, check_dictionary_inclusion_equality
+    report_result, create_holidays_for_year, check_dictionary_inclusion_equality, rectify_query_date
 from datetime import datetime, date, timedelta
 from typing import List, Any, Tuple, Literal, Generator, Type
 from dateutil.parser import parse
@@ -1152,35 +1152,37 @@ class BasicSubmission(BaseClass, LogMixin):
             start_date = cls.__database_session__.query(cls, func.min(cls.submitted_date)).first()[1]
             logger.warning(f"End date with no start date, using first submission date: {start_date}")
         if start_date is not None:
-            match start_date:
+            # match start_date:
-                case date():
+            #     case date():
-                    pass
+            #         pass
-                case datetime():
+            #     case datetime():
-                    start_date = start_date.date()
+            #         start_date = start_date.date()
-                case int():
+            #     case int():
-                    start_date = datetime.fromordinal(
+            #         start_date = datetime.fromordinal(
-                        datetime(1900, 1, 1).toordinal() + start_date - 2).date()
+            #             datetime(1900, 1, 1).toordinal() + start_date - 2).date()
-                case _:
+            #     case _:
-                    start_date = parse(start_date).date()
+            #         start_date = parse(start_date).date()
-            # start_date = start_date.strftime("%Y-%m-%d")
+            # # start_date = start_date.strftime("%Y-%m-%d")
-            match end_date:
+            # match end_date:
-                case date():
+            #     case date():
-                    pass
+            #         pass
-                case datetime():
+            #     case datetime():
-                    end_date = end_date  # + timedelta(days=1)
+            #         end_date = end_date  # + timedelta(days=1)
-                    # pass
+            #         # pass
-                case int():
+            #     case int():
-                    end_date = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + end_date - 2).date()  # \
+            #         end_date = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + end_date - 2).date()  # \
-                    # + timedelta(days=1)
+            #         # + timedelta(days=1)
-                case _:
+            #     case _:
-                    end_date = parse(end_date).date()  # + timedelta(days=1)
+            #         end_date = parse(end_date).date()  # + timedelta(days=1)
-            # end_date = end_date.strftime("%Y-%m-%d")
+            # # end_date = end_date.strftime("%Y-%m-%d")
-            start_date = datetime.combine(start_date, datetime.min.time()).strftime("%Y-%m-%d %H:%M:%S.%f")
+            # start_date = datetime.combine(start_date, datetime.min.time()).strftime("%Y-%m-%d %H:%M:%S.%f")
-            end_date = datetime.combine(end_date, datetime.max.time()).strftime("%Y-%m-%d %H:%M:%S.%f")
+            # end_date = datetime.combine(end_date, datetime.max.time()).strftime("%Y-%m-%d %H:%M:%S.%f")
-            # if start_date == end_date:
+            # # if start_date == end_date:
-            #     start_date = start_date.strftime("%Y-%m-%d %H:%M:%S.%f")
+            # #     start_date = start_date.strftime("%Y-%m-%d %H:%M:%S.%f")
-            #     query = query.filter(model.submitted_date == start_date)
+            # #     query = query.filter(model.submitted_date == start_date)
-            # else:
+            # # else:
+            start_date = rectify_query_date(start_date)
+            end_date = rectify_query_date(end_date, eod=True)
             query = query.filter(model.submitted_date.between(start_date, end_date))
         # NOTE: by reagent (for some reason)
         match reagent:

src/submissions/frontend/widgets/controls_chart.py

@@ -108,6 +108,7 @@ class ControlsViewer(InfoPane):
             parent=self,
             months=months
         )
+        logger.debug(f"Chart settings: {chart_settings}")
         self.fig = self.archetype.instance_class.make_chart(chart_settings=chart_settings, parent=self, ctx=self.app.ctx)
         self.report_obj = ChartReportMaker(df=self.fig.df, sheet_name=self.archetype.name)
         if issubclass(self.fig.__class__, CustomFigure):

src/submissions/frontend/widgets/info_tab.py

@@ -1,7 +1,7 @@
 """
 A pane to show info e.g. cost reports and turnaround times.
 """
-from datetime import date
+from datetime import date, datetime
 from PyQt6.QtCore import QSignalBlocker
 from PyQt6.QtWebEngineWidgets import QWebEngineView
 from PyQt6.QtWidgets import QWidget, QGridLayout
@@ -32,8 +32,11 @@ class InfoPane(QWidget):
     @report_result
     def update_data(self, *args, **kwargs):
         report = Report()
+        # self.start_date = self.datepicker.start_date.date().toPyDate()
+        # self.end_date = self.datepicker.end_date.date().toPyDate()
         self.start_date = self.datepicker.start_date.date().toPyDate()
         self.end_date = self.datepicker.end_date.date().toPyDate()
+        logger.debug(f"Start date: {self.start_date}, End date: {self.end_date}")
         if self.datepicker.start_date.date() > self.datepicker.end_date.date():
             lastmonth = self.datepicker.end_date.date().addDays(-31)
             msg = f"Start date after end date is not allowed! Setting to {lastmonth.toString()}."

src/submissions/tools/__init__.py

@@ -8,8 +8,9 @@ from json import JSONDecodeError
 from threading import Thread
 from inspect import getmembers, isfunction, stack
 from dateutil.easter import easter
+from dateutil.parser import parse
 from jinja2 import Environment, FileSystemLoader
-from logging import handlers
+from logging import handlers, Logger
 from pathlib import Path
 from sqlalchemy.orm import Session, InstrumentedAttribute
 from sqlalchemy import create_engine, text, MetaData
@@ -294,6 +295,7 @@ class GroupWriteRotatingFileHandler(handlers.RotatingFileHandler):
 
 
 class CustomFormatter(logging.Formatter):
+
     class bcolors:
         HEADER = '\033[95m'
         OKBLUE = '\033[94m'
@@ -339,6 +341,42 @@ class StreamToLogger(object):
             self.logger.log(self.log_level, line.rstrip())
 
 
+class CustomLogger(Logger):
+
+    def __init__(self, name: str = "submissions", level=logging.DEBUG):
+        super().__init__(name, level)
+        self.extra_info = None
+        ch = logging.StreamHandler(stream=sys.stdout)
+        ch.name = "Stream"
+        ch.setLevel(self.level)
+        # NOTE: create formatter and add it to the handlers
+        ch.setFormatter(CustomFormatter())
+        # NOTE: add the handlers to the logger
+        self.addHandler(ch)
+        sys.excepthook = self.handle_exception
+
+    def info(self, msg, *args, xtra=None, **kwargs):
+        extra_info = xtra if xtra is not None else self.extra_info
+        super().info(msg, *args, extra=extra_info, **kwargs)
+
+    @classmethod
+    def handle_exception(cls, exc_type, exc_value, exc_traceback):
+        """
+        System won't halt after error, except KeyboardInterrupt
+
+        Args:
+            exc_value ():
+            exc_traceback ():
+
+        Returns:
+
+        """
+        if issubclass(exc_type, KeyboardInterrupt):
+            sys.__excepthook__(exc_type, exc_value, exc_traceback)
+            return
+        logger.critical("Uncaught exception", exc_info=(exc_type, exc_value, exc_traceback))
+
+
 def setup_logger(verbosity: int = 3):
     """
     Set logger levels using settings.
@@ -858,6 +896,35 @@ def check_dictionary_inclusion_equality(listo: List[dict] | dict, dicto: dict) -
         raise TypeError(f"Unsupported variable: {type(listo)}")
 
 
+def rectify_query_date(input_date, eod: bool = False) -> str:
+    """
+    Converts input into a datetime string for querying purposes
+
+    Args:
+        eod (bool, optional): Whether to use max time to indicate end of day.
+        input_date ():
+
+    Returns:
+        datetime: properly formated datetime
+    """
+    match input_date:
+        case datetime():
+            output_date = input_date.strftime("%Y-%m-%d %H:%M:%S")
+        case date():
+            if eod:
+                addition_time = datetime.max.time()
+            else:
+                addition_time = datetime.min.time()
+            output_date = datetime.combine(input_date, addition_time)
+            output_date = output_date.strftime("%Y-%m-%d %H:%M:%S")
+        case int():
+            output_date = datetime.fromordinal(
+                datetime(1900, 1, 1).toordinal() + input_date - 2).date().strftime("%Y-%m-%d %H:%M:%S")
+        case _:
+            output_date = parse(input_date).strftime("%Y-%m-%d %H:%M:%S")
+    return output_date
+
+
 class classproperty(property):
     def __get__(self, owner_self, owner_cls):
         return self.fget(owner_cls)