landon/Submissions-App
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refined query-by-date to use start/end of day times to improve accuracy.

Browse Source
This commit is contained in:
lwark
2025-04-07 08:37:00 -05:00
parent ac85be6414
commit bef2e8cdf0
8 changed files with 145 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@@ -1,3 +1,7 @@
# 202504.02
- Refined query-by-date to use start/end of day times to improve accuracy.
# 202504.01
- Added in method to backup submissions to xlsx (partly).

3
TODO.md
View File

@@ -1,3 +1,6 @@
- [x] Apply below fix to all other date-based queries.
- [x] Fix Control graph chart bug that excludes today's controls.
- [x] Convert logger to a custom class.
- [x] Change "Manage Organizations" to the Pydantic version.
- [x] Can my "to_dict", "to_sub_dict", "to_pydantic" methods be rewritten as properties?
- [ ] Stop displacing date on Irida controls and just do what Turnaround time does.

7
src/submissions/__main__.py
View File

@@ -1,12 +1,15 @@
import logging
import sys, os
from tools import ctx, setup_logger, check_if_app
from tools import ctx, check_if_app, CustomLogger
# NOTE: environment variable must be set to enable qtwebengine in network path
if check_if_app():
os.environ['QTWEBENGINE_DISABLE_SANDBOX'] = "1"
# NOTE: setup custom logger
logger = setup_logger(verbosity=3)
logging.setLoggerClass(CustomLogger)
# logger = logging.getLogger("submissions")
# logger = setup_logger(verbosity=3)
from PyQt6.QtWidgets import QApplication
from frontend.widgets.app import App

47
src/submissions/backend/db/models/controls.py
View File

@@ -12,7 +12,8 @@ from sqlalchemy.orm import relationship, Query, validates
import logging, re
from operator import itemgetter
from . import BaseClass
from tools import setup_lookup, report_result, Result, Report, Settings, get_unique_values_in_df_column, super_splitter
from tools import setup_lookup, report_result, Result, Report, Settings, get_unique_values_in_df_column, super_splitter, \
rectify_query_date
from datetime import date, datetime, timedelta
from typing import List, Literal, Tuple, Generator
from dateutil.parser import parse
@@ -149,8 +150,8 @@ class Control(BaseClass):
def query(cls,
submissiontype: str | None = None,
subtype: str | None = None,
start_date: date | str | int | None = None,
end_date: date | str | int | None = None,
start_date: date | datetime | str | int | None = None,
end_date: date | datetime | str | int | None = None,
name: str | None = None,
limit: int = 0, **kwargs
) -> Control | List[Control]:
@@ -201,22 +202,30 @@ class Control(BaseClass):
logger.warning(f"End date with no start date, using 90 days ago.")
start_date = date.today() - timedelta(days=90)
if start_date is not None:
match start_date:
case date():
start_date = start_date.strftime("%Y-%m-%d")
case int():
start_date = datetime.fromordinal(
datetime(1900, 1, 1).toordinal() + start_date - 2).date().strftime("%Y-%m-%d")
case _:
start_date = parse(start_date).strftime("%Y-%m-%d")
match end_date:
case date():
end_date = end_date.strftime("%Y-%m-%d")
case int():
end_date = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + end_date - 2).date().strftime(
"%Y-%m-%d")
case _:
end_date = parse(end_date).strftime("%Y-%m-%d")
# match start_date:
# case datetime():
# start_date = start_date.strftime("%Y-%m-%d %H:%M:%S")
# case date():
# start_date = datetime.combine(start_date, datetime.min.time())
# start_date = start_date.strftime("%Y-%m-%d %H:%M:%S")
# case int():
# start_date = datetime.fromordinal(
# datetime(1900, 1, 1).toordinal() + start_date - 2).date().strftime("%Y-%m-%d %H:%M:%S")
# case _:
# start_date = parse(start_date).strftime("%Y-%m-%d %H:%M:%S")
start_date = rectify_query_date(start_date)
end_date = rectify_query_date(end_date, eod=True)
# match end_date:
# case datetime():
# end_date = end_date.strftime("%Y-%m-%d %H:%M:%S")
# case date():
# end_date = datetime.combine(end_date, datetime.max.time())
# end_date = end_date.strftime("%Y-%m-%d %H:%M:%S")
# case int():
# end_date = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + end_date - 2).date().strftime(
# "%Y-%m-%d %H:%M:%S")
# case _:
# end_date = parse(end_date).strftime("%Y-%m-%d %H:%M:%S")
query = query.filter(cls.submitted_date.between(start_date, end_date))
match name:
case str():

60
src/submissions/backend/db/models/submissions.py
View File

@@ -27,7 +27,7 @@ from sqlite3 import OperationalError as SQLOperationalError, IntegrityError as S
from openpyxl import Workbook
from openpyxl.drawing.image import Image as OpenpyxlImage
from tools import row_map, setup_lookup, jinja_template_loading, rreplace, row_keys, check_key_or_attr, Result, Report, \
report_result, create_holidays_for_year, check_dictionary_inclusion_equality
report_result, create_holidays_for_year, check_dictionary_inclusion_equality, rectify_query_date
from datetime import datetime, date, timedelta
from typing import List, Any, Tuple, Literal, Generator, Type
from dateutil.parser import parse
@@ -1152,35 +1152,37 @@ class BasicSubmission(BaseClass, LogMixin):
start_date = cls.__database_session__.query(cls, func.min(cls.submitted_date)).first()[1]
logger.warning(f"End date with no start date, using first submission date: {start_date}")
if start_date is not None:
match start_date:
case date():
pass
case datetime():
start_date = start_date.date()
case int():
start_date = datetime.fromordinal(
datetime(1900, 1, 1).toordinal() + start_date - 2).date()
case _:
start_date = parse(start_date).date()
# start_date = start_date.strftime("%Y-%m-%d")
match end_date:
case date():
pass
case datetime():
end_date = end_date # + timedelta(days=1)
# match start_date:
# case date():
# pass
case int():
end_date = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + end_date - 2).date() # \
# + timedelta(days=1)
case _:
end_date = parse(end_date).date() # + timedelta(days=1)
# end_date = end_date.strftime("%Y-%m-%d")
start_date = datetime.combine(start_date, datetime.min.time()).strftime("%Y-%m-%d %H:%M:%S.%f")
end_date = datetime.combine(end_date, datetime.max.time()).strftime("%Y-%m-%d %H:%M:%S.%f")
# if start_date == end_date:
# start_date = start_date.strftime("%Y-%m-%d %H:%M:%S.%f")
# query = query.filter(model.submitted_date == start_date)
# else:
# case datetime():
# start_date = start_date.date()
# case int():
# start_date = datetime.fromordinal(
# datetime(1900, 1, 1).toordinal() + start_date - 2).date()
# case _:
# start_date = parse(start_date).date()
# # start_date = start_date.strftime("%Y-%m-%d")
# match end_date:
# case date():
# pass
# case datetime():
# end_date = end_date # + timedelta(days=1)
# # pass
# case int():
# end_date = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + end_date - 2).date() # \
# # + timedelta(days=1)
# case _:
# end_date = parse(end_date).date() # + timedelta(days=1)
# # end_date = end_date.strftime("%Y-%m-%d")
# start_date = datetime.combine(start_date, datetime.min.time()).strftime("%Y-%m-%d %H:%M:%S.%f")
# end_date = datetime.combine(end_date, datetime.max.time()).strftime("%Y-%m-%d %H:%M:%S.%f")
# # if start_date == end_date:
# # start_date = start_date.strftime("%Y-%m-%d %H:%M:%S.%f")
# # query = query.filter(model.submitted_date == start_date)
# # else:
start_date = rectify_query_date(start_date)
end_date = rectify_query_date(end_date, eod=True)
query = query.filter(model.submitted_date.between(start_date, end_date))
# NOTE: by reagent (for some reason)
match reagent:

1
src/submissions/frontend/widgets/controls_chart.py
View File

@@ -108,6 +108,7 @@ class ControlsViewer(InfoPane):
parent=self,
months=months
)
logger.debug(f"Chart settings: {chart_settings}")
self.fig = self.archetype.instance_class.make_chart(chart_settings=chart_settings, parent=self, ctx=self.app.ctx)
self.report_obj = ChartReportMaker(df=self.fig.df, sheet_name=self.archetype.name)
if issubclass(self.fig.__class__, CustomFigure):

5
src/submissions/frontend/widgets/info_tab.py
View File

@@ -1,7 +1,7 @@
"""
A pane to show info e.g. cost reports and turnaround times.
"""
from datetime import date
from datetime import date, datetime
from PyQt6.QtCore import QSignalBlocker
from PyQt6.QtWebEngineWidgets import QWebEngineView
from PyQt6.QtWidgets import QWidget, QGridLayout
@@ -32,8 +32,11 @@ class InfoPane(QWidget):
@report_result
def update_data(self, *args, **kwargs):
report = Report()
# self.start_date = self.datepicker.start_date.date().toPyDate()
# self.end_date = self.datepicker.end_date.date().toPyDate()
self.start_date = self.datepicker.start_date.date().toPyDate()
self.end_date = self.datepicker.end_date.date().toPyDate()
logger.debug(f"Start date: {self.start_date}, End date: {self.end_date}")
if self.datepicker.start_date.date() > self.datepicker.end_date.date():
lastmonth = self.datepicker.end_date.date().addDays(-31)
msg = f"Start date after end date is not allowed! Setting to {lastmonth.toString()}."

69
src/submissions/tools/__init__.py
View File

@@ -8,8 +8,9 @@ from json import JSONDecodeError
from threading import Thread
from inspect import getmembers, isfunction, stack
from dateutil.easter import easter
from dateutil.parser import parse
from jinja2 import Environment, FileSystemLoader
from logging import handlers
from logging import handlers, Logger
from pathlib import Path
from sqlalchemy.orm import Session, InstrumentedAttribute
from sqlalchemy import create_engine, text, MetaData
@@ -294,6 +295,7 @@ class GroupWriteRotatingFileHandler(handlers.RotatingFileHandler):
class CustomFormatter(logging.Formatter):
class bcolors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
@@ -339,6 +341,42 @@ class StreamToLogger(object):
self.logger.log(self.log_level, line.rstrip())
class CustomLogger(Logger):
def __init__(self, name: str = "submissions", level=logging.DEBUG):
super().__init__(name, level)
self.extra_info = None
ch = logging.StreamHandler(stream=sys.stdout)
ch.name = "Stream"
ch.setLevel(self.level)
# NOTE: create formatter and add it to the handlers
ch.setFormatter(CustomFormatter())
# NOTE: add the handlers to the logger
self.addHandler(ch)
sys.excepthook = self.handle_exception
def info(self, msg, *args, xtra=None, **kwargs):
extra_info = xtra if xtra is not None else self.extra_info
super().info(msg, *args, extra=extra_info, **kwargs)
@classmethod
def handle_exception(cls, exc_type, exc_value, exc_traceback):
"""
System won't halt after error, except KeyboardInterrupt
Args:
exc_value ():
exc_traceback ():
Returns:
"""
if issubclass(exc_type, KeyboardInterrupt):
sys.__excepthook__(exc_type, exc_value, exc_traceback)
return
logger.critical("Uncaught exception", exc_info=(exc_type, exc_value, exc_traceback))
def setup_logger(verbosity: int = 3):
"""
Set logger levels using settings.
@@ -858,6 +896,35 @@ def check_dictionary_inclusion_equality(listo: List[dict] | dict, dicto: dict) -
raise TypeError(f"Unsupported variable: {type(listo)}")
def rectify_query_date(input_date, eod: bool = False) -> str:
"""
Converts input into a datetime string for querying purposes
Args:
eod (bool, optional): Whether to use max time to indicate end of day.
input_date ():
Returns:
datetime: properly formated datetime
"""
match input_date:
case datetime():
output_date = input_date.strftime("%Y-%m-%d %H:%M:%S")
case date():
if eod:
addition_time = datetime.max.time()
else:
addition_time = datetime.min.time()
output_date = datetime.combine(input_date, addition_time)
output_date = output_date.strftime("%Y-%m-%d %H:%M:%S")
case int():
output_date = datetime.fromordinal(
datetime(1900, 1, 1).toordinal() + input_date - 2).date().strftime("%Y-%m-%d %H:%M:%S")
case _:
output_date = parse(input_date).strftime("%Y-%m-%d %H:%M:%S")
return output_date
class classproperty(property):
def __get__(self, owner_self, owner_cls):
return self.fget(owner_cls)