''' Contains miscellaenous functions used by both frontend and backend. ''' from __future__ import annotations import json from pathlib import Path import numpy as np import logging, re, yaml, sys, os, stat, platform, getpass, inspect, csv import pandas as pd from jinja2 import Environment, FileSystemLoader from logging import handlers from pathlib import Path from sqlalchemy.orm import Session from sqlalchemy import create_engine, text from pydantic import field_validator, BaseModel, Field from pydantic_settings import BaseSettings, SettingsConfigDict from typing import Any, Tuple, Literal, List from PyQt6.QtGui import QTextDocument, QPageSize from PyQt6.QtWebEngineWidgets import QWebEngineView from openpyxl.worksheet.worksheet import Worksheet # from PyQt6 import QtPrintSupport, QtCore, QtWebEngineWidgets from PyQt6.QtPrintSupport import QPrinter logger = logging.getLogger(f"submissions.{__name__}") package_dir = Path(__file__).parents[2].resolve() logger.debug(f"Package dir: {package_dir}") if platform.system() == "Windows": os_config_dir = "AppData/local" print(f"Got platform Windows, config_dir: {os_config_dir}") else: os_config_dir = ".config" print(f"Got platform other, config_dir: {os_config_dir}") main_aux_dir = Path.home().joinpath(f"{os_config_dir}/submissions") CONFIGDIR = main_aux_dir.joinpath("config") LOGDIR = main_aux_dir.joinpath("logs") row_map = {1: "A", 2: "B", 3: "C", 4: "D", 5: "E", 6: "F", 7: "G", 8: "H"} row_keys = {v: k for k, v in row_map.items()} def check_not_nan(cell_contents) -> bool: """ Check to ensure excel sheet cell contents are not blank. Args: cell_contents (_type_): The contents of the cell in question. Returns: bool: True if cell has value, else, false. """ # check for nan as a string first exclude = ['unnamed:', 'blank', 'void'] try: # if "Unnamed:" in cell_contents or "blank" in cell_contents.lower(): if cell_contents.lower() in exclude: cell_contents = np.nan cell_contents = cell_contents.lower() except (TypeError, AttributeError): pass try: if np.isnat(cell_contents): cell_contents = np.nan except TypeError as e: pass if cell_contents == "nat": cell_contents = np.nan if cell_contents == 'nan': cell_contents = np.nan if cell_contents == None: cell_contents = np.nan if str(cell_contents).lower() == "none": cell_contents = np.nan try: if pd.isnull(cell_contents): cell_contents = np.nan except ValueError: pass try: return not np.isnan(cell_contents) except TypeError: return True except Exception as e: logger.debug(f"Check encountered unknown error: {type(e).__name__} - {e}") return False def convert_nans_to_nones(input_str) -> str | None: """ Get rid of various "nan", "NAN", "NaN", etc/ Args: input_str (str): input string Returns: str: _description_ """ # logger.debug(f"Input value of: {input_str}") if check_not_nan(input_str): return input_str return None def is_missing(value: Any) -> Tuple[Any, bool]: if check_not_nan(value): return value, False else: return convert_nans_to_nones(value), True def check_regex_match(pattern: str, check: str) -> bool: """ Determines if a pattern matches a str Args: pattern (str): regex pattern string check (str): string to be checked Returns: bool: match found? """ try: return bool(re.match(fr"{pattern}", check)) except TypeError: return False def get_first_blank_df_row(df: pd.DataFrame) -> int: """ For some reason I need a whole function for this. Args: df (pd.DataFrame): Input dataframe. Returns: int: Index of the row after the last used row. """ return df.shape[0] + 1 # Settings class Settings(BaseSettings, extra="allow"): """ Pydantic model to hold settings Raises: FileNotFoundError: Error if database not found. """ directory_path: Path database_path: Path | str | None = None backup_path: Path | str | None = None # super_users: list|None = None # power_users: list|None = None # rerun_regex: str submission_types: dict | None = None database_session: Session | None = None package: Any | None = None model_config = SettingsConfigDict(env_file_encoding='utf-8') @field_validator('backup_path', mode="before") @classmethod def set_backup_path(cls, value, values): match value: case str(): value = Path(value) case None: value = values.data['directory_path'].joinpath("Database backups") if not value.exists(): value.mkdir(parents=True) # metadata.backup_path = value return value @field_validator('directory_path', mode="before") @classmethod def ensure_directory_exists(cls, value): if isinstance(value, str): value = Path(value) if not value.exists(): value = Path().home() # metadata.directory_path = value return value @field_validator('database_path', mode="before") @classmethod def ensure_database_exists(cls, value, values): if value == ":memory:": return value match value: case str(): value = Path(value) case None: value = values.data['directory_path'].joinpath("submissions.db") if value.exists(): return value else: raise FileNotFoundError(f"Couldn't find database at {value}") @field_validator('database_session', mode="before") @classmethod def create_database_session(cls, value, values): if value != None: return value else: database_path = values.data['database_path'] if database_path == None: # check in user's .submissions directory for submissions.db if Path.home().joinpath(".submissions", "submissions.db").exists(): database_path = Path.home().joinpath(".submissions", "submissions.db") # finally, look in the local dir else: database_path = package_dir.joinpath("submissions.db") else: if database_path == ":memory:": pass # check if user defined path is directory elif database_path.is_dir(): database_path = database_path.joinpath("submissions.db") # check if user defined path is a file elif database_path.is_file(): database_path = database_path else: raise FileNotFoundError("No database file found. Exiting program.") logger.debug(f"Using {database_path} for database file.") engine = create_engine(f"sqlite:///{database_path}") #, echo=True, future=True) session = Session(engine) # metadata.session = session return session @field_validator('package', mode="before") @classmethod def import_package(cls, value): import __init__ as package if value == None: return package def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.set_from_db(db_path=kwargs['database_path']) def set_from_db(self, db_path: Path): if 'pytest' in sys.modules: config_items = dict(power_users=['lwark', 'styson', 'ruwang']) else: session = Session(create_engine(f"sqlite:///{db_path}")) config_items = session.execute(text("SELECT * FROM _configitem")).all() session.close() config_items = {item[1]: json.loads(item[2]) for item in config_items} for k, v in config_items.items(): if not hasattr(self, k): self.__setattr__(k, v) def get_config(settings_path: Path | str | None = None) -> Settings: """ Get configuration settings from path or default if blank. Args: settings_path (Path | str | None, optional): Path to config.yml Defaults to None. override (dict | None, optional): dictionary of settings to be used instead of file. Defaults to None. Returns: Settings: Pydantic settings object """ # logger.debug(f"Creating settings...") if isinstance(settings_path, str): settings_path = Path(settings_path) # custom pyyaml constructor to join fields def join(loader, node): seq = loader.construct_sequence(node) return ''.join([str(i) for i in seq]) # register the tag handler yaml.add_constructor('!join', join) # make directories try: CONFIGDIR.mkdir(parents=True) except FileExistsError: logger.warning(f"Config directory {CONFIGDIR} already exists.") try: LOGDIR.mkdir(parents=True) except FileExistsError: logger.warning(f"Logging directory {LOGDIR} already exists.") # NOTE: if user hasn't defined config path in cli args if settings_path == None: # NOTE: Check user .config/submissions directory if CONFIGDIR.joinpath("config.yml").exists(): settings_path = CONFIGDIR.joinpath("config.yml") # NOTE: Check user .submissions directory elif Path.home().joinpath(".submissions", "config.yml").exists(): settings_path = Path.home().joinpath(".submissions", "config.yml") # NOTE: finally look in the local config else: if check_if_app(): settings_path = Path(sys._MEIPASS).joinpath("files", "config.yml") else: settings_path = package_dir.joinpath('config.yml') with open(settings_path, "r") as dset: default_settings = yaml.load(dset, Loader=yaml.Loader) # NOTE: Tell program we need to copy the config.yml to the user directory # NOTE: copy settings to config directory return Settings(**copy_settings(settings_path=CONFIGDIR.joinpath("config.yml"), settings=default_settings)) else: # NOTE: check if user defined path is directory if settings_path.is_dir(): settings_path = settings_path.joinpath("config.yml") # NOTE: check if user defined path is file elif settings_path.is_file(): settings_path = settings_path else: logger.error("No config.yml file found. Writing to directory.") with open(settings_path, "r") as dset: default_settings = yaml.load(dset, Loader=yaml.Loader) return Settings(**copy_settings(settings_path=settings_path, settings=default_settings)) # logger.debug(f"Using {settings_path} for config file.") with open(settings_path, "r") as stream: settings = yaml.load(stream, Loader=yaml.Loader) return Settings(**settings) # Logging formatters class GroupWriteRotatingFileHandler(handlers.RotatingFileHandler): def doRollover(self): """ Override base class method to make the new log file group writable. """ # Rotate the file first. handlers.RotatingFileHandler.doRollover(self) # Add group write to the current permissions. currMode = os.stat(self.baseFilename).st_mode os.chmod(self.baseFilename, currMode | stat.S_IWGRP) def _open(self): prevumask = os.umask(0o002) rtv = handlers.RotatingFileHandler._open(self) os.umask(prevumask) return rtv class CustomFormatter(logging.Formatter): class bcolors: HEADER = '\033[95m' OKBLUE = '\033[94m' OKCYAN = '\033[96m' OKGREEN = '\033[92m' WARNING = '\033[93m' FAIL = '\033[91m' ENDC = '\033[0m' BOLD = '\033[1m' UNDERLINE = '\033[4m' format = "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(message)s" FORMATS = { logging.DEBUG: bcolors.ENDC + format + bcolors.ENDC, logging.INFO: bcolors.ENDC + format + bcolors.ENDC, logging.WARNING: bcolors.WARNING + format + bcolors.ENDC, logging.ERROR: bcolors.FAIL + format + bcolors.ENDC, logging.CRITICAL: bcolors.FAIL + format + bcolors.ENDC } def format(self, record): log_fmt = self.FORMATS.get(record.levelno) formatter = logging.Formatter(log_fmt) return formatter.format(record) class StreamToLogger(object): """ Fake file-like stream object that redirects writes to a logger instance. """ def __init__(self, logger, log_level=logging.INFO): self.logger = logger self.log_level = log_level self.linebuf = '' def write(self, buf): for line in buf.rstrip().splitlines(): self.logger.log(self.log_level, line.rstrip()) def setup_logger(verbosity: int = 3): """ Set logger levels using settings. Args: verbosity (int, optional): Level of verbosity desired 3 is highest. Defaults to 3. Returns: logger: logger object """ logger = logging.getLogger("submissions") logger.setLevel(logging.DEBUG) # NOTE: create file handler which logs even debug messages try: Path(LOGDIR).mkdir(parents=True) except FileExistsError: logger.warning(f"Logging directory {LOGDIR} already exists.") # NOTE: logging to file turned off due to repeated permission errors # fh = GroupWriteRotatingFileHandler(LOGDIR.joinpath('submissions.log'), mode='a', maxBytes=100000, backupCount=3, encoding=None, delay=False) # file logging will always be debug # fh.setLevel(logging.DEBUG) # fh.name = "File" # NOTE: create console handler with a higher log level # NOTE: create custom logger with STERR -> log ch = logging.StreamHandler(stream=sys.stdout) # NOTE: set looging level based on verbosity match verbosity: case 3: ch.setLevel(logging.DEBUG) case 2: ch.setLevel(logging.INFO) case 1: ch.setLevel(logging.WARNING) ch.name = "Stream" # NOTE: create formatter and add it to the handlers formatter = CustomFormatter() # fh.setFormatter(formatter) ch.setFormatter(formatter) # NOTE: add the handlers to the logger # logger.addHandler(fh) logger.addHandler(ch) # NOTE: Output exception and traceback to logger def handle_exception(exc_type, exc_value, exc_traceback): if issubclass(exc_type, KeyboardInterrupt): sys.__excepthook__(exc_type, exc_value, exc_traceback) return logger.critical("Uncaught exception", exc_info=(exc_type, exc_value, exc_traceback)) sys.excepthook = handle_exception return logger def copy_settings(settings_path: Path, settings: dict) -> dict: """ copies relevant settings dictionary from the default config.yml to a new directory Args: settings_path (Path): path to write the file to settings (dict): settings dictionary obtained from default config.yml Returns: dict: output dictionary for use in first run """ # NOTE: if the current user is not a superuser remove the superusers entry if not getpass.getuser() in settings['super_users']: del settings['super_users'] if not getpass.getuser() in settings['power_users']: del settings['power_users'] if not settings_path.exists(): with open(settings_path, 'w') as f: yaml.dump(settings, f) return settings def jinja_template_loading() -> Environment: """ Returns jinja2 template environment. Returns: _type_: _description_ """ # NOTE: determine if pyinstaller launcher is being used if check_if_app(): loader_path = Path(sys._MEIPASS).joinpath("files", "templates") else: loader_path = Path(__file__).parent.joinpath('templates').absolute() #.__str__() # NOTE: jinja template loading loader = FileSystemLoader(loader_path) env = Environment(loader=loader) env.globals['STATIC_PREFIX'] = loader_path.joinpath("static", "css") return env def check_if_app() -> bool: """ Checks if the program is running from pyinstaller compiled Returns: bool: True if running from pyinstaller. Else False. """ if getattr(sys, 'frozen', False): return True else: return False def convert_well_to_row_column(input_str: str) -> Tuple[int, int]: """ Converts typical alphanumeric (i.e. "A2") to row, column Args: input_str (str): Input string. Ex. "A2" Returns: Tuple[int, int]: row, column """ row_keys = {v: k for k, v in row_map.items()} try: row = int(row_keys[input_str[0].upper()]) column = int(input_str[1:]) except IndexError: return None, None return row, column def setup_lookup(func): """ Checks to make sure all args are allowed Args: func (_type_): wrapped function """ def wrapper(*args, **kwargs): sanitized_kwargs = {} for k, v in locals()['kwargs'].items(): if isinstance(v, dict): try: sanitized_kwargs[k] = v['value'] except KeyError: raise ValueError("Could not sanitize dictionary in query. Make sure you parse it first.") elif v is not None: sanitized_kwargs[k] = v return func(*args, **sanitized_kwargs) return wrapper class Result(BaseModel): owner: str = Field(default="", validate_default=True) code: int = Field(default=0) msg: str status: Literal["NoIcon", "Question", "Information", "Warning", "Critical"] = Field(default="NoIcon") @field_validator('status', mode='before') @classmethod def to_title(cls, value: str): if value.lower().replace(" ", "") == "noicon": return "NoIcon" else: return value.title() def __repr__(self) -> str: return f"Result({self.owner})" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.owner = inspect.stack()[1].function def report(self): from frontend.widgets.misc import AlertPop return AlertPop(message=self.msg, status=self.status, owner=self.owner) class Report(BaseModel): results: List[Result] = Field(default=[]) def __repr__(self): return f"Report(result_count:{len(self.results)})" def add_result(self, result: Result | Report | None): match result: case Result(): logger.debug(f"Adding {result} to results.") try: self.results.append(result) except AttributeError: logger.error(f"Problem adding result.") case Report(): # logger.debug(f"Adding all results in report to new report") for res in result.results: logger.debug(f"Adding {res} from to results.") self.results.append(res) case _: logger.error(f"Unknown variable type: {type(result)}") def rreplace(s, old, new): return (s[::-1].replace(old[::-1], new[::-1], 1))[::-1] def html_to_pdf(html, output_file: Path | str): if isinstance(output_file, str): output_file = Path(output_file) document = QWebEngineView() document.setHtml(html) printer = QPrinter(QPrinter.PrinterMode.HighResolution) printer.setOutputFormat(QPrinter.OutputFormat.PdfFormat) printer.setOutputFileName(output_file.absolute().__str__()) printer.setPageSize(QPageSize(QPageSize.PageSizeId.A4)) document.print(printer) def remove_key_from_list_of_dicts(input: list, key: str): for item in input: del item[key] return input def workbook_2_csv(worksheet: Worksheet, filename: Path): with open(filename, 'w', newline="") as f: c = csv.writer(f) for r in worksheet.rows: c.writerow([cell.value for cell in r]) ctx = get_config(None) def is_power_user() -> bool: try: check = getpass.getuser() in ctx.power_users except: check = False return check def check_authorization(func): """ Decorator to check if user is authorized to access function Args: func (_type_): Function to be used. """ def wrapper(*args, **kwargs): logger.debug(f"Checking authorization") if is_power_user(): return func(*args, **kwargs) else: logger.error(f"User {getpass.getuser()} is not authorized for this function.") return dict(code=1, message="This user does not have permission for this function.", status="warning") return wrapper