""" Contains all models for sqlalchemy """ from __future__ import annotations import sys, logging, json from dateutil.parser import parse from pandas import DataFrame from pydantic import BaseModel from sqlalchemy import Column, INTEGER, String, JSON from sqlalchemy.ext.associationproxy import AssociationProxy from sqlalchemy.orm import DeclarativeMeta, declarative_base, Query, Session, InstrumentedAttribute, ColumnProperty from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.exc import ArgumentError from typing import Any, List, ClassVar from pathlib import Path from sqlalchemy.orm.relationships import _RelationshipDeclared from tools import report_result, list_sort_dict # NOTE: Load testing environment if 'pytest' in sys.modules: sys.path.append(Path(__file__).parents[4].absolute().joinpath("tests").__str__()) # NOTE: For inheriting in LogMixin Base: DeclarativeMeta = declarative_base() logger = logging.getLogger(f"submissions.{__name__}") class BaseClass(Base): """ Abstract class to pass ctx values to all SQLAlchemy objects. """ __abstract__ = True #: NOTE: Will not be added to DB as a table __table_args__ = {'extend_existing': True} #: NOTE Will only add new columns singles = ['id'] # omni_removes = ["id", 'run', "omnigui_class_dict", "omnigui_instance_dict"] # omni_sort = ["name"] # omni_inheritable = [] searchables = [] _misc_info = Column(JSON) def __repr__(self) -> str: try: return f"<{self.__class__.__name__}({self.name})>" except AttributeError: return f"<{self.__class__.__name__}(Name Unavailable)>" @classproperty def aliases(cls) -> List[str]: """ List of other names this class might be known by. Returns: List[str]: List of names """ return [cls.query_alias] @classproperty def query_alias(cls) -> str: """ What to query this class as. Returns: str: query name """ return cls.__name__.lower() @classmethod @declared_attr def __tablename__(cls) -> str: """ Sets table name to lower case class name. Returns: str: lower case class name """ return f"_{cls.__name__.lower()}" @classmethod @declared_attr def __database_session__(cls) -> Session: """ Pull db session from ctx to be used in operations Returns: Session: DB session from ctx settings. """ if 'pytest' not in sys.modules: from tools import ctx else: from test_settings import ctx return ctx.database_session @classmethod @declared_attr def __directory_path__(cls) -> Path: """ Pull directory path from ctx to be used in operations. Returns: Path: Location of the Submissions directory in Settings object """ if 'pytest' not in sys.modules: from tools import ctx else: from test_settings import ctx return ctx.directory_path @classmethod @declared_attr def __backup_path__(cls) -> Path: """ Pull backup directory path from ctx to be used in operations. Returns: Path: Location of the Submissions backup directory in Settings object """ if 'pytest' not in sys.modules: from tools import ctx else: from test_settings import ctx return ctx.backup_path def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._misc_info = dict() @classproperty def jsons(cls) -> List[str]: """ Get list of JSON db columns Returns: List[str]: List of column names """ try: return [item.name for item in cls.__table__.columns if isinstance(item.type, JSON)] except AttributeError: return [] @classproperty def timestamps(cls) -> List[str]: """ Get list of TIMESTAMP columns Returns: List[str]: List of column names """ try: return [item.name for item in cls.__table__.columns if isinstance(item.type, TIMESTAMP)] except AttributeError: return [] @classmethod def get_default_info(cls, *args) -> dict | list | str: """ Returns default info for a model Returns: dict | list | str: Output of key:value dict or single (list, str) desired variable """ # NOTE: singles is a list of fields that need to be limited to 1 result. singles = list(set(cls.singles + BaseClass.singles)) return dict(singles=singles) @classmethod def find_regular_subclass(cls, name: str | None = None) -> Any: """ Args: name (str): name of subclass of interest. Returns: Any: Subclass of this object. """ if name: if " " in name: search = name.title().replace(" ", "") else: search = name return next((item for item in cls.__subclasses__() if item.__name__ == search), cls) else: return cls.__subclasses__() @classmethod def fuzzy_search(cls, **kwargs) -> List[Any]: """ Uses approximation of fields to get list of query results. Args: **kwargs (): Returns: List[Any]: Results of sqlalchemy query. """ query: Query = cls.__database_session__.query(cls) for k, v in kwargs.items(): # NOTE: Not sure why this is necessary, but it is. search = f"%{v}%" try: attr = getattr(cls, k) # NOTE: the secret sauce is in attr.like query = query.filter(attr.like(search)) except (ArgumentError, AttributeError) as e: logger.error(f"Attribute {k} unavailable due to:\n\t{e}\nSkipping.") return query.limit(50).all() @classmethod def results_to_df(cls, objects: list | None = None, **kwargs) -> DataFrame: """ Converts class sub_dicts into a Dataframe for all control of the class. Args: objects (list): Objects to be converted to dataframe. **kwargs (): Arguments necessary for the to_sub_dict method. eg kittype=X Returns: Dataframe """ if not objects: try: # records = [obj.to_sub_dict(**kwargs) for obj in cls.query()] records = [obj.details_dict(**kwargs) for obj in cls.query()] except AttributeError: # records = [obj.to_dict(**kwargs) for obj in cls.query(page_size=0)] records = [obj.details_dict(**kwargs) for obj in cls.query(page_size=0)] else: try: records = [obj.to_sub_dict(**kwargs) for obj in objects] except AttributeError: records = [{k: v['instance_attr'] for k, v in obj.omnigui_instance_dict.items()} for obj in objects] return DataFrame.from_records(records) @classmethod def query_or_create(cls, **kwargs) -> Tuple[Any, bool]: new = False allowed = [k for k, v in cls.__dict__.items() if isinstance(v, InstrumentedAttribute) or isinstance(v, hybrid_property)] # and not isinstance(v.property, _RelationshipDeclared)] sanitized_kwargs = {k: v for k, v in kwargs.items() if k in allowed} outside_kwargs = {k: v for k, v in kwargs.items() if k not in allowed} logger.debug(f"Sanitized kwargs: {sanitized_kwargs}") instance = cls.query(limit=1, **sanitized_kwargs) if not instance or isinstance(instance, list): instance = cls() new = True for k, v in sanitized_kwargs.items(): # logger.debug(f"QorC Setting {k} to {v}") if k == "id": continue try: setattr(instance, k, v) except AttributeError as e: from backend.validators.pydant import PydBaseClass if issubclass(v.__class__, PydBaseClass): setattr(instance, k, v.to_sql()) # else: # logger.error(f"Could not set {k} due to {e}") instance._misc_info.update(outside_kwargs) # logger.info(f"Instance from query or create: {instance}, new: {new}") return instance, new @classmethod def query(cls, **kwargs) -> Any | List[Any]: """ Default query function for models. Overridden in most models with additional filters. Returns: Any | List[Any]: Result of query execution. """ if "name" in kwargs.keys(): kwargs['limit'] = 1 return cls.execute_query(**kwargs) @classmethod def execute_query(cls, query: Query = None, model=None, limit: int = 0, offset: int | None = None, **kwargs) -> Any | List[Any]: """ Execute sqlalchemy query with relevant defaults. Args: model (Any, optional): model to be queried, allows for plugging in. Defaults to None query (Query, optional): input query object. Defaults to None limit (int): Maximum number of results. (0 = all). Defaults to 0 Returns: Any | List[Any]: Single result if limit = 1 or List if other. """ # logger.debug(f"Kwargs: {kwargs}") # if model is None: # model = cls # logger.debug(f"Model: {model}") if query is None: query: Query = cls.__database_session__.query(cls) # else: # logger.debug(f"Incoming query: {query}") singles = cls.get_default_info('singles') for k, v in kwargs.items(): # logger.info(f"Using key: {k} with value: {v} against {cls}") try: attr = getattr(cls, k) except (ArgumentError, AttributeError) as e: logger.error(f"Attribute {k} unavailable due to:\n\t{e}\n.") continue # NOTE: account for attrs that use list. try: check = attr.property.uselist except AttributeError: check = False if check: logger.debug("Got uselist") try: query = query.filter(attr.contains(v)) except ArgumentError: continue else: # logger.debug("Single item.") try: query = query.filter(attr == v) except ArgumentError: continue if k in singles: logger.warning(f"{k} is in singles. Returning only one value.") limit = 1 if offset: query.offset(offset) with query.session.no_autoflush: match limit: case 0: return query.all() case 1: return query.first() case _: return query.limit(limit).all() @report_result def save(self) -> Report | None: """ Add the object to the database and commit """ report = Report() del_keys = [] try: items = self._misc_info.items() except AttributeError: items = [] for key, value in items: try: json.dumps(value) except TypeError as e: del_keys.append(key) for dk in del_keys: del self._misc_info[dk] try: self.__database_session__.add(self) self.__database_session__.commit() # except sqlalchemy.exc.IntegrityError as i: # logger.error(f"Integrity error saving {self} due to: {i}") # logger.error(pformat(self.__dict__)) except Exception as e: logger.critical(f"Problem saving {self} due to: {e}") logger.error(f"Error message: {type(e)}") logger.error(pformat(self.__dict__)) self.__database_session__.rollback() report.add_result(Result(msg=e, status="Critical")) return report @property def omnigui_instance_dict(self) -> dict: """ For getting any object in an omni-thing friendly output. Returns: dict: Dictionary of object minus _sa_instance_state with id at the front. """ dicto = {key: dict(class_attr=getattr(self.__class__, key), instance_attr=getattr(self, key)) for key in dir(self.__class__) if isinstance(getattr(self.__class__, key), InstrumentedAttribute) and key not in self.omni_removes } for k, v in dicto.items(): try: v['instance_attr'] = v['instance_attr'].name except AttributeError: continue try: dicto = list_sort_dict(input_dict=dicto, sort_list=self.__class__.omni_sort) except TypeError as e: logger.error(f"Could not sort {self.__class__.__name__} by list due to :{e}") try: dicto = {'id': dicto.pop('id'), **dicto} except KeyError: pass return dicto @classproperty def pydantic_model(cls) -> BaseModel: """ Gets the pydantic model corresponding to this object. Returns: Pydantic model with name "Pyd{cls.__name__}" """ from backend.validators import pydant try: model = getattr(pydant, f"Pyd{cls.__name__}") except AttributeError: logger.warning(f"Couldn't get {cls.__name__} pydantic model.") return pydant.PydElastic return model @classproperty def add_edit_tooltips(cls) -> dict: """ Gets tooltips for Omni-add-edit Returns: dict: custom dictionary for this class. """ return dict() @classproperty def details_template(cls) -> Template: """ Get the details jinja template for the correct class Args: base_dict (dict): incoming dictionary of Submission fields Returns: Tuple(dict, Template): (Updated dictionary, Template to be rendered) """ env = jinja_template_loading() temp_name = f"{cls.__name__.lower()}_details.html" try: template = env.get_template(temp_name) except TemplateNotFound as e: # logger.error(f"Couldn't find template {e}") template = env.get_template("details.html") return template def check_all_attributes(self, attributes: dict) -> bool: """ Checks this instance against a dictionary of attributes to determine if they are a match. Args: attributes (dict): A dictionary of attributes to be check for equivalence Returns: bool: If a single unequivocal value is found will be false, else true. """ # logger.debug(f"Incoming attributes: {attributes}") for key, value in attributes.items(): if value.lower() == "none": value = None # logger.debug(f"Attempting to grab attribute: {key}") self_value = getattr(self, key) class_attr = getattr(self.__class__, key) # logger.debug(f"Self value: {self_value}, class attr: {class_attr} of type: {type(class_attr)}") if isinstance(class_attr, property): filter = "property" else: filter = class_attr.property match filter: case ColumnProperty(): match class_attr.type: case INTEGER(): if value.lower() == "true": value = 1 elif value.lower() == "false": value = 0 else: value = int(value) case FLOAT(): value = float(value) case "property": pass case _RelationshipDeclared(): # logger.debug(f"Checking {self_value}") try: self_value = self_value.name except AttributeError: pass if class_attr.property.uselist: self_value = self_value.__str__() try: # logger.debug(f"Check if {self_value.__class__} is subclass of {self.__class__}") check = issubclass(self_value.__class__, self.__class__) except TypeError as e: logger.error(f"Couldn't check if {self_value.__class__} is subclass of {self.__class__} due to {e}") check = False if check: # logger.debug(f"Checking for subclass name.") self_value = self_value.name # logger.debug(f"Checking self_value {self_value} of type {type(self_value)} against attribute {value} of type {type(value)}") if self_value != value: output = False # logger.debug(f"Value {key} is False, returning.") return output return True def __setattr__(self, key, value): """ Custom dunder method to handle potential list relationship issues. """ # logger.debug(f"Attempting to set: {key} to {value}") if key.startswith("_"): return super().__setattr__(key, value) # try: check = not hasattr(self, key) # except: # return if check: try: value = json.dumps(value) except TypeError: value = str(value) try: self._misc_info.update({key: value}) except AttributeError: self._misc_info = {key: value} return try: field_type = getattr(self.__class__, key) except AttributeError: return super().__setattr__(key, value) if isinstance(field_type, InstrumentedAttribute): # logger.debug(f"{key} is an InstrumentedAttribute.") match field_type.property: case ColumnProperty(): # logger.debug(f"Setting ColumnProperty to {value}") return super().__setattr__(key, value) case _RelationshipDeclared(): # logger.debug(f"{self.__class__.__name__} Setting _RelationshipDeclared for {key} to {value}") if field_type.property.uselist: # logger.debug(f"Setting with uselist") existing = self.__getattribute__(key) # NOTE: This is causing problems with removal of items from lists. Have to overhaul it. if existing is not None: logger.debug(f"{key} Existing: {existing}, incoming: {value}") if isinstance(value, list): # value = existing + value value = value else: value = existing + [value] else: if isinstance(value, list): # value = value pass else: value = [value] try: value = list(set(value)) except TypeError: pass # logger.debug(f"Final value for {key}: {value}") return super().__setattr__(key, value) else: if isinstance(value, list): if len(value) == 1: value = value[0] else: raise ValueError("Object is too long to parse a single value.") try: return super().__setattr__(key, value) except AttributeError: logger.warning(f"Possible attempt to set relationship {key} to simple var type. {value}") relationship_class = field_type.property.entity.entity value = relationship_class.query(name=value) try: return super().__setattr__(key, value) except AttributeError: return super().__setattr__(key, None) case _: return super().__setattr__(key, value) else: return super().__setattr__(key, value) def delete(self, **kwargs): logger.error(f"Delete has not been implemented for {self.__class__.__name__}") def rectify_query_date(input_date: datetime, eod: bool = False) -> str: """ Converts input into a datetime string for querying purposes Args: eod (bool, optional): Whether to use max time to indicate end of day. input_date (): Returns: datetime: properly formated datetime """ match input_date: case datetime() | date(): output_date = input_date case int(): output_date = datetime.fromordinal( datetime(1900, 1, 1).toordinal() + input_date - 2) case _: output_date = parse(input_date) if eod: addition_time = datetime.max.time() else: addition_time = datetime.min.time() output_date = datetime.combine(output_date, addition_time).strftime("%Y-%m-%d %H:%M:%S") return output_date def details_dict(self, **kwargs) -> dict: relevant = {k: v for k, v in self.__class__.__dict__.items() if isinstance(v, InstrumentedAttribute) or isinstance(v, AssociationProxy)} # output = OrderedDict() output = dict(excluded=["excluded", "misc_info", "_misc_info", "id"]) for k, v in relevant.items(): try: check = v.foreign_keys except AttributeError: check = False if check: continue try: value = getattr(self, k) except AttributeError: continue # try: # logger.debug(f"Setting {k} to {value} for details dict.") # except AttributeError as e: # logger.error(f"Can't log {k} value due to {type(e)}") # continue output[k.strip("_")] = value if self._misc_info: for key, value in self._misc_info.items(): # logger.debug(f"Misc info key {key}") output[key] = value return output @classmethod def clean_details_for_render(cls, dictionary): output = {} for k, value in dictionary.items(): match value: case datetime() | date(): value = value.strftime("%Y-%m-%d") case bytes(): continue case dict(): try: value = value['name'] except KeyError: if k == "_misc_info" or k == "misc_info": value = value else: continue case x if issubclass(value.__class__, cls): try: value = value.name except AttributeError: continue case _: pass output[k] = value return output def to_pydantic(self, pyd_model_name: str | None = None, **kwargs): from backend.validators import pydant if not pyd_model_name: pyd_model_name = f"Pyd{self.__class__.__name__}" logger.debug(f"Looking for pydant model {pyd_model_name}") try: pyd = getattr(pydant, pyd_model_name) except AttributeError: raise AttributeError(f"Could not get pydantic class {pyd_model_name}") logger.debug(f"Kwargs: {kwargs}") return pyd(**self.details_dict(**kwargs)) def show_details(self, obj): logger.debug("Show Details") from frontend.widgets.submission_details import SubmissionDetails dlg = SubmissionDetails(parent=obj, sub=self) if dlg.exec(): pass def export(self, obj, output_filepath: str | Path | None = None): # if not hasattr(self, "template_file"): # logger.error(f"Export not implemented for {self.__class__.__name__}") # return # pyd = self.to_pydantic() # if not output_filepath: # from frontend import select_save_file # output_filepath = select_save_file(obj=obj, default_name=pyd.construct_filename(), extension="xlsx") # Writer = getattr(writers, f"{self.__class__.__name__}Writer") # writer = Writer(output_filepath=output_filepath, pydant_obj=pyd, range_dict=self.range_dict) # workbook = writer from backend import managers Manager = getattr(managers, f"Default{self.__class__.__name__}") manager = Manager(parent=obj, input_object=self) class LogMixin(Base): tracking_exclusion: ClassVar = ['artic_technician', 'clientsubmissionsampleassociation', 'submission_reagent_associations', 'submission_equipment_associations', 'submission_tips_associations', 'contact_id', 'gel_info', 'gel_controls', 'source_plates'] __abstract__ = True @property def truncated_name(self): name = str(self) if len(name) > 64: name = name.replace("<", "").replace(">", "") if len(name) > 64: # NOTE: As if re'agent' name = name.replace("agent", "") if len(name) > 64: name = f"...{name[-61:]}" return name class ConfigItem(BaseClass): """ Key:JSON objects to store config settings in database. """ id = Column(INTEGER, primary_key=True) key = Column(String(32)) #: Name of the configuration item. value = Column(JSON) #: Value associated with the config item. def __repr__(self) -> str: return f"" @classmethod def get_config_items(cls, *args) -> ConfigItem | List[ConfigItem]: """ Get desired config items, or all from database Returns: ConfigItem|List[ConfigItem]: Config item(s) """ query = cls.__database_session__.query(cls) match len(args): case 0: config_items = query.all() # NOTE: If only one item sought, don't use a list, just return it. case 1: config_items = query.filter(cls.key == args[0]).first() case _: # NOTE: All items whose key field is in args. config_items = query.filter(cls.key.in_(args)).all() return config_items from .controls import * # NOTE: import order must go: orgs, kittype, run due to circular import issues from .organizations import * from .procedures import * from .submissions import * from .audit import AuditLog # NOTE: Add a creator to the procedure for reagent association. Assigned here due to circular import constraints. # https://docs.sqlalchemy.org/en/20/orm/extensions/associationproxy.html#sqlalchemy.ext.associationproxy.association_proxy.params.creator # Procedure.reagents.creator = lambda reg: ProcedureReagentAssociation(reagent=reg)