Split Concentration controls on the chart so they are individually selectable.

src/submissions/backend/db/models/__init__.py

@@ -11,9 +11,7 @@ from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.exc import ArgumentError
 from typing import Any, List
 from pathlib import Path
-
 from sqlalchemy.orm.relationships import _RelationshipDeclared
-
 from tools import report_result, list_sort_dict
 
 # NOTE: Load testing environment
@@ -48,7 +46,7 @@ class BaseClass(Base):
     """
     __abstract__ = True  #: NOTE: Will not be added to DB as a table
 
-    __table_args__ = {'extend_existing': True}  #: Will only add new columns
+    __table_args__ = {'extend_existing': True}  #: NOTE Will only add new columns
 
     singles = ['id']
     omni_removes = ["id", 'submissions', "omnigui_class_dict", "omnigui_instance_dict"]
@@ -308,7 +306,6 @@ class BaseClass(Base):
             dicto = {'id': dicto.pop('id'), **dicto}
         except KeyError:
             pass
-        # logger.debug(f"{self.__class__.__name__} omnigui dict:\n\n{pformat(dicto)}")
         return dicto
 
     @classproperty
@@ -337,11 +334,6 @@ class BaseClass(Base):
         """
         return dict()
 
-    @classmethod
-    def relevant_relationships(cls, relationship_instance):
-        query_kwargs = {relationship_instance.query_alias: relationship_instance}
-        return cls.query(**query_kwargs)
-
     def check_all_attributes(self, attributes: dict) -> bool:
         """
         Checks this instance against a dictionary of attributes to determine if they are a match.
@@ -352,14 +344,14 @@ class BaseClass(Base):
         Returns:
             bool: If a single unequivocal value is found will be false, else true.
         """
-        logger.debug(f"Incoming attributes: {attributes}")
+        # logger.debug(f"Incoming attributes: {attributes}")
         for key, value in attributes.items():
             if value.lower() == "none":
                 value = None
-            logger.debug(f"Attempting to grab attribute: {key}")
+            # logger.debug(f"Attempting to grab attribute: {key}")
             self_value = getattr(self, key)
             class_attr = getattr(self.__class__, key)
-            logger.debug(f"Self value: {self_value}, class attr: {class_attr} of type: {type(class_attr)}")
+            # logger.debug(f"Self value: {self_value}, class attr: {class_attr} of type: {type(class_attr)}")
             if isinstance(class_attr, property):
                 filter = "property"
             else:
@@ -379,7 +371,7 @@ class BaseClass(Base):
                 case "property":
                     pass
                 case _RelationshipDeclared():
-                    logger.debug(f"Checking {self_value}")
+                    # logger.debug(f"Checking {self_value}")
                     try:
                         self_value = self_value.name
                     except AttributeError:
@@ -387,19 +379,18 @@ class BaseClass(Base):
                     if class_attr.property.uselist:
                         self_value = self_value.__str__()
             try:
-                logger.debug(f"Check if {self_value.__class__} is subclass of {self.__class__}")
+                # logger.debug(f"Check if {self_value.__class__} is subclass of {self.__class__}")
                 check = issubclass(self_value.__class__, self.__class__)
             except TypeError as e:
                 logger.error(f"Couldn't check if {self_value.__class__} is subclass of {self.__class__} due to {e}")
                 check = False
             if check:
-                logger.debug(f"Checking for subclass name.")
+                # logger.debug(f"Checking for subclass name.")
                 self_value = self_value.name
-            logger.debug(
-                f"Checking self_value {self_value} of type {type(self_value)} against attribute {value} of type {type(value)}")
+            # logger.debug(f"Checking self_value {self_value} of type {type(self_value)} against attribute {value} of type {type(value)}")
             if self_value != value:
                 output = False
-                logger.debug(f"Value {key} is False, returning.")
+                # logger.debug(f"Value {key} is False, returning.")
                 return output
         return True
 
@@ -444,7 +435,6 @@ class BaseClass(Base):
                                 value = value[0]
                             else:
                                 raise ValueError("Object is too long to parse a single value.")
-                                # value = value
                         return super().__setattr__(key, value)
                 case _:
                     return super().__setattr__(key, value)
@@ -454,6 +444,32 @@ class BaseClass(Base):
     def delete(self):
         logger.error(f"Delete has not been implemented for {self.__class__.__name__}")
 
+    def rectify_query_date(input_date, eod: bool = False) -> str:
+        """
+        Converts input into a datetime string for querying purposes
+
+        Args:
+            eod (bool, optional): Whether to use max time to indicate end of day.
+            input_date ():
+
+        Returns:
+            datetime: properly formated datetime
+        """
+        match input_date:
+            case datetime() | date():
+                output_date = input_date#.strftime("%Y-%m-%d %H:%M:%S")
+            case int():
+                output_date = datetime.fromordinal(
+                    datetime(1900, 1, 1).toordinal() + input_date - 2)#.date().strftime("%Y-%m-%d %H:%M:%S")
+            case _:
+                output_date = parse(input_date)#.strftime("%Y-%m-%d %H:%M:%S")
+        if eod:
+            addition_time = datetime.max.time()
+        else:
+            addition_time = datetime.min.time()
+        output_date = datetime.combine(output_date, addition_time).strftime("%Y-%m-%d %H:%M:%S")
+        return output_date
+
 
 class ConfigItem(BaseClass):
     """

src/submissions/backend/db/models/controls.py

@@ -2,7 +2,6 @@
 All control related models.
 """
 from __future__ import annotations
-
 import itertools
 from pprint import pformat
 from PyQt6.QtWidgets import QWidget, QCheckBox, QLabel
@@ -13,10 +12,9 @@ import logging, re
 from operator import itemgetter
 from . import BaseClass
 from tools import setup_lookup, report_result, Result, Report, Settings, get_unique_values_in_df_column, super_splitter, \
-    rectify_query_date
+    flatten_list, timer
 from datetime import date, datetime, timedelta
 from typing import List, Literal, Tuple, Generator
-from dateutil.parser import parse
 from re import Pattern
 
 logger = logging.getLogger(f"submissions.{__name__}")
@@ -31,9 +29,6 @@ class ControlType(BaseClass):
     targets = Column(JSON)  #: organisms checked for
     instances = relationship("Control", back_populates="controltype")  #: control samples created of this type.
 
-    # def __repr__(self) -> str:
-    #     return f"<ControlType({self.name})>"
-
     @classmethod
     @setup_lookup
     def query(cls,
@@ -113,6 +108,7 @@ class ControlType(BaseClass):
             Pattern: Constructed pattern
         """
         strings = list(set([super_splitter(item, "-", 0) for item in cls.get_positive_control_types(control_type)]))
+        # NOTE: This will build a string like ^(ATCC49226|MCS)-.*
         return re.compile(rf"(^{'|^'.join(strings)})-.*", flags=re.IGNORECASE)
 
 
@@ -159,7 +155,7 @@ class Control(BaseClass):
         Lookup control objects in the database based on a number of parameters.
 
         Args:
-            submission_type (str | None, optional): Submission type associated with control. Defaults to None.
+            submissiontype (str | None, optional): Submission type associated with control. Defaults to None.
             subtype (str | None, optional): Control subtype, eg IridaControl. Defaults to None.
             start_date (date | str | int | None, optional): Beginning date to search by. Defaults to 2023-01-01 if end_date not None.
             end_date (date | str | int | None, optional): End date to search by. Defaults to today if start_date not None.
@@ -202,30 +198,8 @@ class Control(BaseClass):
             logger.warning(f"End date with no start date, using 90 days ago.")
             start_date = date.today() - timedelta(days=90)
         if start_date is not None:
-            # match start_date:
-            #     case datetime():
-            #         start_date = start_date.strftime("%Y-%m-%d %H:%M:%S")
-            #     case date():
-            #         start_date = datetime.combine(start_date, datetime.min.time())
-            #         start_date = start_date.strftime("%Y-%m-%d %H:%M:%S")
-            #     case int():
-            #         start_date = datetime.fromordinal(
-            #             datetime(1900, 1, 1).toordinal() + start_date - 2).date().strftime("%Y-%m-%d %H:%M:%S")
-            #     case _:
-            #         start_date = parse(start_date).strftime("%Y-%m-%d %H:%M:%S")
-            start_date = rectify_query_date(start_date)
-            end_date = rectify_query_date(end_date, eod=True)
-            # match end_date:
-            #     case datetime():
-            #         end_date = end_date.strftime("%Y-%m-%d %H:%M:%S")
-            #     case date():
-            #         end_date = datetime.combine(end_date, datetime.max.time())
-            #         end_date = end_date.strftime("%Y-%m-%d %H:%M:%S")
-            #     case int():
-            #         end_date = datetime.fromordinal(datetime(1900, 1, 1).toordinal() + end_date - 2).date().strftime(
-            #             "%Y-%m-%d %H:%M:%S")
-            #     case _:
-            #         end_date = parse(end_date).strftime("%Y-%m-%d %H:%M:%S")
+            start_date = cls.rectify_query_date(start_date)
+            end_date = cls.rectify_query_date(end_date, eod=True)
             query = query.filter(cls.submitted_date.between(start_date, end_date))
         match name:
             case str():
@@ -372,7 +346,8 @@ class PCRControl(Control):
 
     def to_pydantic(self):
         from backend.validators import PydPCRControl
-        return PydPCRControl(**self.to_sub_dict(), controltype_name=self.controltype_name,
+        return PydPCRControl(**self.to_sub_dict(),
+                             controltype_name=self.controltype_name,
                              submission_id=self.submission_id)
 
 
@@ -565,7 +540,8 @@ class IridaControl(Control):
                                         consolidate=consolidate) for
                 control in controls]
         # NOTE: flatten data to one dimensional list
-        data = [item for sublist in data for item in sublist]
+        # data = [item for sublist in data for item in sublist]
+        data = flatten_list(data)
         if not data:
             report.add_result(Result(status="Critical", msg="No data found for controls in given date range."))
             return report, None
@@ -731,11 +707,11 @@ class IridaControl(Control):
         Returns:
             DataFrame: dataframe with originals removed in favour of repeats.
         """
-        if 'rerun_regex' in ctx:
+        if 'rerun_regex' in ctx.model_extra:
             sample_names = get_unique_values_in_df_column(df, column_name="name")
             rerun_regex = re.compile(fr"{ctx.rerun_regex}")
             exclude = [re.sub(rerun_regex, "", sample) for sample in sample_names if rerun_regex.search(sample)]
-            df = df[df.name not in exclude]
+            df = df[~df.name.isin(exclude)]
         return df
 
     def to_pydantic(self) -> "PydIridaControl":

src/submissions/backend/db/models/kits.py

@@ -2,8 +2,7 @@
 All kit and reagent related models
 """
 from __future__ import annotations
-import json, zipfile, yaml, logging, re
-import sys
+import json, zipfile, yaml, logging, re, sys
 from pprint import pformat
 from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey, Interval, Table, FLOAT, BLOB
 from sqlalchemy.orm import relationship, validates, Query
@@ -11,7 +10,7 @@ from sqlalchemy.ext.associationproxy import association_proxy
 from sqlalchemy.ext.hybrid import hybrid_property
 from datetime import date, datetime, timedelta
 from tools import check_authorization, setup_lookup, Report, Result, check_regex_match, yaml_regex_creator, timezone
-from typing import List, Literal, Generator, Any, Tuple, Dict, AnyStr
+from typing import List, Literal, Generator, Any, Tuple
 from pandas import ExcelFile
 from pathlib import Path
 from . import Base, BaseClass, Organization, LogMixin
@@ -136,18 +135,18 @@ class KitType(BaseClass):
         return self.used_for
 
     def get_reagents(self,
-                     required: bool = False,
+                     required_only: bool = False,
                      submission_type: str | SubmissionType | None = None
                      ) -> Generator[ReagentRole, None, None]:
         """
         Return ReagentTypes linked to kit through KitTypeReagentTypeAssociation.
 
         Args:
-            required (bool, optional): If true only return required types. Defaults to False.
+            required_only (bool, optional): If true only return required types. Defaults to False.
             submission_type (str | Submissiontype | None, optional): Submission type to narrow results. Defaults to None.
 
         Returns:
-            Generator[ReagentRole, None, None]: List of reagents linked to this kit.
+            Generator[ReagentRole, None, None]: List of reagent roles linked to this kit.
         """
         match submission_type:
             case SubmissionType():
@@ -158,7 +157,7 @@ class KitType(BaseClass):
                                          item.submission_type.name == submission_type]
             case _:
                 relevant_associations = [item for item in self.kit_reagentrole_associations]
-        if required:
+        if required_only:
             return (item.reagent_role for item in relevant_associations if item.required == 1)
         else:
             return (item.reagent_role for item in relevant_associations)
@@ -168,7 +167,6 @@ class KitType(BaseClass):
         Creates map of locations in Excel workbook for a SubmissionType
 
         Args:
-            new_kit ():
             submission_type (str | SubmissionType): Submissiontype.name
 
         Returns:
@@ -240,7 +238,7 @@ class KitType(BaseClass):
 
         Args:
             name (str, optional): Name of desired kit (returns single instance). Defaults to None.
-            used_for (str | Submissiontype | None, optional): Submission type the kit is used for. Defaults to None.
+            submissiontype (str | Submissiontype | None, optional): Submission type the kit is used for. Defaults to None.
             id (int | None, optional): Kit id in the database. Defaults to None.
             limit (int, optional): Maximum number of results to return (0 = all). Defaults to 0.
 
@@ -276,108 +274,108 @@ class KitType(BaseClass):
     def save(self):
         super().save()
 
-    def to_export_dict(self, submission_type: SubmissionType) -> dict:
-        """
-        Creates dictionary for exporting to yml used in new SubmissionType Construction
+    # def to_export_dict(self, submission_type: SubmissionType) -> dict:
+    #     """
+    #     Creates dictionary for exporting to yml used in new SubmissionType Construction
+    #
+    #     Args:
+    #         submission_type (SubmissionType): SubmissionType of interest.
+    #
+    #     Returns:
+    #         dict: Dictionary containing relevant info for SubmissionType construction
+    #     """
+    #     base_dict = dict(name=self.name, reagent_roles=[], equipment_roles=[])
+    #     for key, value in self.construct_xl_map_for_use(submission_type=submission_type):
+    #         try:
+    #             assoc = next(item for item in self.kit_reagentrole_associations if item.reagent_role.name == key)
+    #         except StopIteration as e:
+    #             continue
+    #         for kk, vv in assoc.to_export_dict().items():
+    #             value[kk] = vv
+    #         base_dict['reagent_roles'].append(value)
+    #     for key, value in submission_type.construct_field_map("equipment"):
+    #         try:
+    #             assoc = next(item for item in submission_type.submissiontype_equipmentrole_associations if
+    #                          item.equipment_role.name == key)
+    #         except StopIteration:
+    #             continue
+    #         for kk, vv in assoc.to_export_dict(extraction_kit=self).items():
+    #             value[kk] = vv
+    #         base_dict['equipment_roles'].append(value)
+    #     return base_dict
 
-        Args:
-            submission_type (SubmissionType): SubmissionType of interest.
-
-        Returns:
-            dict: Dictionary containing relevant info for SubmissionType construction
-        """
-        base_dict = dict(name=self.name, reagent_roles=[], equipment_roles=[])
-        for key, value in self.construct_xl_map_for_use(submission_type=submission_type):
-            try:
-                assoc = next(item for item in self.kit_reagentrole_associations if item.reagent_role.name == key)
-            except StopIteration as e:
-                continue
-            for kk, vv in assoc.to_export_dict().items():
-                value[kk] = vv
-            base_dict['reagent_roles'].append(value)
-        for key, value in submission_type.construct_field_map("equipment"):
-            try:
-                assoc = next(item for item in submission_type.submissiontype_equipmentrole_associations if
-                             item.equipment_role.name == key)
-            except StopIteration:
-                continue
-            for kk, vv in assoc.to_export_dict(extraction_kit=self).items():
-                value[kk] = vv
-            base_dict['equipment_roles'].append(value)
-        return base_dict
-
-    @classmethod
-    def import_from_yml(cls, submission_type: str | SubmissionType, filepath: Path | str | None = None,
-                        import_dict: dict | None = None) -> KitType:
-        if isinstance(submission_type, str):
-            submission_type = SubmissionType.query(name=submission_type)
-        if filepath:
-            yaml.add_constructor("!regex", yaml_regex_creator)
-            if isinstance(filepath, str):
-                filepath = Path(filepath)
-            if not filepath.exists():
-                logging.critical(f"Given file could not be found.")
-                return None
-            with open(filepath, "r") as f:
-                if filepath.suffix == ".json":
-                    import_dict = json.load(fp=f)
-                elif filepath.suffix == ".yml":
-                    import_dict = yaml.load(stream=f, Loader=yaml.Loader)
-                else:
-                    raise Exception(f"Filetype {filepath.suffix} not supported.")
-        new_kit = KitType.query(name=import_dict['kit_type']['name'])
-        if not new_kit:
-            new_kit = KitType(name=import_dict['kit_type']['name'])
-        for role in import_dict['kit_type']['reagent_roles']:
-            new_role = ReagentRole.query(name=role['role'])
-            if new_role:
-                check = input(f"Found existing role: {new_role.name}. Use this? [Y/n]: ")
-                if check.lower() == "n":
-                    new_role = None
-                else:
-                    pass
-            if not new_role:
-                eol = timedelta(role['extension_of_life'])
-                new_role = ReagentRole(name=role['role'], eol_ext=eol)
-            uses = dict(expiry=role['expiry'], lot=role['lot'], name=role['name'], sheet=role['sheet'])
-            ktrr_assoc = KitTypeReagentRoleAssociation(kit_type=new_kit, reagent_role=new_role, uses=uses)
-            ktrr_assoc.submission_type = submission_type
-            ktrr_assoc.required = role['required']
-        ktst_assoc = SubmissionTypeKitTypeAssociation(
-            kit_type=new_kit,
-            submission_type=submission_type,
-            mutable_cost_sample=import_dict['mutable_cost_sample'],
-            mutable_cost_column=import_dict['mutable_cost_column'],
-            constant_cost=import_dict['constant_cost']
-        )
-        for role in import_dict['kit_type']['equipment_roles']:
-            new_role = EquipmentRole.query(name=role['role'])
-            if new_role:
-                check = input(f"Found existing role: {new_role.name}. Use this? [Y/n]: ")
-                if check.lower() == "n":
-                    new_role = None
-                else:
-                    pass
-            if not new_role:
-                new_role = EquipmentRole(name=role['role'])
-                for equipment in Equipment.assign_equipment(equipment_role=new_role):
-                    new_role.instances.append(equipment)
-            ster_assoc = SubmissionTypeEquipmentRoleAssociation(submission_type=submission_type,
-                                                                equipment_role=new_role)
-            try:
-                uses = dict(name=role['name'], process=role['process'], sheet=role['sheet'],
-                            static=role['static'])
-            except KeyError:
-                uses = None
-            ster_assoc.uses = uses
-            for process in role['processes']:
-                new_process = Process.query(name=process)
-                if not new_process:
-                    new_process = Process(name=process)
-                new_process.submission_types.append(submission_type)
-                new_process.kit_types.append(new_kit)
-                new_process.equipment_roles.append(new_role)
-        return new_kit
+    # @classmethod
+    # def import_from_yml(cls, submission_type: str | SubmissionType, filepath: Path | str | None = None,
+    #                     import_dict: dict | None = None) -> KitType:
+    #     if isinstance(submission_type, str):
+    #         submission_type = SubmissionType.query(name=submission_type)
+    #     if filepath:
+    #         yaml.add_constructor("!regex", yaml_regex_creator)
+    #         if isinstance(filepath, str):
+    #             filepath = Path(filepath)
+    #         if not filepath.exists():
+    #             logging.critical(f"Given file could not be found.")
+    #             return None
+    #         with open(filepath, "r") as f:
+    #             if filepath.suffix == ".json":
+    #                 import_dict = json.load(fp=f)
+    #             elif filepath.suffix == ".yml":
+    #                 import_dict = yaml.load(stream=f, Loader=yaml.Loader)
+    #             else:
+    #                 raise Exception(f"Filetype {filepath.suffix} not supported.")
+    #     new_kit = KitType.query(name=import_dict['kit_type']['name'])
+    #     if not new_kit:
+    #         new_kit = KitType(name=import_dict['kit_type']['name'])
+    #     for role in import_dict['kit_type']['reagent_roles']:
+    #         new_role = ReagentRole.query(name=role['role'])
+    #         if new_role:
+    #             check = input(f"Found existing role: {new_role.name}. Use this? [Y/n]: ")
+    #             if check.lower() == "n":
+    #                 new_role = None
+    #             else:
+    #                 pass
+    #         if not new_role:
+    #             eol = timedelta(role['extension_of_life'])
+    #             new_role = ReagentRole(name=role['role'], eol_ext=eol)
+    #         uses = dict(expiry=role['expiry'], lot=role['lot'], name=role['name'], sheet=role['sheet'])
+    #         ktrr_assoc = KitTypeReagentRoleAssociation(kit_type=new_kit, reagent_role=new_role, uses=uses)
+    #         ktrr_assoc.submission_type = submission_type
+    #         ktrr_assoc.required = role['required']
+    #     ktst_assoc = SubmissionTypeKitTypeAssociation(
+    #         kit_type=new_kit,
+    #         submission_type=submission_type,
+    #         mutable_cost_sample=import_dict['mutable_cost_sample'],
+    #         mutable_cost_column=import_dict['mutable_cost_column'],
+    #         constant_cost=import_dict['constant_cost']
+    #     )
+    #     for role in import_dict['kit_type']['equipment_roles']:
+    #         new_role = EquipmentRole.query(name=role['role'])
+    #         if new_role:
+    #             check = input(f"Found existing role: {new_role.name}. Use this? [Y/n]: ")
+    #             if check.lower() == "n":
+    #                 new_role = None
+    #             else:
+    #                 pass
+    #         if not new_role:
+    #             new_role = EquipmentRole(name=role['role'])
+    #             for equipment in Equipment.assign_equipment(equipment_role=new_role):
+    #                 new_role.instances.append(equipment)
+    #         ster_assoc = SubmissionTypeEquipmentRoleAssociation(submission_type=submission_type,
+    #                                                             equipment_role=new_role)
+    #         try:
+    #             uses = dict(name=role['name'], process=role['process'], sheet=role['sheet'],
+    #                         static=role['static'])
+    #         except KeyError:
+    #             uses = None
+    #         ster_assoc.uses = uses
+    #         for process in role['processes']:
+    #             new_process = Process.query(name=process)
+    #             if not new_process:
+    #                 new_process = Process(name=process)
+    #             new_process.submission_types.append(submission_type)
+    #             new_process.kit_types.append(new_kit)
+    #             new_process.equipment_roles.append(new_role)
+    #     return new_kit
 
     def to_omni(self, expand: bool = False) -> "OmniKitType":
         from backend.validators.omni_gui_objects import OmniKitType
@@ -395,7 +393,7 @@ class KitType(BaseClass):
             kit_reagentrole_associations=kit_reagentrole_associations,
             kit_submissiontype_associations=kit_submissiontype_associations
         )
-        logger.debug(f"Creating omni for {pformat(data)}")
+        # logger.debug(f"Creating omni for {pformat(data)}")
         return OmniKitType(instance_object=self, **data)
 
 
@@ -405,7 +403,6 @@ class ReagentRole(BaseClass):
     """
 
     skip_on_edit = False
-
     id = Column(INTEGER, primary_key=True)  #: primary key
     name = Column(String(64))  #: name of role reagent plays
     instances = relationship("Reagent", back_populates="role",
@@ -453,7 +450,7 @@ class ReagentRole(BaseClass):
         Args:
             id (id | None, optional): Id of the object. Defaults to None.
             name (str | None, optional): Reagent type name. Defaults to None.
-            kit_type (KitType | str | None, optional): Kit the type of interest belongs to. Defaults to None.
+            kittype (KitType | str | None, optional): Kit the type of interest belongs to. Defaults to None.
             reagent (Reagent | str | None, optional): Concrete instance of the type of interest. Defaults to None.
             limit (int, optional): maxmimum number of results to return (0 = all). Defaults to 0.
 
@@ -507,14 +504,14 @@ class ReagentRole(BaseClass):
         from backend.validators.pydant import PydReagent
         return PydReagent(lot=None, role=self.name, name=self.name, expiry=date.today())
 
-    def to_export_dict(self) -> dict:
-        """
-        Creates dictionary for exporting to yml used in new SubmissionType Construction
-
-        Returns:
-            dict: Dictionary containing relevant info for SubmissionType construction
-        """
-        return dict(role=self.name, extension_of_life=self.eol_ext.days)
+    # def to_export_dict(self) -> dict:
+    #     """
+    #     Creates dictionary for exporting to yml used in new SubmissionType Construction
+    #
+    #     Returns:
+    #         dict: Dictionary containing relevant info for SubmissionType construction
+    #     """
+    #     return dict(role=self.name, extension_of_life=self.eol_ext.days)
 
     @check_authorization
     def save(self):
@@ -1278,20 +1275,20 @@ class SubmissionType(BaseClass):
                 pass
         return cls.execute_query(query=query, limit=limit)
 
-    def to_export_dict(self):
-        """
-        Creates dictionary for exporting to yml used in new SubmissionType Construction
-
-        Returns:
-            dict: Dictionary containing relevant info for SubmissionType construction
-        """
-        base_dict = dict(name=self.name)
-        base_dict['info'] = self.construct_info_map(mode='export')
-        base_dict['defaults'] = self.defaults
-        # base_dict['samples'] = self.construct_sample_map()
-        base_dict['samples'] = self.sample_map
-        base_dict['kits'] = [item.to_export_dict() for item in self.submissiontype_kit_associations]
-        return base_dict
+    # def to_export_dict(self):
+    #     """
+    #     Creates dictionary for exporting to yml used in new SubmissionType Construction
+    #
+    #     Returns:
+    #         dict: Dictionary containing relevant info for SubmissionType construction
+    #     """
+    #     base_dict = dict(name=self.name)
+    #     base_dict['info'] = self.construct_info_map(mode='export')
+    #     base_dict['defaults'] = self.defaults
+    #     # base_dict['samples'] = self.construct_sample_map()
+    #     base_dict['samples'] = self.sample_map
+    #     base_dict['kits'] = [item.to_export_dict() for item in self.submissiontype_kit_associations]
+    #     return base_dict
 
     @check_authorization
     def save(self):
@@ -1499,17 +1496,17 @@ class SubmissionTypeKitTypeAssociation(BaseClass):
         # limit = query.count()
         return cls.execute_query(query=query, limit=limit)
 
-    def to_export_dict(self):
-        """
-        Creates a dictionary of relevant values in this object.
-
-        Returns:
-            dict: dictionary of Association and related kittype
-        """
-        exclude = ['_sa_instance_state', 'submission_types_id', 'kits_id', 'submission_type', 'kit_type']
-        base_dict = {k: v for k, v in self.__dict__.items() if k not in exclude}
-        base_dict['kit_type'] = self.kit_type.to_export_dict(submission_type=self.submission_type)
-        return base_dict
+    # def to_export_dict(self):
+    #     """
+    #     Creates a dictionary of relevant values in this object.
+    #
+    #     Returns:
+    #         dict: dictionary of Association and related kittype
+    #     """
+    #     exclude = ['_sa_instance_state', 'submission_types_id', 'kits_id', 'submission_type', 'kit_type']
+    #     base_dict = {k: v for k, v in self.__dict__.items() if k not in exclude}
+    #     base_dict['kit_type'] = self.kit_type.to_export_dict(submission_type=self.submission_type)
+    #     return base_dict
 
     def to_omni(self, expand: bool = False):
         from backend.validators.omni_gui_objects import OmniSubmissionTypeKitTypeAssociation
@@ -1719,17 +1716,17 @@ class KitTypeReagentRoleAssociation(BaseClass):
             limit = 1
         return cls.execute_query(query=query, limit=limit)
 
-    def to_export_dict(self) -> dict:
-        """
-        Creates a dictionary of relevant values in this object.
-
-        Returns:
-            dict: dictionary of Association and related reagent role
-        """
-        base_dict = dict(required=self.required)
-        for k, v in self.reagent_role.to_export_dict().items():
-            base_dict[k] = v
-        return base_dict
+    # def to_export_dict(self) -> dict:
+    #     """
+    #     Creates a dictionary of relevant values in this object.
+    #
+    #     Returns:
+    #         dict: dictionary of Association and related reagent role
+    #     """
+    #     base_dict = dict(required=self.required)
+    #     for k, v in self.reagent_role.to_export_dict().items():
+    #         base_dict[k] = v
+    #     return base_dict
 
     def get_all_relevant_reagents(self) -> Generator[Reagent, None, None]:
         """
@@ -1915,13 +1912,6 @@ class Equipment(BaseClass, LogMixin):
     submissions = association_proxy("equipment_submission_associations",
                                     "submission")  #: proxy to equipment_submission_associations.submission
 
-    # def __repr__(self) -> str:
-    #     """
-    #     Returns:
-    #         str: representation of this Equipment
-    #     """
-    #     return f"<Equipment({self.name})>"
-
     def to_dict(self, processes: bool = False) -> dict:
         """
         This Equipment as a dictionary
@@ -2085,13 +2075,6 @@ class EquipmentRole(BaseClass):
     submission_types = association_proxy("equipmentrole_submissiontype_associations",
                                          "submission_type")  #: proxy to equipmentrole_submissiontype_associations.submission_type
 
-    # def __repr__(self) -> str:
-    #     """
-    #     Returns:
-    #         str: Representation of this EquipmentRole
-    #     """
-    #     return f"<EquipmentRole({self.name})>"
-
     def to_dict(self) -> dict:
         """
         This EquipmentRole as a dictionary
@@ -2192,16 +2175,6 @@ class EquipmentRole(BaseClass):
                 continue
             yield process.name
 
-    def to_export_dict(self, submission_type: SubmissionType, kit_type: KitType):
-        """
-        Creates a dictionary of relevant values in this object.
-
-        Returns:
-            dict: dictionary of Association and related reagent role
-        """
-        processes = self.get_processes(submission_type=submission_type, extraction_kit=kit_type)
-        return dict(role=self.name, processes=[item for item in processes])
-
     def to_omni(self, expand: bool = False) -> "OmniEquipmentRole":
         from backend.validators.omni_gui_objects import OmniEquipmentRole
         return OmniEquipmentRole(instance_object=self, name=self.name)
@@ -2320,23 +2293,6 @@ class SubmissionTypeEquipmentRoleAssociation(BaseClass):
     def save(self):
         super().save()
 
-    def to_export_dict(self, extraction_kit: KitType | str) -> dict:
-        """
-        Creates dictionary for exporting to yml used in new SubmissionType Construction
-
-        Args:
-            extraction_kit (KitType | str): KitType of interest.
-
-        Returns:
-            dict: Dictionary containing relevant info for SubmissionType construction
-        """
-        if isinstance(extraction_kit, str):
-            extraction_kit = KitType.query(name=extraction_kit)
-        base_dict = {k: v for k, v in self.equipment_role.to_export_dict(submission_type=self.submission_type,
-                                                                         kit_type=extraction_kit).items()}
-        base_dict['static'] = self.static
-        return base_dict
-
 
 class Process(BaseClass):
     """
@@ -2360,14 +2316,6 @@ class Process(BaseClass):
     tip_roles = relationship("TipRole", back_populates='processes',
                              secondary=process_tiprole)  #: relation to KitType
 
-
-    # def __repr__(self) -> str:
-    #     """
-    #     Returns:
-    #         str: Representation of this Process
-    #     """
-    #     return f"<Process({self.name})>"
-
     def set_attribute(self, key, value):
         match key:
             case "name":
@@ -2496,9 +2444,6 @@ class TipRole(BaseClass):
     def tips(self):
         return self.instances
 
-    # def __repr__(self):
-    #     return f"<TipRole({self.name})>"
-
     @classmethod
     def query_or_create(cls, **kwargs) -> Tuple[TipRole, bool]:
         new = False
@@ -2567,9 +2512,6 @@ class Tips(BaseClass, LogMixin):
     def tiprole(self):
         return self.role
 
-    # def __repr__(self):
-    #     return f"<Tips({self.name})>"
-
     @classmethod
     def query_or_create(cls, **kwargs) -> Tuple[Tips, bool]:
         new = False

src/submissions/backend/db/models/organizations.py

@@ -2,14 +2,14 @@
 All client organization related models.
 '''
 from __future__ import annotations
-import json, yaml, logging
+import logging
 from pathlib import Path
 from pprint import pformat
 from sqlalchemy import Column, String, INTEGER, ForeignKey, Table
 from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.orm import relationship, Query
 from . import Base, BaseClass
-from tools import check_authorization, setup_lookup, yaml_regex_creator
+from tools import check_authorization, setup_lookup
 from typing import List, Tuple
 
 logger = logging.getLogger(f"submissions.{__name__}")
@@ -41,9 +41,6 @@ class Organization(BaseClass):
     def contact(self):
         return self.contacts
 
-    # def __repr__(self) -> str:
-    #     return f"<Organization({self.name})>"
-
     @classmethod
     @setup_lookup
     def query(cls,
@@ -80,49 +77,6 @@ class Organization(BaseClass):
     def save(self):
         super().save()
 
-    @classmethod
-    @check_authorization
-    def import_from_yml(cls, filepath: Path | str):
-        """
-        An ambitious project to create a Organization from a yml file
-
-        Args:
-            filepath (Path): Filepath of the yml.
-
-        Returns:
-
-        """
-        yaml.add_constructor("!regex", yaml_regex_creator)
-        if isinstance(filepath, str):
-            filepath = Path(filepath)
-        if not filepath.exists():
-            logging.critical(f"Given file could not be found.")
-            return None
-        with open(filepath, "r") as f:
-            if filepath.suffix == ".json":
-                import_dict = json.load(fp=f)
-            elif filepath.suffix == ".yml":
-                import_dict = yaml.load(stream=f, Loader=yaml.Loader)
-            else:
-                raise Exception(f"Filetype {filepath.suffix} not supported.")
-        data = import_dict['orgs']
-        for org in data:
-            organ = Organization.query(name=org['name'])
-            if organ is None:
-                organ = Organization(name=org['name'])
-                try:
-                    organ.cost_centre = org['cost_centre']
-                except KeyError:
-                    organ.cost_centre = "xxx"
-            for contact in org['contacts']:
-                cont = Contact.query(name=contact['name'])
-                if cont is None:
-                    cont = Contact()
-                for k, v in contact.items():
-                    cont.__setattr__(k, v)
-                organ.contacts.append(cont)
-            organ.save()
-
     def to_omni(self, expand: bool = False):
         from backend.validators.omni_gui_objects import OmniOrganization
         if self.cost_centre:
@@ -151,9 +105,6 @@ class Contact(BaseClass):
                                 secondary=orgs_contacts)  #: relationship to joined organization
     submissions = relationship("BasicSubmission", back_populates="contact")  #: submissions this contact has submitted
 
-    # def __repr__(self) -> str:
-    #     return f"<Contact({self.name})>"
-
     @classproperty
     def searchables(cls):
         return []

src/submissions/backend/db/models/submissions.py

@@ -13,7 +13,6 @@ from zipfile import ZipFile, BadZipfile
 from tempfile import TemporaryDirectory, TemporaryFile
 from operator import itemgetter
 from pprint import pformat
-
 from pandas import DataFrame
 from sqlalchemy.ext.hybrid import hybrid_property
 from . import BaseClass, Reagent, SubmissionType, KitType, Organization, Contact, LogMixin, SubmissionReagentAssociation
@@ -27,10 +26,9 @@ from sqlite3 import OperationalError as SQLOperationalError, IntegrityError as S
 from openpyxl import Workbook
 from openpyxl.drawing.image import Image as OpenpyxlImage
 from tools import row_map, setup_lookup, jinja_template_loading, rreplace, row_keys, check_key_or_attr, Result, Report, \
-    report_result, create_holidays_for_year, check_dictionary_inclusion_equality, rectify_query_date
-from datetime import datetime, date, timedelta
+    report_result, create_holidays_for_year, check_dictionary_inclusion_equality
+from datetime import datetime, date
 from typing import List, Any, Tuple, Literal, Generator, Type
-from dateutil.parser import parse
 from pathlib import Path
 from jinja2.exceptions import TemplateNotFound
 from jinja2 import Template
@@ -271,7 +269,6 @@ class BasicSubmission(BaseClass, LogMixin):
         Returns:
             dict: sample location map
         """
-        # return cls.get_submission_type(submission_type).construct_sample_map()
         return cls.get_submission_type(submission_type).sample_map
 
     def generate_associations(self, name: str, extra: str | None = None):
@@ -445,11 +442,11 @@ class BasicSubmission(BaseClass, LogMixin):
         except Exception as e:
             logger.error(f"Column count error: {e}")
         # NOTE: Get kit associated with this submission
-        logger.debug(f"Checking associations with submission type: {self.submission_type_name}")
+        # logger.debug(f"Checking associations with submission type: {self.submission_type_name}")
         assoc = next((item for item in self.extraction_kit.kit_submissiontype_associations if
                       item.submission_type == self.submission_type),
                      None)
-        logger.debug(f"Got association: {assoc}")
+        # logger.debug(f"Got association: {assoc}")
         # NOTE: If every individual cost is 0 this is probably an old plate.
         if all(item == 0.0 for item in [assoc.constant_cost, assoc.mutable_cost_column, assoc.mutable_cost_sample]):
             try:
@@ -635,16 +632,13 @@ class BasicSubmission(BaseClass, LogMixin):
         # NOTE: No longer searches for association here, done in caller function
         for k, v in input_dict.items():
             try:
-                # logger.debug(f"Setting assoc {assoc} with key {k} to value {v}")
                 setattr(assoc, k, v)
                 # NOTE: for some reason I don't think assoc.__setattr__(k, v) works here.
             except AttributeError:
-                # logger.error(f"Can't set {k} to {v}")
                 pass
         return assoc
 
     def update_reagentassoc(self, reagent: Reagent, role: str):
-        from backend.db import SubmissionReagentAssociation
         # NOTE: get the first reagent assoc that fills the given role.
         try:
             assoc = next(item for item in self.submission_reagent_associations if
@@ -1134,7 +1128,7 @@ class BasicSubmission(BaseClass, LogMixin):
         Returns:
             models.BasicSubmission | List[models.BasicSubmission]: Submission(s) of interest
         """
-        from ... import SubmissionReagentAssociation
+        # from ... import SubmissionReagentAssociation
         # NOTE: if you go back to using 'model' change the appropriate cls to model in the query filters
         if submissiontype is not None:
             model = cls.find_polymorphic_subclass(polymorphic_identity=submissiontype)
@@ -1181,8 +1175,8 @@ class BasicSubmission(BaseClass, LogMixin):
             # #     start_date = start_date.strftime("%Y-%m-%d %H:%M:%S.%f")
             # #     query = query.filter(model.submitted_date == start_date)
             # # else:
-            start_date = rectify_query_date(start_date)
-            end_date = rectify_query_date(end_date, eod=True)
+            start_date = cls.rectify_query_date(start_date)
+            end_date = cls.rectify_query_date(end_date, eod=True)
             query = query.filter(model.submitted_date.between(start_date, end_date))
         # NOTE: by reagent (for some reason)
         match reagent:
@@ -1575,19 +1569,40 @@ class BacterialCulture(BasicSubmission):
                                                       column=lookup_table['sample_columns']['concentration']).value
             yield sample
 
-    def get_provisional_controls(self, controls_only: bool = True):
-        if controls_only:
-            if self.controls:
-                provs = (control.sample for control in self.controls)
-            else:
-                regex = re.compile(r"^(ATCC)|(MCS)|(EN)")
-                provs = (sample for sample in self.samples if bool(regex.match(sample.submitter_id)))
-        else:
-            provs = self.samples
-        for prov in provs:
-            prov.submission = self.rsl_plate_num
-            prov.submitted_date = self.submitted_date
-            yield prov
+    # def get_provisional_controls(self, controls_only: bool = True):
+    def get_provisional_controls(self, include: List[str] = []):
+        # NOTE To ensure Samples are done last.
+        include = sorted(include)
+        logger.debug(include)
+        pos_str = "(ATCC)|(MCS)"
+        pos_regex = re.compile(rf"^{pos_str}")
+        neg_str = "(EN)"
+        neg_regex = re.compile(rf"^{neg_str}")
+        total_str = pos_str + "|" + neg_str
+        total_regex = re.compile(rf"^{total_str}")
+        output = []
+        for item in include:
+            # if self.controls:
+            # logger.debug(item)
+            match item:
+                case "Positive":
+                    if self.controls:
+                        provs = (control.sample for control in self.controls if control.is_positive_control)
+                    else:
+                        provs = (sample for sample in self.samples if bool(pos_regex.match(sample.submitter_id)))
+                case "Negative":
+                    if self.controls:
+                        provs = (control.sample for control in self.controls if not control.is_positive_control)
+                    else:
+                        provs = (sample for sample in self.samples if bool(neg_regex.match(sample.submitter_id)))
+                case _:
+                    provs = (sample for sample in self.samples if not sample.control and sample not in output)
+            for prov in provs:
+                # logger.debug(f"Prov: {prov}")
+                prov.submission = self.rsl_plate_num
+                prov.submitted_date = self.submitted_date
+                output.append(prov)
+        return output
 
 
 class Wastewater(BasicSubmission):
@@ -2794,8 +2809,7 @@ class WastewaterSample(BasicSample):
             output_dict['rsl_number'] = "RSL-WW-" + output_dict['ww_processing_num']
         if output_dict['ww_full_sample_id'] is not None and output_dict["submitter_id"] in disallowed:
             output_dict["submitter_id"] = output_dict['ww_full_sample_id']
-        check = check_key_or_attr("rsl_number", output_dict, check_none=True)
-        # logger.debug(pformat(output_dict, indent=4))
+        # check = check_key_or_attr("rsl_number", output_dict, check_none=True)
         return output_dict
 
     @classproperty
@@ -3089,7 +3103,6 @@ class SubmissionSampleAssociation(BaseClass):
        Returns:
             SubmissionSampleAssociation: Queried or new association.
         """
-        # disallowed = ['id']
         match submission:
             case BasicSubmission():
                 pass
@@ -3184,7 +3197,6 @@ class WastewaterAssociation(SubmissionSampleAssociation):
         sample['background_color'] = f"rgb({red}, {grn}, {blu})"
         try:
             sample[
-                # 'tooltip'] += f"<br>- ct N1: {'{:.2f}'.format(self.ct_n1)} ({self.n1_status})<br>- ct N2: {'{:.2f}'.format(self.ct_n2)} ({self.n2_status})"
                 'tooltip'] += f"<br>- ct N1: {'{:.2f}'.format(self.ct_n1)}<br>- ct N2: {'{:.2f}'.format(self.ct_n2)}"
         except (TypeError, AttributeError) as e:
             logger.error(f"Couldn't set tooltip for {self.sample.rsl_number}. Looks like there isn't PCR data.")