Bug fixes

src/submissions/backend/db/models/__init__.py

@@ -227,9 +227,11 @@ class BaseClass(Base):
         """
         if not objects:
             try:
-                records = [obj.to_sub_dict(**kwargs) for obj in cls.query()]
+                # records = [obj.to_sub_dict(**kwargs) for obj in cls.query()]
+                records = [obj.details_dict(**kwargs) for obj in cls.query()]
             except AttributeError:
-                records = [obj.to_dict(**kwargs) for obj in cls.query(page_size=0)]
+                # records = [obj.to_dict(**kwargs) for obj in cls.query(page_size=0)]
+                records = [obj.details_dict(**kwargs) for obj in cls.query(page_size=0)]
         else:
             try:
                 records = [obj.to_sub_dict(**kwargs) for obj in objects]
@@ -244,7 +246,7 @@ class BaseClass(Base):
         # and not isinstance(v.property, _RelationshipDeclared)]
         sanitized_kwargs = {k: v for k, v in kwargs.items() if k in allowed}
         outside_kwargs = {k: v for k, v in kwargs.items() if k not in allowed}
-        logger.debug(f"Sanitized kwargs: {sanitized_kwargs}")
+        # logger.debug(f"Sanitized kwargs: {sanitized_kwargs}")
         instance = cls.query(**sanitized_kwargs)
         if not instance or isinstance(instance, list):
             instance = cls()
@@ -259,10 +261,10 @@ class BaseClass(Base):
                 from backend.validators.pydant import PydBaseClass
                 if issubclass(v.__class__, PydBaseClass):
                     setattr(instance, k, v.to_sql())
-                else:
+                # else:
-                    logger.error(f"Could not set {k} due to {e}")
+                #     logger.error(f"Could not set {k} due to {e}")
         instance._misc_info.update(outside_kwargs)
-        logger.info(f"Instance from query or create: {instance}, new: {new}")
+        # logger.info(f"Instance from query or create: {instance}, new: {new}")
         return instance, new
 
     @classmethod
@@ -300,7 +302,7 @@ class BaseClass(Base):
         #     logger.debug(f"Incoming query: {query}")
         singles = cls.get_default_info('singles')
         for k, v in kwargs.items():
-            logger.info(f"Using key: {k} with value: {v} against {cls}")
+            # logger.info(f"Using key: {k} with value: {v} against {cls}")
             try:
                 attr = getattr(cls, k)
             except (ArgumentError, AttributeError) as e:
@@ -318,7 +320,7 @@ class BaseClass(Base):
                 except ArgumentError:
                     continue
             else:
-                logger.debug("Single item.")
+                # logger.debug("Single item.")
                 try:
                     query = query.filter(attr == v)
                 except ArgumentError:

src/submissions/backend/db/models/procedures.py

@@ -6,7 +6,7 @@ import zipfile, logging, re
 from operator import itemgetter
 from pprint import pformat
 import numpy as np
-from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey, Interval, Table, FLOAT, BLOB
+from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey, Interval, Table, FLOAT, BLOB, func
 from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.orm import relationship, validates, Query
 from sqlalchemy.ext.associationproxy import association_proxy
@@ -1447,7 +1447,7 @@ class Procedure(BaseClass):
     )  #: Relation to ProcedureReagentAssociation
 
     reagentlot = association_proxy("procedurereagentlotassociation",
-                                "reagent", creator=lambda reg: ProcedureReagentLotAssociation(
+                                "reagentlot", creator=lambda reg: ProcedureReagentLotAssociation(
             reagent=reg))  #: Association proxy to RunReagentAssociation.reagent
 
     procedureequipmentassociation = relationship(
@@ -1477,9 +1477,22 @@ class Procedure(BaseClass):
 
     @classmethod
     @setup_lookup
-    def query(cls, id: int | None = None, name: str | None = None, limit: int = 0, **kwargs) -> Procedure | List[
+    def query(cls, id: int | None = None, name: str | None = None, start_date: date | datetime | str | int | None = None,
+              end_date: date | datetime | str | int | None = None, limit: int = 0, **kwargs) -> Procedure | List[
         Procedure]:
         query: Query = cls.__database_session__.query(cls)
+        if start_date is not None and end_date is None:
+            logger.warning(f"Start date with no end date, using today.")
+            end_date = date.today()
+        if end_date is not None and start_date is None:
+            # NOTE: this query returns a tuple of (object, datetime), need to get only datetime.
+            start_date = cls.__database_session__.query(cls, func.min(cls.submitted_date)).first()[1]
+            logger.warning(f"End date with no start date, using first procedure date: {start_date}")
+        if start_date is not None:
+            start_date = cls.rectify_query_date(start_date)
+            end_date = cls.rectify_query_date(end_date, eod=True)
+            logger.debug(f"Start date: {start_date}, end date: {end_date}")
+            query = query.filter(cls.started_date.between(start_date, end_date))
         match id:
             case int():
                 query = query.filter(cls.id == id)
@@ -1574,8 +1587,8 @@ class Procedure(BaseClass):
 
     def details_dict(self, **kwargs):
         output = super().details_dict()
-        output['kittype'] = output['kittype'].details_dict()
+        # output['kittype'] = output['kittype'].details_dict()
-        output['kit_type'] = self.kittype.name
+        # output['kit_type'] = self.kittype.name
         output['proceduretype'] = output['proceduretype'].details_dict()['name']
         output['results'] = [result.details_dict() for result in output['results']]
         run_samples = [sample for sample in self.run.sample]
@@ -1601,6 +1614,9 @@ class Procedure(BaseClass):
                                "procedurereagentlotassociation",
                                "procedureequipmentassociation", "proceduretipsassociation", "reagent", "equipment",
                                "tips", "control", "kittype"]
+        output['sample_count'] = len(active_samples)
+        output['clientlab'] = self.run.clientsubmission.clientlab.name
+        output['cost'] = 0.00
         # output = self.clean_details_dict(output)
         return output
 
@@ -1650,6 +1666,27 @@ class Procedure(BaseClass):
         from backend.db.models import ProcedureSampleAssociation
         return ProcedureSampleAssociation(procedure=self, sample=sample)
 
+    @classmethod
+    def get_default_info(cls, *args) -> dict | list | str:
+        dicto = super().get_default_info()
+        recover = ['filepath', 'sample', 'csv', 'comment', 'equipment']
+        dicto.update(dict(
+            details_ignore=['excluded', 'reagents', 'sample',
+                            'extraction_info', 'comment', 'barcode',
+                            'platemap', 'export_map', 'equipment', 'tips', 'custom', 'reagentlot',
+                            'procedurereagentassociation'],
+            # NOTE: Fields not placed in ui form
+            form_ignore=['reagents', 'ctx', 'id', 'cost', 'extraction_info', 'signed_by', 'comment', 'namer',
+                         'submission_object', "tips", 'contact_phone', 'custom', 'cost_centre', 'completed_date',
+                         'control', "origin_plate"] + recover,
+            # NOTE: Fields not placed in ui form to be moved to pydantic
+            form_recover=recover
+        ))
+        if args:
+            output = {k: v for k, v in dicto.items() if k in args}
+        else:
+            output = {k: v for k, v in dicto.items()}
+        return output
 
 # class ProcedureTypeKitTypeAssociation(BaseClass):
 #     """
@@ -2118,8 +2155,8 @@ class ProcedureReagentLotAssociation(BaseClass):
         output = super().details_dict()
         # NOTE: Figure out how to merge the misc_info if doing .update instead.
         relevant = {k: v for k, v in output.items() if k not in ['reagent']}
-        output = output['reagent'].details_dict()
+        output = output['reagentlot'].details_dict()
-
+        output['reagent_name'] = self.reagentlot.reagent.name
         misc = output['misc_info']
         output.update(relevant)
         output['reagentrole'] = self.reagentrole
@@ -2538,7 +2575,7 @@ class EquipmentRole(BaseClass):
         # output['process'] = [item.details_dict() for item in output['process']]
         output['process'] = [version.details_dict() for version in
                              flatten_list([process.processversion for process in self.process])]
-        logger.debug(f"\n\nProcess: {pformat(output['process'])}")
+        # logger.debug(f"\n\nProcess: {pformat(output['process'])}")
         try:
             output['tips'] = [item.details_dict() for item in output['tips']]
         except KeyError:
@@ -2848,6 +2885,7 @@ class Process(BaseClass):
     def details_dict(self, **kwargs):
         output = super().details_dict(**kwargs)
         output['processversion'] = [item.details_dict() for item in self.processversion]
+        logger.debug(f"Process output dict: {pformat(output)}")
         return output
 
     def to_pydantic(self):

src/submissions/backend/db/models/submissions.py

@@ -633,7 +633,7 @@ class Run(BaseClass, LogMixin):
             samples = self.generate_associations(name="clientsubmissionsampleassociation")
             equipment = self.generate_associations(name="submission_equipment_associations")
             tips = self.generate_associations(name="submission_tips_associations")
-            procedures = [item.to_dict(full_data=True) for item in self.procedure]
+            procedures = [item.details_dict() for item in self.procedure]
             custom = self.custom
         else:
             samples = None
@@ -696,7 +696,8 @@ class Run(BaseClass, LogMixin):
         output['excluded'] += ['procedure', "runsampleassociation", 'excluded', 'expanded', 'sample', 'id', 'custom',
                                'permission', "clientsubmission"]
         output['sample_count'] = self.sample_count
-        output['client_submission'] = self.clientsubmission.name
+        output['clientsubmission'] = self.clientsubmission.name
+        output['clientlab'] = self.clientsubmission.clientlab
         output['started_date'] = self.started_date
         output['completed_date'] = self.completed_date
         return output
@@ -718,7 +719,8 @@ class Run(BaseClass, LogMixin):
             query_out = cls.query(page_size=0, start_date=start_date, end_date=end_date)
         records = []
         for sub in query_out:
-            output = sub.to_dict(full_data=True)
+            # output = sub.to_dict(full_data=True)
+            output = sub.details_dict()
             for k, v in output.items():
                 if isinstance(v, types.GeneratorType):
                     output[k] = [item for item in v]
@@ -839,7 +841,8 @@ class Run(BaseClass, LogMixin):
             pd.DataFrame: Pandas Dataframe of all relevant procedure
         """
         # NOTE: use lookup function to create list of dicts
-        subs = [item.to_dict() for item in
+        # subs = [item.to_dict() for item in
+        subs = [item.details_dict() for item in
                 cls.query(submissiontype=submission_type, limit=limit, chronologic=chronologic, page=page,
                           page_size=page_size)]
         df = pd.DataFrame.from_records(subs)

src/submissions/backend/excel/parsers/__init__.py

@@ -131,7 +131,8 @@ class DefaultTABLEParser(DefaultParser):
         df = df.dropna(axis=1, how='all')
         for ii, row in enumerate(df.iterrows()):
             output = {}
-            for key, value in row[1].to_dict().items():
+            # for key, value in row[1].to_dict().items():
+            for key, value in row[1].details_dict().items():
                 if isinstance(key, str):
                     key = key.lower().replace(" ", "_")
                     key = re.sub(r"_(\(.*\)|#)", "", key)

src/submissions/backend/excel/reports.py

@@ -7,7 +7,9 @@ from pandas import DataFrame, ExcelWriter
 from pathlib import Path
 from datetime import date
 from typing import Tuple, List
-from backend.db.models import Run
+
+# from backend import Procedure
+from backend.db.models import Procedure, Run
 from tools import jinja_template_loading, get_first_blank_df_row, row_map, flatten_list
 from PyQt6.QtWidgets import QWidget
 from openpyxl.worksheet.worksheet import Worksheet
@@ -45,9 +47,10 @@ class ReportMaker(object):
         self.start_date = start_date
         self.end_date = end_date
         # NOTE: Set page size to zero to override limiting query size.
-        self.runs = Run.query(start_date=start_date, end_date=end_date, page_size=0)
+        # self.runs = Run.query(start_date=start_date, end_date=end_date, page_size=0)
+        self.procedures = Procedure.query(start_date=start_date, end_date=end_date, page_size=0)
         if organizations is not None:
-            self.runs = [run for run in self.runs if run.clientsubmission.clientlab.name in organizations]
+            self.procedures = [procedure for procedure in self.procedures if procedure.run.clientsubmission.clientlab.name in organizations]
         self.detailed_df, self.summary_df = self.make_report_xlsx()
         self.html = self.make_report_html(df=self.summary_df)
 
@@ -58,15 +61,17 @@ class ReportMaker(object):
         Returns:
             DataFrame: output dataframe
         """
-        if not self.runs:
+        if not self.procedures:
             return DataFrame(), DataFrame()
-        df = DataFrame.from_records([item.to_dict(report=True) for item in self.runs])
+        # df = DataFrame.from_records([item.to_dict(report=True) for item in self.runs])
+        df = DataFrame.from_records([item.details_dict() for item in self.procedures])
+        logger.debug(df.columns)
         # NOTE: put procedure with the same lab together
         df = df.sort_values("clientlab")
         # NOTE: aggregate cost and sample count columns
-        df2 = df.groupby(["clientlab", "kittype"]).agg(
+        df2 = df.groupby(["clientlab", "proceduretype"]).agg(
-            {'kittype': 'count', 'cost': 'sum', 'sample_count': 'sum'})
+            {'proceduretype': 'count', 'cost': 'sum', 'sample_count': 'sum'})
-        df2 = df2.rename(columns={"kittype": 'run_count'})
+        df2 = df2.rename(columns={"proceduretype": 'run_count'})
         df = df.drop('id', axis=1)
         df = df.sort_values(['clientlab', "started_date"])
         return df, df2

src/submissions/backend/validators/pydant.py

@@ -25,9 +25,13 @@ logger = logging.getLogger(f"submission.{__name__}")
 
 class PydBaseClass(BaseModel, extra='allow', validate_assignment=True):
 
-    _sql_object: ClassVar = None
+    # _sql_object: ClassVar = None
     key_value_order: ClassVar = []
 
+    @classproperty
+    def _sql_object(cls):
+        return getattr(models, cls.__name__.replace("Pyd", ""))
+
     @model_validator(mode="before")
     @classmethod
     def prevalidate(cls, data):
@@ -36,7 +40,7 @@ class PydBaseClass(BaseModel, extra='allow', validate_assignment=True):
         try:
             items = data.items()
         except AttributeError as e:
-            logger.error(f"Could not prevalidate {cls.__name__} due to {e}")
+            logger.error(f"Could not prevalidate {cls.__name__} due to {e} for {pformat(data)}")
             return data
         for key, value in items:
             new_key = key.replace("_", "")
@@ -67,7 +71,8 @@ class PydBaseClass(BaseModel, extra='allow', validate_assignment=True):
 
     def __init__(self, **data):
         # NOTE: Grab the sql model for validation purposes.
-        self.__class__._sql_object = getattr(models, self.__class__.__name__.replace("Pyd", ""))
+        # self.__class__._sql_object = getattr(models, self.__class__.__name__.replace("Pyd", ""))
+        logger.debug(f"Initial data: {data}")
         super().__init__(**data)
 
     def filter_field(self, key: str) -> Any:
@@ -398,14 +403,17 @@ class PydEquipment(PydBaseClass):
         # if isinstance(value, dict):
         #     value = value['processes']
         if isinstance(value, GeneratorType):
-            value = [item.name for item in value]
+            value = [item for item in value]
         value = convert_nans_to_nones(value)
         if not value:
             value = ['']
         # logger.debug(value)
         try:
             # value = [item.strip() for item in value]
-            value = next((PydProcess(**process.details_dict()) for process in value))
+            d = next((process for process in value), None)
+            logger.debug(f"Next process: {d.detail_dict()}")
+            value = PydProcess(d.details_dict())
+            # value = next((process.to_pydantic() for process in value))
         except AttributeError:
             pass
         return value
@@ -1461,7 +1469,7 @@ class PydProcedure(PydBaseClass, arbitrary_types_allowed=True):
             idx = 0
         insertable = PydReagent(reagentrole=reagentrole, name=name, lot=lot, expiry=expiry)
         self.reagent.insert(idx, insertable)
-        # logger.debug(self.reagent)
+        logger.debug(self.reagent)
 
     @classmethod
     def update_new_reagents(cls, reagent: PydReagent):
@@ -1501,9 +1509,9 @@ class PydProcedure(PydBaseClass, arbitrary_types_allowed=True):
         for reagent in self.reagent:
             if not reagent.lot or reagent.name == "--New--":
                 continue
-            self.update_new_reagents(reagent)
+            # self.update_new_reagents(reagent)
         # NOTE: reset reagent associations.
-        sql.procedurereagentassociation = []
+        # sql.procedurereagentassociation = []
         for reagent in self.reagent:
             if isinstance(reagent, dict):
                 reagent = PydReagent(**reagent)
@@ -1542,12 +1550,13 @@ class PydProcedure(PydBaseClass, arbitrary_types_allowed=True):
                     logger.debug(f"sample {sample_sql} not found in {sql.run.sample}")
                     run_assoc = RunSampleAssociation(sample=sample_sql, run=self.run, row=sample.row,
                                                      column=sample.column)
-                else:
+                # else:
-                    logger.debug(f"sample {sample_sql} found in {sql.run.sample}")
+                #     logger.debug(f"sample {sample_sql} found in {sql.run.sample}")
             if sample_sql not in sql.sample:
                 proc_assoc = ProcedureSampleAssociation(new_id=assoc_id_range[iii], procedure=sql, sample=sample_sql,
                                                         row=sample.row, column=sample.column,
                                                         procedure_rank=sample.procedure_rank)
+        sys.exit(pformat(self.equipment))
         for equipment in self.equipment:
             equip = Equipment.query(name=equipment.name)
             if equip not in sql.equipment:
@@ -1555,8 +1564,6 @@ class PydProcedure(PydBaseClass, arbitrary_types_allowed=True):
                                                             equipmentrole=equip.equipmentrole[0])
                 process = equipment.process.to_sql()
                 equip_assoc.process = process
-        # logger.debug(f"Output sql: {[pformat(item.__dict__) for item in sql.procedureequipmentassociation]}")
-        logger.debug(pformat(sql.__dict__))
         return sql, None
 
 

src/submissions/frontend/widgets/procedure_creation.py

@@ -73,8 +73,8 @@ class ProcedureCreation(QDialog):
         from .equipment_usage_2 import EquipmentUsage
         # logger.debug(f"Edit: {self.edit}")
         proceduretype_dict = self.proceduretype.details_dict()
-        logger.debug(f"Reagent roles: {self.procedure.reagentrole}")
+        # logger.debug(f"Reagent roles: {self.procedure.reagentrole}")
-        logger.debug(f"Equipment roles: {pformat(proceduretype_dict['equipment'])}")
+        # logger.debug(f"Equipment roles: {pformat(proceduretype_dict['equipment'])}")
         # NOTE: Add --New-- as an option for reagents.
         for key, value in self.procedure.reagentrole.items():
             value.append(dict(name="--New--"))
@@ -124,7 +124,7 @@ class ProcedureCreation(QDialog):
         if equipment_of_interest:
             eoi = self.procedure.equipment.pop(self.procedure.equipment.index(equipment_of_interest))
         else:
-            eoi = equipment.to_pydantic(proceduretype=self.procedure.proceduretype)
+            eoi = equipment.to_pydantic(equipmentrole=equipmentrole, proceduretype=self.procedure.proceduretype)
         eoi.name = equipment.name
         eoi.asset_number = equipment.asset_number
         eoi.nickname = equipment.nickname
@@ -185,6 +185,7 @@ class ProcedureCreation(QDialog):
 
     @pyqtSlot(str, str)
     def update_reagent(self, reagentrole: str, name_lot_expiry: str):
+        logger.debug(f"{reagentrole}: {name_lot_expiry}")
         try:
             name, lot, expiry = name_lot_expiry.split(" - ")
         except ValueError as e:

src/submissions/frontend/widgets/submission_details.py

@@ -175,6 +175,7 @@ class SubmissionDetails(QDialog):
         if isinstance(proceduretype, str):
             self.proceduretype = ProcedureType.query(name=proceduretype)
         base_dict = reagent.to_sub_dict(proceduretype=self.proceduretype, full_data=True)
+        # base_dict = reagent.details_dict(proceduretype=self.proceduretype, full_data=True)
         env = jinja_template_loading()
         temp_name = "reagent_details.html"
         try:
@@ -224,7 +225,8 @@ class SubmissionDetails(QDialog):
         if isinstance(run, str):
             run = Run.query(name=run)
         self.rsl_plate_number = run.rsl_plate_number
-        self.base_dict = run.to_dict(full_data=True)
+        # self.base_dict = run.to_dict(full_data=True)
+        self.base_dict = run.details_dict()
         # NOTE: don't want id
         self.base_dict['platemap'] = run.make_plate_map(sample_list=run.hitpicked)
         self.base_dict['excluded'] = run.get_default_info("details_ignore")

src/submissions/frontend/widgets/summary.py

@@ -43,7 +43,7 @@ class Summary(InfoPane):
         orgs = self.org_select.get_checked()
         self.report_obj = ReportMaker(start_date=self.start_date, end_date=self.end_date, organizations=orgs)
         self.webview.setHtml(self.report_obj.html)
-        if self.report_obj.runs:
+        if self.report_obj.procedures:
             self.save_pdf_button.setEnabled(True)
             self.save_excel_button.setEnabled(True)
         else:

src/submissions/templates/js/procedure_form.js

@@ -94,6 +94,7 @@ for(let i = 0; i < reagentRoles.length; i++) {
         }
         new_reg.appendChild(new_form);
     } else {
+        backend.update_reagent(reagentRoles[i].id, reagentRoles[i].value);
         newregform = document.getElementById(reagentRoles[i].id + "_addition");
         try {
             newregform.remove();
@@ -101,16 +102,16 @@ for(let i = 0; i < reagentRoles.length; i++) {
         catch(err) {
             console.log("Missed it.");
         }
-        backend.update_reagent(reagentRoles[i].id, reagentRoles[i].value);
-
-
     }
   });
 };
 
+var equipmentroles = document.getElementsByClassName("equipmentrole");
+
 window.onload = function() {
     for(let i = 0; i < reagentRoles.length; i++) {
         backend.update_reagent(reagentRoles[i].id, reagentRoles[i].value);
     }
+
 }
 

src/submissions/templates/support/reagent_list.html

@@ -1,6 +1,6 @@
 <tr>
     <td style="border: 1px solid black;">{{ reagent['reagentrole'] }}</td>
-    <td style="border: 1px solid black;">{{ reagent['name'] }}</td>
+    <td style="border: 1px solid black;">{{ reagent['reagent_name'] }}</td>
     <td style="border: 1px solid black;">{{ reagent['lot'] }}</td>
     <td style="border: 1px solid black;">{{ reagent['expiry'].strftime('%Y-%m-%d') }}</td>
 </tr>