diff --git a/src/submissions/backend/db/models/__init__.py b/src/submissions/backend/db/models/__init__.py index 793a171..5084f30 100644 --- a/src/submissions/backend/db/models/__init__.py +++ b/src/submissions/backend/db/models/__init__.py @@ -227,9 +227,11 @@ class BaseClass(Base): """ if not objects: try: - records = [obj.to_sub_dict(**kwargs) for obj in cls.query()] + # records = [obj.to_sub_dict(**kwargs) for obj in cls.query()] + records = [obj.details_dict(**kwargs) for obj in cls.query()] except AttributeError: - records = [obj.to_dict(**kwargs) for obj in cls.query(page_size=0)] + # records = [obj.to_dict(**kwargs) for obj in cls.query(page_size=0)] + records = [obj.details_dict(**kwargs) for obj in cls.query(page_size=0)] else: try: records = [obj.to_sub_dict(**kwargs) for obj in objects] @@ -244,7 +246,7 @@ class BaseClass(Base): # and not isinstance(v.property, _RelationshipDeclared)] sanitized_kwargs = {k: v for k, v in kwargs.items() if k in allowed} outside_kwargs = {k: v for k, v in kwargs.items() if k not in allowed} - logger.debug(f"Sanitized kwargs: {sanitized_kwargs}") + # logger.debug(f"Sanitized kwargs: {sanitized_kwargs}") instance = cls.query(**sanitized_kwargs) if not instance or isinstance(instance, list): instance = cls() @@ -259,10 +261,10 @@ class BaseClass(Base): from backend.validators.pydant import PydBaseClass if issubclass(v.__class__, PydBaseClass): setattr(instance, k, v.to_sql()) - else: - logger.error(f"Could not set {k} due to {e}") + # else: + # logger.error(f"Could not set {k} due to {e}") instance._misc_info.update(outside_kwargs) - logger.info(f"Instance from query or create: {instance}, new: {new}") + # logger.info(f"Instance from query or create: {instance}, new: {new}") return instance, new @classmethod @@ -300,7 +302,7 @@ class BaseClass(Base): # logger.debug(f"Incoming query: {query}") singles = cls.get_default_info('singles') for k, v in kwargs.items(): - logger.info(f"Using key: {k} with value: {v} against {cls}") + # logger.info(f"Using key: {k} with value: {v} against {cls}") try: attr = getattr(cls, k) except (ArgumentError, AttributeError) as e: @@ -318,7 +320,7 @@ class BaseClass(Base): except ArgumentError: continue else: - logger.debug("Single item.") + # logger.debug("Single item.") try: query = query.filter(attr == v) except ArgumentError: diff --git a/src/submissions/backend/db/models/procedures.py b/src/submissions/backend/db/models/procedures.py index 87ae3f7..4971074 100644 --- a/src/submissions/backend/db/models/procedures.py +++ b/src/submissions/backend/db/models/procedures.py @@ -6,7 +6,7 @@ import zipfile, logging, re from operator import itemgetter from pprint import pformat import numpy as np -from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey, Interval, Table, FLOAT, BLOB +from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey, Interval, Table, FLOAT, BLOB, func from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import relationship, validates, Query from sqlalchemy.ext.associationproxy import association_proxy @@ -1447,7 +1447,7 @@ class Procedure(BaseClass): ) #: Relation to ProcedureReagentAssociation reagentlot = association_proxy("procedurereagentlotassociation", - "reagent", creator=lambda reg: ProcedureReagentLotAssociation( + "reagentlot", creator=lambda reg: ProcedureReagentLotAssociation( reagent=reg)) #: Association proxy to RunReagentAssociation.reagent procedureequipmentassociation = relationship( @@ -1477,9 +1477,22 @@ class Procedure(BaseClass): @classmethod @setup_lookup - def query(cls, id: int | None = None, name: str | None = None, limit: int = 0, **kwargs) -> Procedure | List[ + def query(cls, id: int | None = None, name: str | None = None, start_date: date | datetime | str | int | None = None, + end_date: date | datetime | str | int | None = None, limit: int = 0, **kwargs) -> Procedure | List[ Procedure]: query: Query = cls.__database_session__.query(cls) + if start_date is not None and end_date is None: + logger.warning(f"Start date with no end date, using today.") + end_date = date.today() + if end_date is not None and start_date is None: + # NOTE: this query returns a tuple of (object, datetime), need to get only datetime. + start_date = cls.__database_session__.query(cls, func.min(cls.submitted_date)).first()[1] + logger.warning(f"End date with no start date, using first procedure date: {start_date}") + if start_date is not None: + start_date = cls.rectify_query_date(start_date) + end_date = cls.rectify_query_date(end_date, eod=True) + logger.debug(f"Start date: {start_date}, end date: {end_date}") + query = query.filter(cls.started_date.between(start_date, end_date)) match id: case int(): query = query.filter(cls.id == id) @@ -1574,8 +1587,8 @@ class Procedure(BaseClass): def details_dict(self, **kwargs): output = super().details_dict() - output['kittype'] = output['kittype'].details_dict() - output['kit_type'] = self.kittype.name + # output['kittype'] = output['kittype'].details_dict() + # output['kit_type'] = self.kittype.name output['proceduretype'] = output['proceduretype'].details_dict()['name'] output['results'] = [result.details_dict() for result in output['results']] run_samples = [sample for sample in self.run.sample] @@ -1601,6 +1614,9 @@ class Procedure(BaseClass): "procedurereagentlotassociation", "procedureequipmentassociation", "proceduretipsassociation", "reagent", "equipment", "tips", "control", "kittype"] + output['sample_count'] = len(active_samples) + output['clientlab'] = self.run.clientsubmission.clientlab.name + output['cost'] = 0.00 # output = self.clean_details_dict(output) return output @@ -1650,6 +1666,27 @@ class Procedure(BaseClass): from backend.db.models import ProcedureSampleAssociation return ProcedureSampleAssociation(procedure=self, sample=sample) + @classmethod + def get_default_info(cls, *args) -> dict | list | str: + dicto = super().get_default_info() + recover = ['filepath', 'sample', 'csv', 'comment', 'equipment'] + dicto.update(dict( + details_ignore=['excluded', 'reagents', 'sample', + 'extraction_info', 'comment', 'barcode', + 'platemap', 'export_map', 'equipment', 'tips', 'custom', 'reagentlot', + 'procedurereagentassociation'], + # NOTE: Fields not placed in ui form + form_ignore=['reagents', 'ctx', 'id', 'cost', 'extraction_info', 'signed_by', 'comment', 'namer', + 'submission_object', "tips", 'contact_phone', 'custom', 'cost_centre', 'completed_date', + 'control', "origin_plate"] + recover, + # NOTE: Fields not placed in ui form to be moved to pydantic + form_recover=recover + )) + if args: + output = {k: v for k, v in dicto.items() if k in args} + else: + output = {k: v for k, v in dicto.items()} + return output # class ProcedureTypeKitTypeAssociation(BaseClass): # """ @@ -2118,8 +2155,8 @@ class ProcedureReagentLotAssociation(BaseClass): output = super().details_dict() # NOTE: Figure out how to merge the misc_info if doing .update instead. relevant = {k: v for k, v in output.items() if k not in ['reagent']} - output = output['reagent'].details_dict() - + output = output['reagentlot'].details_dict() + output['reagent_name'] = self.reagentlot.reagent.name misc = output['misc_info'] output.update(relevant) output['reagentrole'] = self.reagentrole @@ -2538,7 +2575,7 @@ class EquipmentRole(BaseClass): # output['process'] = [item.details_dict() for item in output['process']] output['process'] = [version.details_dict() for version in flatten_list([process.processversion for process in self.process])] - logger.debug(f"\n\nProcess: {pformat(output['process'])}") + # logger.debug(f"\n\nProcess: {pformat(output['process'])}") try: output['tips'] = [item.details_dict() for item in output['tips']] except KeyError: @@ -2848,6 +2885,7 @@ class Process(BaseClass): def details_dict(self, **kwargs): output = super().details_dict(**kwargs) output['processversion'] = [item.details_dict() for item in self.processversion] + logger.debug(f"Process output dict: {pformat(output)}") return output def to_pydantic(self): diff --git a/src/submissions/backend/db/models/submissions.py b/src/submissions/backend/db/models/submissions.py index bb14676..89b9ce5 100644 --- a/src/submissions/backend/db/models/submissions.py +++ b/src/submissions/backend/db/models/submissions.py @@ -633,7 +633,7 @@ class Run(BaseClass, LogMixin): samples = self.generate_associations(name="clientsubmissionsampleassociation") equipment = self.generate_associations(name="submission_equipment_associations") tips = self.generate_associations(name="submission_tips_associations") - procedures = [item.to_dict(full_data=True) for item in self.procedure] + procedures = [item.details_dict() for item in self.procedure] custom = self.custom else: samples = None @@ -696,7 +696,8 @@ class Run(BaseClass, LogMixin): output['excluded'] += ['procedure', "runsampleassociation", 'excluded', 'expanded', 'sample', 'id', 'custom', 'permission', "clientsubmission"] output['sample_count'] = self.sample_count - output['client_submission'] = self.clientsubmission.name + output['clientsubmission'] = self.clientsubmission.name + output['clientlab'] = self.clientsubmission.clientlab output['started_date'] = self.started_date output['completed_date'] = self.completed_date return output @@ -718,7 +719,8 @@ class Run(BaseClass, LogMixin): query_out = cls.query(page_size=0, start_date=start_date, end_date=end_date) records = [] for sub in query_out: - output = sub.to_dict(full_data=True) + # output = sub.to_dict(full_data=True) + output = sub.details_dict() for k, v in output.items(): if isinstance(v, types.GeneratorType): output[k] = [item for item in v] @@ -839,7 +841,8 @@ class Run(BaseClass, LogMixin): pd.DataFrame: Pandas Dataframe of all relevant procedure """ # NOTE: use lookup function to create list of dicts - subs = [item.to_dict() for item in + # subs = [item.to_dict() for item in + subs = [item.details_dict() for item in cls.query(submissiontype=submission_type, limit=limit, chronologic=chronologic, page=page, page_size=page_size)] df = pd.DataFrame.from_records(subs) diff --git a/src/submissions/backend/excel/parsers/__init__.py b/src/submissions/backend/excel/parsers/__init__.py index 7f12dc1..fde094a 100644 --- a/src/submissions/backend/excel/parsers/__init__.py +++ b/src/submissions/backend/excel/parsers/__init__.py @@ -131,7 +131,8 @@ class DefaultTABLEParser(DefaultParser): df = df.dropna(axis=1, how='all') for ii, row in enumerate(df.iterrows()): output = {} - for key, value in row[1].to_dict().items(): + # for key, value in row[1].to_dict().items(): + for key, value in row[1].details_dict().items(): if isinstance(key, str): key = key.lower().replace(" ", "_") key = re.sub(r"_(\(.*\)|#)", "", key) diff --git a/src/submissions/backend/excel/reports.py b/src/submissions/backend/excel/reports.py index 5edbb0c..9159892 100644 --- a/src/submissions/backend/excel/reports.py +++ b/src/submissions/backend/excel/reports.py @@ -7,7 +7,9 @@ from pandas import DataFrame, ExcelWriter from pathlib import Path from datetime import date from typing import Tuple, List -from backend.db.models import Run + +# from backend import Procedure +from backend.db.models import Procedure, Run from tools import jinja_template_loading, get_first_blank_df_row, row_map, flatten_list from PyQt6.QtWidgets import QWidget from openpyxl.worksheet.worksheet import Worksheet @@ -45,9 +47,10 @@ class ReportMaker(object): self.start_date = start_date self.end_date = end_date # NOTE: Set page size to zero to override limiting query size. - self.runs = Run.query(start_date=start_date, end_date=end_date, page_size=0) + # self.runs = Run.query(start_date=start_date, end_date=end_date, page_size=0) + self.procedures = Procedure.query(start_date=start_date, end_date=end_date, page_size=0) if organizations is not None: - self.runs = [run for run in self.runs if run.clientsubmission.clientlab.name in organizations] + self.procedures = [procedure for procedure in self.procedures if procedure.run.clientsubmission.clientlab.name in organizations] self.detailed_df, self.summary_df = self.make_report_xlsx() self.html = self.make_report_html(df=self.summary_df) @@ -58,15 +61,17 @@ class ReportMaker(object): Returns: DataFrame: output dataframe """ - if not self.runs: + if not self.procedures: return DataFrame(), DataFrame() - df = DataFrame.from_records([item.to_dict(report=True) for item in self.runs]) + # df = DataFrame.from_records([item.to_dict(report=True) for item in self.runs]) + df = DataFrame.from_records([item.details_dict() for item in self.procedures]) + logger.debug(df.columns) # NOTE: put procedure with the same lab together df = df.sort_values("clientlab") # NOTE: aggregate cost and sample count columns - df2 = df.groupby(["clientlab", "kittype"]).agg( - {'kittype': 'count', 'cost': 'sum', 'sample_count': 'sum'}) - df2 = df2.rename(columns={"kittype": 'run_count'}) + df2 = df.groupby(["clientlab", "proceduretype"]).agg( + {'proceduretype': 'count', 'cost': 'sum', 'sample_count': 'sum'}) + df2 = df2.rename(columns={"proceduretype": 'run_count'}) df = df.drop('id', axis=1) df = df.sort_values(['clientlab', "started_date"]) return df, df2 diff --git a/src/submissions/backend/validators/pydant.py b/src/submissions/backend/validators/pydant.py index be80510..751b225 100644 --- a/src/submissions/backend/validators/pydant.py +++ b/src/submissions/backend/validators/pydant.py @@ -25,9 +25,13 @@ logger = logging.getLogger(f"submission.{__name__}") class PydBaseClass(BaseModel, extra='allow', validate_assignment=True): - _sql_object: ClassVar = None + # _sql_object: ClassVar = None key_value_order: ClassVar = [] + @classproperty + def _sql_object(cls): + return getattr(models, cls.__name__.replace("Pyd", "")) + @model_validator(mode="before") @classmethod def prevalidate(cls, data): @@ -36,7 +40,7 @@ class PydBaseClass(BaseModel, extra='allow', validate_assignment=True): try: items = data.items() except AttributeError as e: - logger.error(f"Could not prevalidate {cls.__name__} due to {e}") + logger.error(f"Could not prevalidate {cls.__name__} due to {e} for {pformat(data)}") return data for key, value in items: new_key = key.replace("_", "") @@ -67,7 +71,8 @@ class PydBaseClass(BaseModel, extra='allow', validate_assignment=True): def __init__(self, **data): # NOTE: Grab the sql model for validation purposes. - self.__class__._sql_object = getattr(models, self.__class__.__name__.replace("Pyd", "")) + # self.__class__._sql_object = getattr(models, self.__class__.__name__.replace("Pyd", "")) + logger.debug(f"Initial data: {data}") super().__init__(**data) def filter_field(self, key: str) -> Any: @@ -398,14 +403,17 @@ class PydEquipment(PydBaseClass): # if isinstance(value, dict): # value = value['processes'] if isinstance(value, GeneratorType): - value = [item.name for item in value] + value = [item for item in value] value = convert_nans_to_nones(value) if not value: value = [''] # logger.debug(value) try: # value = [item.strip() for item in value] - value = next((PydProcess(**process.details_dict()) for process in value)) + d = next((process for process in value), None) + logger.debug(f"Next process: {d.detail_dict()}") + value = PydProcess(d.details_dict()) + # value = next((process.to_pydantic() for process in value)) except AttributeError: pass return value @@ -1461,7 +1469,7 @@ class PydProcedure(PydBaseClass, arbitrary_types_allowed=True): idx = 0 insertable = PydReagent(reagentrole=reagentrole, name=name, lot=lot, expiry=expiry) self.reagent.insert(idx, insertable) - # logger.debug(self.reagent) + logger.debug(self.reagent) @classmethod def update_new_reagents(cls, reagent: PydReagent): @@ -1501,9 +1509,9 @@ class PydProcedure(PydBaseClass, arbitrary_types_allowed=True): for reagent in self.reagent: if not reagent.lot or reagent.name == "--New--": continue - self.update_new_reagents(reagent) + # self.update_new_reagents(reagent) # NOTE: reset reagent associations. - sql.procedurereagentassociation = [] + # sql.procedurereagentassociation = [] for reagent in self.reagent: if isinstance(reagent, dict): reagent = PydReagent(**reagent) @@ -1542,12 +1550,13 @@ class PydProcedure(PydBaseClass, arbitrary_types_allowed=True): logger.debug(f"sample {sample_sql} not found in {sql.run.sample}") run_assoc = RunSampleAssociation(sample=sample_sql, run=self.run, row=sample.row, column=sample.column) - else: - logger.debug(f"sample {sample_sql} found in {sql.run.sample}") + # else: + # logger.debug(f"sample {sample_sql} found in {sql.run.sample}") if sample_sql not in sql.sample: proc_assoc = ProcedureSampleAssociation(new_id=assoc_id_range[iii], procedure=sql, sample=sample_sql, row=sample.row, column=sample.column, procedure_rank=sample.procedure_rank) + sys.exit(pformat(self.equipment)) for equipment in self.equipment: equip = Equipment.query(name=equipment.name) if equip not in sql.equipment: @@ -1555,8 +1564,6 @@ class PydProcedure(PydBaseClass, arbitrary_types_allowed=True): equipmentrole=equip.equipmentrole[0]) process = equipment.process.to_sql() equip_assoc.process = process - # logger.debug(f"Output sql: {[pformat(item.__dict__) for item in sql.procedureequipmentassociation]}") - logger.debug(pformat(sql.__dict__)) return sql, None diff --git a/src/submissions/frontend/widgets/procedure_creation.py b/src/submissions/frontend/widgets/procedure_creation.py index 7f7355d..d646c85 100644 --- a/src/submissions/frontend/widgets/procedure_creation.py +++ b/src/submissions/frontend/widgets/procedure_creation.py @@ -73,8 +73,8 @@ class ProcedureCreation(QDialog): from .equipment_usage_2 import EquipmentUsage # logger.debug(f"Edit: {self.edit}") proceduretype_dict = self.proceduretype.details_dict() - logger.debug(f"Reagent roles: {self.procedure.reagentrole}") - logger.debug(f"Equipment roles: {pformat(proceduretype_dict['equipment'])}") + # logger.debug(f"Reagent roles: {self.procedure.reagentrole}") + # logger.debug(f"Equipment roles: {pformat(proceduretype_dict['equipment'])}") # NOTE: Add --New-- as an option for reagents. for key, value in self.procedure.reagentrole.items(): value.append(dict(name="--New--")) @@ -124,7 +124,7 @@ class ProcedureCreation(QDialog): if equipment_of_interest: eoi = self.procedure.equipment.pop(self.procedure.equipment.index(equipment_of_interest)) else: - eoi = equipment.to_pydantic(proceduretype=self.procedure.proceduretype) + eoi = equipment.to_pydantic(equipmentrole=equipmentrole, proceduretype=self.procedure.proceduretype) eoi.name = equipment.name eoi.asset_number = equipment.asset_number eoi.nickname = equipment.nickname @@ -185,6 +185,7 @@ class ProcedureCreation(QDialog): @pyqtSlot(str, str) def update_reagent(self, reagentrole: str, name_lot_expiry: str): + logger.debug(f"{reagentrole}: {name_lot_expiry}") try: name, lot, expiry = name_lot_expiry.split(" - ") except ValueError as e: diff --git a/src/submissions/frontend/widgets/submission_details.py b/src/submissions/frontend/widgets/submission_details.py index 8c080bc..aa3cd6b 100644 --- a/src/submissions/frontend/widgets/submission_details.py +++ b/src/submissions/frontend/widgets/submission_details.py @@ -175,6 +175,7 @@ class SubmissionDetails(QDialog): if isinstance(proceduretype, str): self.proceduretype = ProcedureType.query(name=proceduretype) base_dict = reagent.to_sub_dict(proceduretype=self.proceduretype, full_data=True) + # base_dict = reagent.details_dict(proceduretype=self.proceduretype, full_data=True) env = jinja_template_loading() temp_name = "reagent_details.html" try: @@ -224,7 +225,8 @@ class SubmissionDetails(QDialog): if isinstance(run, str): run = Run.query(name=run) self.rsl_plate_number = run.rsl_plate_number - self.base_dict = run.to_dict(full_data=True) + # self.base_dict = run.to_dict(full_data=True) + self.base_dict = run.details_dict() # NOTE: don't want id self.base_dict['platemap'] = run.make_plate_map(sample_list=run.hitpicked) self.base_dict['excluded'] = run.get_default_info("details_ignore") diff --git a/src/submissions/frontend/widgets/summary.py b/src/submissions/frontend/widgets/summary.py index f546f52..056b1d6 100644 --- a/src/submissions/frontend/widgets/summary.py +++ b/src/submissions/frontend/widgets/summary.py @@ -43,7 +43,7 @@ class Summary(InfoPane): orgs = self.org_select.get_checked() self.report_obj = ReportMaker(start_date=self.start_date, end_date=self.end_date, organizations=orgs) self.webview.setHtml(self.report_obj.html) - if self.report_obj.runs: + if self.report_obj.procedures: self.save_pdf_button.setEnabled(True) self.save_excel_button.setEnabled(True) else: diff --git a/src/submissions/templates/js/procedure_form.js b/src/submissions/templates/js/procedure_form.js index 627d56b..f628248 100644 --- a/src/submissions/templates/js/procedure_form.js +++ b/src/submissions/templates/js/procedure_form.js @@ -94,6 +94,7 @@ for(let i = 0; i < reagentRoles.length; i++) { } new_reg.appendChild(new_form); } else { + backend.update_reagent(reagentRoles[i].id, reagentRoles[i].value); newregform = document.getElementById(reagentRoles[i].id + "_addition"); try { newregform.remove(); @@ -101,16 +102,16 @@ for(let i = 0; i < reagentRoles.length; i++) { catch(err) { console.log("Missed it."); } - backend.update_reagent(reagentRoles[i].id, reagentRoles[i].value); - - } }); }; +var equipmentroles = document.getElementsByClassName("equipmentrole"); + window.onload = function() { for(let i = 0; i < reagentRoles.length; i++) { backend.update_reagent(reagentRoles[i].id, reagentRoles[i].value); } + } diff --git a/src/submissions/templates/support/reagent_list.html b/src/submissions/templates/support/reagent_list.html index b295470..b1ecf91 100644 --- a/src/submissions/templates/support/reagent_list.html +++ b/src/submissions/templates/support/reagent_list.html @@ -1,6 +1,6 @@