landon/Submissions-App
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bug fixes

Browse Source
This commit is contained in:
lwark
2025-08-13 09:45:51 -05:00
parent 6380f1e2a9
commit 6f58030e75
11 changed files with 110 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
src/submissions/backend/db/models/__init__.py
View File

@@ -227,9 +227,11 @@ class BaseClass(Base):
"""
if not objects:
try:
records = [obj.to_sub_dict(**kwargs) for obj in cls.query()]
# records = [obj.to_sub_dict(**kwargs) for obj in cls.query()]
records = [obj.details_dict(**kwargs) for obj in cls.query()]
except AttributeError:
records = [obj.to_dict(**kwargs) for obj in cls.query(page_size=0)]
# records = [obj.to_dict(**kwargs) for obj in cls.query(page_size=0)]
records = [obj.details_dict(**kwargs) for obj in cls.query(page_size=0)]
else:
try:
records = [obj.to_sub_dict(**kwargs) for obj in objects]
@@ -244,7 +246,7 @@ class BaseClass(Base):
# and not isinstance(v.property, _RelationshipDeclared)]
sanitized_kwargs = {k: v for k, v in kwargs.items() if k in allowed}
outside_kwargs = {k: v for k, v in kwargs.items() if k not in allowed}
logger.debug(f"Sanitized kwargs: {sanitized_kwargs}")
# logger.debug(f"Sanitized kwargs: {sanitized_kwargs}")
instance = cls.query(**sanitized_kwargs)
if not instance or isinstance(instance, list):
instance = cls()
@@ -259,10 +261,10 @@ class BaseClass(Base):
from backend.validators.pydant import PydBaseClass
if issubclass(v.__class__, PydBaseClass):
setattr(instance, k, v.to_sql())
else:
logger.error(f"Could not set {k} due to {e}")
# else:
# logger.error(f"Could not set {k} due to {e}")
instance._misc_info.update(outside_kwargs)
logger.info(f"Instance from query or create: {instance}, new: {new}")
# logger.info(f"Instance from query or create: {instance}, new: {new}")
return instance, new
@classmethod
@@ -300,7 +302,7 @@ class BaseClass(Base):
# logger.debug(f"Incoming query: {query}")
singles = cls.get_default_info('singles')
for k, v in kwargs.items():
logger.info(f"Using key: {k} with value: {v} against {cls}")
# logger.info(f"Using key: {k} with value: {v} against {cls}")
try:
attr = getattr(cls, k)
except (ArgumentError, AttributeError) as e:
@@ -318,7 +320,7 @@ class BaseClass(Base):
except ArgumentError:
continue
else:
logger.debug("Single item.")
# logger.debug("Single item.")
try:
query = query.filter(attr == v)
except ArgumentError:

54
src/submissions/backend/db/models/procedures.py
View File

@@ -6,7 +6,7 @@ import zipfile, logging, re
from operator import itemgetter
from pprint import pformat
import numpy as np
from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey, Interval, Table, FLOAT, BLOB
from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey, Interval, Table, FLOAT, BLOB, func
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.orm import relationship, validates, Query
from sqlalchemy.ext.associationproxy import association_proxy
@@ -1447,7 +1447,7 @@ class Procedure(BaseClass):
) #: Relation to ProcedureReagentAssociation
reagentlot = association_proxy("procedurereagentlotassociation",
"reagent", creator=lambda reg: ProcedureReagentLotAssociation(
"reagentlot", creator=lambda reg: ProcedureReagentLotAssociation(
reagent=reg)) #: Association proxy to RunReagentAssociation.reagent
procedureequipmentassociation = relationship(
@@ -1477,9 +1477,22 @@ class Procedure(BaseClass):
@classmethod
@setup_lookup
def query(cls, id: int | None = None, name: str | None = None, limit: int = 0, **kwargs) -> Procedure | List[
def query(cls, id: int | None = None, name: str | None = None, start_date: date | datetime | str | int | None = None,
end_date: date | datetime | str | int | None = None, limit: int = 0, **kwargs) -> Procedure | List[
Procedure]:
query: Query = cls.__database_session__.query(cls)
if start_date is not None and end_date is None:
logger.warning(f"Start date with no end date, using today.")
end_date = date.today()
if end_date is not None and start_date is None:
# NOTE: this query returns a tuple of (object, datetime), need to get only datetime.
start_date = cls.__database_session__.query(cls, func.min(cls.submitted_date)).first()[1]
logger.warning(f"End date with no start date, using first procedure date: {start_date}")
if start_date is not None:
start_date = cls.rectify_query_date(start_date)
end_date = cls.rectify_query_date(end_date, eod=True)
logger.debug(f"Start date: {start_date}, end date: {end_date}")
query = query.filter(cls.started_date.between(start_date, end_date))
match id:
case int():
query = query.filter(cls.id == id)
@@ -1574,8 +1587,8 @@ class Procedure(BaseClass):
def details_dict(self, **kwargs):
output = super().details_dict()
output['kittype'] = output['kittype'].details_dict()
output['kit_type'] = self.kittype.name
# output['kittype'] = output['kittype'].details_dict()
# output['kit_type'] = self.kittype.name
output['proceduretype'] = output['proceduretype'].details_dict()['name']
output['results'] = [result.details_dict() for result in output['results']]
run_samples = [sample for sample in self.run.sample]
@@ -1601,6 +1614,9 @@ class Procedure(BaseClass):
"procedurereagentlotassociation",
"procedureequipmentassociation", "proceduretipsassociation", "reagent", "equipment",
"tips", "control", "kittype"]
output['sample_count'] = len(active_samples)
output['clientlab'] = self.run.clientsubmission.clientlab.name
output['cost'] = 0.00
# output = self.clean_details_dict(output)
return output
@@ -1650,6 +1666,27 @@ class Procedure(BaseClass):
from backend.db.models import ProcedureSampleAssociation
return ProcedureSampleAssociation(procedure=self, sample=sample)
@classmethod
def get_default_info(cls, *args) -> dict | list | str:
dicto = super().get_default_info()
recover = ['filepath', 'sample', 'csv', 'comment', 'equipment']
dicto.update(dict(
details_ignore=['excluded', 'reagents', 'sample',
'extraction_info', 'comment', 'barcode',
'platemap', 'export_map', 'equipment', 'tips', 'custom', 'reagentlot',
'procedurereagentassociation'],
# NOTE: Fields not placed in ui form
form_ignore=['reagents', 'ctx', 'id', 'cost', 'extraction_info', 'signed_by', 'comment', 'namer',
'submission_object', "tips", 'contact_phone', 'custom', 'cost_centre', 'completed_date',
'control', "origin_plate"] + recover,
# NOTE: Fields not placed in ui form to be moved to pydantic
form_recover=recover
))
if args:
output = {k: v for k, v in dicto.items() if k in args}
else:
output = {k: v for k, v in dicto.items()}
return output
# class ProcedureTypeKitTypeAssociation(BaseClass):
# """
@@ -2118,8 +2155,8 @@ class ProcedureReagentLotAssociation(BaseClass):
output = super().details_dict()
# NOTE: Figure out how to merge the misc_info if doing .update instead.
relevant = {k: v for k, v in output.items() if k not in ['reagent']}
output = output['reagent'].details_dict()
output = output['reagentlot'].details_dict()
output['reagent_name'] = self.reagentlot.reagent.name
misc = output['misc_info']
output.update(relevant)
output['reagentrole'] = self.reagentrole
@@ -2538,7 +2575,7 @@ class EquipmentRole(BaseClass):
# output['process'] = [item.details_dict() for item in output['process']]
output['process'] = [version.details_dict() for version in
flatten_list([process.processversion for process in self.process])]
logger.debug(f"\n\nProcess: {pformat(output['process'])}")
# logger.debug(f"\n\nProcess: {pformat(output['process'])}")
try:
output['tips'] = [item.details_dict() for item in output['tips']]
except KeyError:
@@ -2848,6 +2885,7 @@ class Process(BaseClass):
def details_dict(self, **kwargs):
output = super().details_dict(**kwargs)
output['processversion'] = [item.details_dict() for item in self.processversion]
logger.debug(f"Process output dict: {pformat(output)}")
return output
def to_pydantic(self):

11
src/submissions/backend/db/models/submissions.py
View File

@@ -633,7 +633,7 @@ class Run(BaseClass, LogMixin):
samples = self.generate_associations(name="clientsubmissionsampleassociation")
equipment = self.generate_associations(name="submission_equipment_associations")
tips = self.generate_associations(name="submission_tips_associations")
procedures = [item.to_dict(full_data=True) for item in self.procedure]
procedures = [item.details_dict() for item in self.procedure]
custom = self.custom
else:
samples = None
@@ -696,7 +696,8 @@ class Run(BaseClass, LogMixin):
output['excluded'] += ['procedure', "runsampleassociation", 'excluded', 'expanded', 'sample', 'id', 'custom',
'permission', "clientsubmission"]
output['sample_count'] = self.sample_count
output['client_submission'] = self.clientsubmission.name
output['clientsubmission'] = self.clientsubmission.name
output['clientlab'] = self.clientsubmission.clientlab
output['started_date'] = self.started_date
output['completed_date'] = self.completed_date
return output
@@ -718,7 +719,8 @@ class Run(BaseClass, LogMixin):
query_out = cls.query(page_size=0, start_date=start_date, end_date=end_date)
records = []
for sub in query_out:
output = sub.to_dict(full_data=True)
# output = sub.to_dict(full_data=True)
output = sub.details_dict()
for k, v in output.items():
if isinstance(v, types.GeneratorType):
output[k] = [item for item in v]
@@ -839,7 +841,8 @@ class Run(BaseClass, LogMixin):
pd.DataFrame: Pandas Dataframe of all relevant procedure
"""
# NOTE: use lookup function to create list of dicts
subs = [item.to_dict() for item in
# subs = [item.to_dict() for item in
subs = [item.details_dict() for item in
cls.query(submissiontype=submission_type, limit=limit, chronologic=chronologic, page=page,
page_size=page_size)]
df = pd.DataFrame.from_records(subs)

3
src/submissions/backend/excel/parsers/__init__.py
View File

@@ -131,7 +131,8 @@ class DefaultTABLEParser(DefaultParser):
df = df.dropna(axis=1, how='all')
for ii, row in enumerate(df.iterrows()):
output = {}
for key, value in row[1].to_dict().items():
# for key, value in row[1].to_dict().items():
for key, value in row[1].details_dict().items():
if isinstance(key, str):
key = key.lower().replace(" ", "_")
key = re.sub(r"_(\(.*\)|#)", "", key)

21
src/submissions/backend/excel/reports.py
View File

@@ -7,7 +7,9 @@ from pandas import DataFrame, ExcelWriter
from pathlib import Path
from datetime import date
from typing import Tuple, List
from backend.db.models import Run
# from backend import Procedure
from backend.db.models import Procedure, Run
from tools import jinja_template_loading, get_first_blank_df_row, row_map, flatten_list
from PyQt6.QtWidgets import QWidget
from openpyxl.worksheet.worksheet import Worksheet
@@ -45,9 +47,10 @@ class ReportMaker(object):
self.start_date = start_date
self.end_date = end_date
# NOTE: Set page size to zero to override limiting query size.
self.runs = Run.query(start_date=start_date, end_date=end_date, page_size=0)
# self.runs = Run.query(start_date=start_date, end_date=end_date, page_size=0)
self.procedures = Procedure.query(start_date=start_date, end_date=end_date, page_size=0)
if organizations is not None:
self.runs = [run for run in self.runs if run.clientsubmission.clientlab.name in organizations]
self.procedures = [procedure for procedure in self.procedures if procedure.run.clientsubmission.clientlab.name in organizations]
self.detailed_df, self.summary_df = self.make_report_xlsx()
self.html = self.make_report_html(df=self.summary_df)
@@ -58,15 +61,17 @@ class ReportMaker(object):
Returns:
DataFrame: output dataframe
"""
if not self.runs:
if not self.procedures:
return DataFrame(), DataFrame()
df = DataFrame.from_records([item.to_dict(report=True) for item in self.runs])
# df = DataFrame.from_records([item.to_dict(report=True) for item in self.runs])
df = DataFrame.from_records([item.details_dict() for item in self.procedures])
logger.debug(df.columns)
# NOTE: put procedure with the same lab together
df = df.sort_values("clientlab")
# NOTE: aggregate cost and sample count columns
df2 = df.groupby(["clientlab", "kittype"]).agg(
{'kittype': 'count', 'cost': 'sum', 'sample_count': 'sum'})
df2 = df2.rename(columns={"kittype": 'run_count'})
df2 = df.groupby(["clientlab", "proceduretype"]).agg(
{'proceduretype': 'count', 'cost': 'sum', 'sample_count': 'sum'})
df2 = df2.rename(columns={"proceduretype": 'run_count'})
df = df.drop('id', axis=1)
df = df.sort_values(['clientlab', "started_date"])
return df, df2

31
src/submissions/backend/validators/pydant.py
View File

@@ -25,9 +25,13 @@ logger = logging.getLogger(f"submission.{__name__}")
class PydBaseClass(BaseModel, extra='allow', validate_assignment=True):
_sql_object: ClassVar = None
# _sql_object: ClassVar = None
key_value_order: ClassVar = []
@classproperty
def _sql_object(cls):
return getattr(models, cls.__name__.replace("Pyd", ""))
@model_validator(mode="before")
@classmethod
def prevalidate(cls, data):
@@ -36,7 +40,7 @@ class PydBaseClass(BaseModel, extra='allow', validate_assignment=True):
try:
items = data.items()
except AttributeError as e:
logger.error(f"Could not prevalidate {cls.__name__} due to {e}")
logger.error(f"Could not prevalidate {cls.__name__} due to {e} for {pformat(data)}")
return data
for key, value in items:
new_key = key.replace("_", "")
@@ -67,7 +71,8 @@ class PydBaseClass(BaseModel, extra='allow', validate_assignment=True):
def __init__(self, **data):
# NOTE: Grab the sql model for validation purposes.
self.__class__._sql_object = getattr(models, self.__class__.__name__.replace("Pyd", ""))
# self.__class__._sql_object = getattr(models, self.__class__.__name__.replace("Pyd", ""))
logger.debug(f"Initial data: {data}")
super().__init__(**data)
def filter_field(self, key: str) -> Any:
@@ -398,14 +403,17 @@ class PydEquipment(PydBaseClass):
# if isinstance(value, dict):
# value = value['processes']
if isinstance(value, GeneratorType):
value = [item.name for item in value]
value = [item for item in value]
value = convert_nans_to_nones(value)
if not value:
value = ['']
# logger.debug(value)
try:
# value = [item.strip() for item in value]
value = next((PydProcess(**process.details_dict()) for process in value))
d = next((process for process in value), None)
logger.debug(f"Next process: {d.detail_dict()}")
value = PydProcess(d.details_dict())
# value = next((process.to_pydantic() for process in value))
except AttributeError:
pass
return value
@@ -1461,7 +1469,7 @@ class PydProcedure(PydBaseClass, arbitrary_types_allowed=True):
idx = 0
insertable = PydReagent(reagentrole=reagentrole, name=name, lot=lot, expiry=expiry)
self.reagent.insert(idx, insertable)
# logger.debug(self.reagent)
logger.debug(self.reagent)
@classmethod
def update_new_reagents(cls, reagent: PydReagent):
@@ -1501,9 +1509,9 @@ class PydProcedure(PydBaseClass, arbitrary_types_allowed=True):
for reagent in self.reagent:
if not reagent.lot or reagent.name == "--New--":
continue
self.update_new_reagents(reagent)
# self.update_new_reagents(reagent)
# NOTE: reset reagent associations.
sql.procedurereagentassociation = []
# sql.procedurereagentassociation = []
for reagent in self.reagent:
if isinstance(reagent, dict):
reagent = PydReagent(**reagent)
@@ -1542,12 +1550,13 @@ class PydProcedure(PydBaseClass, arbitrary_types_allowed=True):
logger.debug(f"sample {sample_sql} not found in {sql.run.sample}")
run_assoc = RunSampleAssociation(sample=sample_sql, run=self.run, row=sample.row,
column=sample.column)
else:
logger.debug(f"sample {sample_sql} found in {sql.run.sample}")
# else:
# logger.debug(f"sample {sample_sql} found in {sql.run.sample}")
if sample_sql not in sql.sample:
proc_assoc = ProcedureSampleAssociation(new_id=assoc_id_range[iii], procedure=sql, sample=sample_sql,
row=sample.row, column=sample.column,
procedure_rank=sample.procedure_rank)
sys.exit(pformat(self.equipment))
for equipment in self.equipment:
equip = Equipment.query(name=equipment.name)
if equip not in sql.equipment:
@@ -1555,8 +1564,6 @@ class PydProcedure(PydBaseClass, arbitrary_types_allowed=True):
equipmentrole=equip.equipmentrole[0])
process = equipment.process.to_sql()
equip_assoc.process = process
# logger.debug(f"Output sql: {[pformat(item.__dict__) for item in sql.procedureequipmentassociation]}")
logger.debug(pformat(sql.__dict__))
return sql, None

7
src/submissions/frontend/widgets/procedure_creation.py
View File

@@ -73,8 +73,8 @@ class ProcedureCreation(QDialog):
from .equipment_usage_2 import EquipmentUsage
# logger.debug(f"Edit: {self.edit}")
proceduretype_dict = self.proceduretype.details_dict()
logger.debug(f"Reagent roles: {self.procedure.reagentrole}")
logger.debug(f"Equipment roles: {pformat(proceduretype_dict['equipment'])}")
# logger.debug(f"Reagent roles: {self.procedure.reagentrole}")
# logger.debug(f"Equipment roles: {pformat(proceduretype_dict['equipment'])}")
# NOTE: Add --New-- as an option for reagents.
for key, value in self.procedure.reagentrole.items():
value.append(dict(name="--New--"))
@@ -124,7 +124,7 @@ class ProcedureCreation(QDialog):
if equipment_of_interest:
eoi = self.procedure.equipment.pop(self.procedure.equipment.index(equipment_of_interest))
else:
eoi = equipment.to_pydantic(proceduretype=self.procedure.proceduretype)
eoi = equipment.to_pydantic(equipmentrole=equipmentrole, proceduretype=self.procedure.proceduretype)
eoi.name = equipment.name
eoi.asset_number = equipment.asset_number
eoi.nickname = equipment.nickname
@@ -185,6 +185,7 @@ class ProcedureCreation(QDialog):
@pyqtSlot(str, str)
def update_reagent(self, reagentrole: str, name_lot_expiry: str):
logger.debug(f"{reagentrole}: {name_lot_expiry}")
try:
name, lot, expiry = name_lot_expiry.split(" - ")
except ValueError as e:

4
src/submissions/frontend/widgets/submission_details.py
View File

@@ -175,6 +175,7 @@ class SubmissionDetails(QDialog):
if isinstance(proceduretype, str):
self.proceduretype = ProcedureType.query(name=proceduretype)
base_dict = reagent.to_sub_dict(proceduretype=self.proceduretype, full_data=True)
# base_dict = reagent.details_dict(proceduretype=self.proceduretype, full_data=True)
env = jinja_template_loading()
temp_name = "reagent_details.html"
try:
@@ -224,7 +225,8 @@ class SubmissionDetails(QDialog):
if isinstance(run, str):
run = Run.query(name=run)
self.rsl_plate_number = run.rsl_plate_number
self.base_dict = run.to_dict(full_data=True)
# self.base_dict = run.to_dict(full_data=True)
self.base_dict = run.details_dict()
# NOTE: don't want id
self.base_dict['platemap'] = run.make_plate_map(sample_list=run.hitpicked)
self.base_dict['excluded'] = run.get_default_info("details_ignore")

2
src/submissions/frontend/widgets/summary.py
View File

@@ -43,7 +43,7 @@ class Summary(InfoPane):
orgs = self.org_select.get_checked()
self.report_obj = ReportMaker(start_date=self.start_date, end_date=self.end_date, organizations=orgs)
self.webview.setHtml(self.report_obj.html)
if self.report_obj.runs:
if self.report_obj.procedures:
self.save_pdf_button.setEnabled(True)
self.save_excel_button.setEnabled(True)
else:

7
src/submissions/templates/js/procedure_form.js
View File

@@ -94,6 +94,7 @@ for(let i = 0; i < reagentRoles.length; i++) {
}
new_reg.appendChild(new_form);
} else {
backend.update_reagent(reagentRoles[i].id, reagentRoles[i].value);
newregform = document.getElementById(reagentRoles[i].id + "_addition");
try {
newregform.remove();
@@ -101,16 +102,16 @@ for(let i = 0; i < reagentRoles.length; i++) {
catch(err) {
console.log("Missed it.");
}
backend.update_reagent(reagentRoles[i].id, reagentRoles[i].value);
}
});
};
var equipmentroles = document.getElementsByClassName("equipmentrole");
window.onload = function() {
for(let i = 0; i < reagentRoles.length; i++) {
backend.update_reagent(reagentRoles[i].id, reagentRoles[i].value);
}
}

2
src/submissions/templates/support/reagent_list.html
View File

@@ -1,6 +1,6 @@
<tr>
<td style="border: 1px solid black;">{{ reagent['reagentrole'] }}</td>
<td style="border: 1px solid black;">{{ reagent['name'] }}</td>
<td style="border: 1px solid black;">{{ reagent['reagent_name'] }}</td>
<td style="border: 1px solid black;">{{ reagent['lot'] }}</td>
<td style="border: 1px solid black;">{{ reagent['expiry'].strftime('%Y-%m-%d') }}</td>
</tr>