landon/Submissions-App
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Addition of procedure parser in import.

Browse Source
This commit is contained in:
lwark
2025-06-17 15:09:51 -05:00
parent 0233bc3ac2
commit d8c3f3bbb2
31 changed files with 688 additions and 304 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/submissions/backend/db/models/__init__.py
View File

@@ -2,7 +2,8 @@
Contains all models for sqlalchemy
"""
from __future__ import annotations
import sys, logging
import sys, logging, json
from dateutil.parser import parse
from pandas import DataFrame
@@ -565,15 +566,24 @@ class BaseClass(Base):
check = False
if check:
continue
value = getattr(self, k)
try:
value = getattr(self, k)
except AttributeError:
continue
match value:
case datetime():
value = value.strftime("%Y-%m-%d %H:%M:%S")
case _:
pass
output[k] = value
output[k.strip("_")] = value
return output
def show_details(self, obj):
logger.debug("Show Details")
from frontend.widgets.submission_details import SubmissionDetails
dlg = SubmissionDetails(parent=obj, sub=self)
if dlg.exec():
pass
class LogMixin(Base):
tracking_exclusion: ClassVar = ['artic_technician', 'clientsubmissionsampleassociation',

96
src/submissions/backend/db/models/kits.py
View File

@@ -2,19 +2,16 @@
All kittype and reagent related models
"""
from __future__ import annotations
import json, zipfile, yaml, logging, re, sys
import zipfile, logging, re
from operator import itemgetter
from pprint import pformat
import numpy as np
from jinja2 import Template, TemplateNotFound
from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey, Interval, Table, FLOAT, BLOB
from sqlalchemy.orm import relationship, validates, Query
from sqlalchemy.ext.associationproxy import association_proxy
from sqlalchemy.ext.hybrid import hybrid_property
from datetime import date, datetime, timedelta
from tools import check_authorization, setup_lookup, Report, Result, check_regex_match, yaml_regex_creator, timezone, \
jinja_template_loading, ctx
from tools import check_authorization, setup_lookup, Report, Result, check_regex_match, timezone, \
jinja_template_loading
from typing import List, Literal, Generator, Any, Tuple, TYPE_CHECKING
from pandas import ExcelFile
from pathlib import Path
@@ -632,7 +629,7 @@ class Reagent(BaseClass, LogMixin):
missing=False
)
if full_data:
output['procedure'] = [sub.rsl_plate_num for sub in self.procedures]
output['procedure'] = [sub.rsl_plate_number for sub in self.procedures]
output['excluded'] = ['missing', 'procedure', 'excluded', 'editable']
output['editable'] = ['lot', 'expiry']
return output
@@ -743,7 +740,7 @@ class Reagent(BaseClass, LogMixin):
role = ReagentRole.query(name=value, limit=1)
case _:
return
if role and role not in self.role:
if role and role not in self.reagentrole:
self.reagentrole.append(role)
return
case "comment":
@@ -1097,9 +1094,13 @@ class ProcedureType(BaseClass):
id = Column(INTEGER, primary_key=True)
name = Column(String(64))
reagent_map = Column(JSON)
info_map = Column(JSON)
sample_map = Column(JSON)
equipment_map = Column(JSON)
plate_columns = Column(INTEGER, default=0)
plate_rows = Column(INTEGER, default=0)
allowed_result_methods = Column(JSON)
template_file = Column(BLOB)
procedure = relationship("Procedure",
back_populates="proceduretype") #: Concrete control of this type.
@@ -1218,6 +1219,15 @@ class ProcedureType(BaseClass):
plate_columns=self.plate_columns
)
def details_dict(self, **kwargs):
output = super().details_dict(**kwargs)
output['kittype'] = [item.details_dict() for item in output['kittype']]
# output['process'] = [item.details_dict() for item in output['process']]
output['equipment'] = [item.details_dict() for item in output['equipment']]
return output
def construct_dummy_procedure(self, run: Run|None=None):
from backend.validators.pydant import PydProcedure
if run:
@@ -1277,6 +1287,7 @@ class Procedure(BaseClass):
id = Column(INTEGER, primary_key=True)
name = Column(String, unique=True)
repeat = Column(INTEGER, nullable=False)
technician = Column(String(64)) #: name of processing tech(s)
results = relationship("Results", back_populates="procedure", uselist=True)
proceduretype_id = Column(INTEGER, ForeignKey("_proceduretype.id", ondelete="SET NULL",
@@ -1372,7 +1383,7 @@ class Procedure(BaseClass):
def add_results(self, obj, resultstype_name:str):
logger.debug(f"Add Results! {resultstype_name}")
from frontend.widgets import results
from ...managers import results
results_class = getattr(results, resultstype_name)
rs = results_class(procedure=self, parent=obj)
@@ -1412,8 +1423,8 @@ class Procedure(BaseClass):
def add_comment(self, obj):
logger.debug("Add Comment!")
def show_details(self, obj):
logger.debug("Show Details!")
# def show_details(self, obj):
# logger.debug("Show Details!")
def delete(self, obj):
logger.debug("Delete!")
@@ -1423,10 +1434,25 @@ class Procedure(BaseClass):
output['kittype'] = output['kittype'].details_dict()
output['proceduretype'] = output['proceduretype'].details_dict()
output['results'] = [result.details_dict() for result in output['results']]
output['sample'] = [sample.details_dict() for sample in output['sample']]
run_samples = [sample for sample in self.run.sample]
active_samples = [sample.details_dict() for sample in output['proceduresampleassociation']
if sample.sample.sample_id in [s.sample_id for s in run_samples]]
for sample in active_samples:
sample['active'] = True
inactive_samples = [sample.details_dict() for sample in run_samples if sample.name not in [s['sample_id'] for s in active_samples]]
# logger.debug(f"Inactive samples:{pformat(inactive_samples)}")
for sample in inactive_samples:
sample['active'] = False
# output['sample'] = [sample.details_dict() for sample in output['runsampleassociation']]
output['sample'] = active_samples + inactive_samples
# output['sample'] = [sample.details_dict() for sample in output['sample']]
output['reagent'] = [reagent.details_dict() for reagent in output['procedurereagentassociation']]
output['equipment'] = [equipment.details_dict() for equipment in output['procedureequipmentassociation']]
output['tips'] = [tips.details_dict() for tips in output['proceduretipsassociation']]
output['repeat'] = bool(output['repeat'])
output['excluded'] = ['id', "results", "proceduresampleassociation", "sample", "procedurereagentassociation",
"procedureequipmentassociation", "proceduretipsassociation", "reagent", "equipment", "tips",
"excluded"]
return output
class ProcedureTypeKitTypeAssociation(BaseClass):
@@ -1814,7 +1840,7 @@ class ProcedureReagentAssociation(BaseClass):
str: Representation of this RunReagentAssociation
"""
try:
return f"<ProcedureReagentAssociation({self.procedure.procedure.rsl_plate_num} & {self.reagent.lot})>"
return f"<ProcedureReagentAssociation({self.procedure.procedure.rsl_plate_number} & {self.reagent.lot})>"
except AttributeError:
logger.error(f"Reagent {self.reagent.lot} procedure association {self.reagent_id} has no procedure!")
return f"<ProcedureReagentAssociation(Unknown Submission & {self.reagent.lot})>"
@@ -1887,9 +1913,9 @@ class ProcedureReagentAssociation(BaseClass):
# NOTE: Figure out how to merge the misc_info if doing .update instead.
relevant = {k: v for k, v in output.items() if k not in ['reagent']}
output = output['reagent'].details_dict()
misc = output['_misc_info']
misc = output['misc_info']
output.update(relevant)
output['_misc_info'] = misc
output['misc_info'] = misc
output['results'] = [result.details_dict() for result in output['results']]
return output
@@ -2087,9 +2113,9 @@ class Equipment(BaseClass, LogMixin):
asset_number=self.asset_number
)
if full_data:
subs = [dict(plate=item.procedure.procedure.rsl_plate_num, process=item.process.name,
subs = [dict(plate=item.procedure.procedure.rsl_plate_number, process=item.process.name,
sub_date=item.procedure.procedure.start_date)
if item.process else dict(plate=item.procedure.procedure.rsl_plate_num, process="NA")
if item.process else dict(plate=item.procedure.procedure.rsl_plate_number, process="NA")
for item in self.equipmentprocedureassociation]
output['procedure'] = sorted(subs, key=itemgetter("sub_date"), reverse=True)
output['excluded'] = ['missing', 'procedure', 'excluded', 'editable']
@@ -2240,6 +2266,11 @@ class EquipmentRole(BaseClass):
from backend.validators.omni_gui_objects import OmniEquipmentRole
return OmniEquipmentRole(instance_object=self, name=self.name)
def details_dict(self, **kwargs):
output = super().details_dict(**kwargs)
output['equipment'] = [item.details_dict() for item in output['equipment']]
output['process'] = [item.details_dict() for item in output['process']]
return output
class ProcedureEquipmentAssociation(BaseClass):
"""
@@ -2342,9 +2373,9 @@ class ProcedureEquipmentAssociation(BaseClass):
# NOTE: Figure out how to merge the misc_info if doing .update instead.
relevant = {k: v for k, v in output.items() if k not in ['equipment']}
output = output['equipment'].details_dict()
misc = output['_misc_info']
misc = output['misc_info']
output.update(relevant)
output['_misc_info'] = misc
output['misc_info'] = misc
output['process'] = self.process.details_dict()
return output
@@ -2530,12 +2561,25 @@ class Process(BaseClass):
name=self.name,
)
if full_data:
subs = [dict(plate=sub.run.rsl_plate_num, equipment=sub.equipment.name,
subs = [dict(plate=sub.run.rsl_plate_number, equipment=sub.equipment.name,
submitted_date=sub.run.clientsubmission.submitted_date) for sub in self.procedure]
output['procedure'] = sorted(subs, key=itemgetter("submitted_date"), reverse=True)
output['excluded'] = ['missing', 'procedure', 'excluded', 'editable']
return output
def to_pydantic(self):
from backend.validators.pydant import PydProcess
output = {}
for k, v in self.details_dict().items():
if isinstance(v, list):
output[k] = [item.name for item in v]
elif issubclass(v.__class__, BaseClass):
output[k] = v.name
else:
output[k] = v
return PydProcess(**output)
# @classproperty
# def details_template(cls) -> Template:
# """
@@ -2591,7 +2635,10 @@ class TipRole(BaseClass):
@classmethod
@setup_lookup
def query(cls, name: str | None = None, limit: int = 0, **kwargs) -> TipRole | List[TipRole]:
def query(cls,
name: str | None = None,
limit: int = 0,
**kwargs) -> TipRole | List[TipRole]:
query = cls.__database_session__.query(cls)
match name:
case str():
@@ -2707,7 +2754,7 @@ class Tips(BaseClass, LogMixin):
)
if full_data:
subs = [
dict(plate=item.procedure.procedure.rsl_plate_num, role=item.role_name,
dict(plate=item.procedure.procedure.rsl_plate_number, role=item.role_name,
sub_date=item.procedure.procedure.clientsubmission.submitted_date)
for item in self.tipsprocedureassociation]
output['procedure'] = sorted(subs, key=itemgetter("sub_date"), reverse=True)
@@ -2819,15 +2866,16 @@ class ProcedureTipsAssociation(BaseClass):
# NOTE: Figure out how to merge the misc_info if doing .update instead.
relevant = {k: v for k, v in output.items() if k not in ['tips']}
output = output['tips'].details_dict()
misc = output['_misc_info']
misc = output['misc_info']
output.update(relevant)
output['_misc_info'] = misc
output['misc_info'] = misc
return output
class Results(BaseClass):
id = Column(INTEGER, primary_key=True)
result_type = Column(String(32))
result = Column(JSON)
procedure_id = Column(INTEGER, ForeignKey("_procedure.id", ondelete='SET NULL',
name="fk_RES_procedure_id"))

98
src/submissions/backend/db/models/submissions.py
View File

@@ -26,7 +26,7 @@ from sqlite3 import OperationalError as SQLOperationalError, IntegrityError as S
from openpyxl import Workbook
from openpyxl.drawing.image import Image as OpenpyxlImage
from tools import row_map, setup_lookup, jinja_template_loading, rreplace, row_keys, check_key_or_attr, Result, Report, \
report_result, create_holidays_for_year, check_dictionary_inclusion_equality
report_result, create_holidays_for_year, check_dictionary_inclusion_equality, is_power_user
from datetime import datetime, date
from typing import List, Any, Tuple, Literal, Generator, Type, TYPE_CHECKING
from pathlib import Path
@@ -100,7 +100,7 @@ class ClientSubmission(BaseClass, LogMixin):
Args:
submission_type (str | models.SubmissionType | None, optional): Submission type of interest. Defaults to None.
id (int | str | None, optional): Submission id in the database (limits results to 1). Defaults to None.
rsl_plate_num (str | None, optional): Submission name in the database (limits results to 1). Defaults to None.
rsl_plate_number (str | None, optional): Submission name in the database (limits results to 1). Defaults to None.
start_date (date | str | int | None, optional): Beginning date to search by. Defaults to None.
end_date (date | str | int | None, optional): Ending date to search by. Defaults to None.
reagent (models.Reagent | str | None, optional): A reagent used in the procedure. Defaults to None.
@@ -304,7 +304,7 @@ class ClientSubmission(BaseClass, LogMixin):
samples = [sample.to_pydantic() for sample in self.clientsubmissionsampleassociation]
checker = SampleChecker(parent=None, title="Create Run", samples=samples, clientsubmission=self)
if checker.exec():
run = Run(clientsubmission=self, rsl_plate_num=checker.rsl_plate_num)
run = Run(clientsubmission=self, rsl_plate_number=checker.rsl_plate_number)
active_samples = [sample for sample in samples if sample.enabled]
logger.debug(active_samples)
for sample in active_samples:
@@ -323,8 +323,12 @@ class ClientSubmission(BaseClass, LogMixin):
def add_comment(self, obj):
logger.debug("Add Comment")
def show_details(self, obj):
logger.debug("Show Details")
# def show_details(self, obj):
# logger.debug("Show Details")
# from frontend.widgets.submission_details import SubmissionDetails
# dlg = SubmissionDetails(parent=obj, sub=self)
# if dlg.exec():
# pass
def details_dict(self, **kwargs):
output = super().details_dict(**kwargs)
@@ -334,6 +338,11 @@ class ClientSubmission(BaseClass, LogMixin):
output['run'] = [run.details_dict() for run in output['run']]
output['sample'] = [sample.details_dict() for sample in output['clientsubmissionsampleassociation']]
output['name'] = self.name
output['client_lab'] = output['clientlab']
output['submission_type'] = output['submissiontype']
output['excluded'] = ['run', "sample", "clientsubmissionsampleassociation", "excluded",
"expanded", 'clientlab', 'submissiontype', 'id']
output['expanded'] = ["clientlab", "contact", "submissiontype"]
return output
@@ -343,7 +352,7 @@ class Run(BaseClass, LogMixin):
"""
id = Column(INTEGER, primary_key=True) #: primary key
rsl_plate_num = Column(String(32), unique=True, nullable=False) #: RSL name (e.g. RSL-22-0012)
rsl_plate_number = Column(String(32), unique=True, nullable=False) #: RSL name (e.g. RSL-22-0012)
clientsubmission_id = Column(INTEGER, ForeignKey("_clientsubmission.id", ondelete="SET NULL",
name="fk_BS_clientsub_id")) #: client lab id from _organizations)
clientsubmission = relationship("ClientSubmission", back_populates="run")
@@ -387,7 +396,7 @@ class Run(BaseClass, LogMixin):
@hybrid_property
def name(self):
return self.rsl_plate_num
return self.rsl_plate_number
@classmethod
def get_default_info(cls, *args, submissiontype: SubmissionType | None = None) -> dict:
@@ -593,8 +602,23 @@ class Run(BaseClass, LogMixin):
def details_dict(self, **kwargs):
output = super().details_dict()
output['sample'] = [sample.details_dict() for sample in output['runsampleassociation']]
submission_samples = [sample for sample in self.clientsubmission.sample]
# logger.debug(f"Submission samples:{pformat(submission_samples)}")
active_samples = [sample.details_dict() for sample in output['runsampleassociation']
if sample.sample.sample_id in [s.sample_id for s in submission_samples]]
# logger.debug(f"Active samples:{pformat(active_samples)}")
for sample in active_samples:
sample['active'] = True
inactive_samples = [sample.details_dict() for sample in submission_samples if sample.name not in [s['sample_id'] for s in active_samples]]
# logger.debug(f"Inactive samples:{pformat(inactive_samples)}")
for sample in inactive_samples:
sample['active'] = False
# output['sample'] = [sample.details_dict() for sample in output['runsampleassociation']]
output['sample'] = active_samples + inactive_samples
output['procedure'] = [procedure.details_dict() for procedure in output['procedure']]
output['permission'] = is_power_user()
output['excluded'] = ['procedure', "runsampleassociation", 'excluded', 'expanded', 'sample', 'id', 'custom', 'permission']
return output
@classmethod
@@ -890,10 +914,10 @@ class Run(BaseClass, LogMixin):
field_value = dict(value=self.__getattribute__(key).name, missing=missing)
case "plate_number":
key = 'name'
field_value = dict(value=self.rsl_plate_num, missing=missing)
field_value = dict(value=self.rsl_plate_number, missing=missing)
case "submitter_plate_number":
key = "submitter_plate_id"
field_value = dict(value=self.submitter_plate_num, missing=missing)
field_value = dict(value=self.submitter_plate_number, missing=missing)
case "id":
continue
case _:
@@ -1168,8 +1192,8 @@ class Run(BaseClass, LogMixin):
e: SQLIntegrityError or SQLOperationalError if problem with commit.
"""
from frontend.widgets.pop_ups import QuestionAsker
fname = self.__backup_path__.joinpath(f"{self.rsl_plate_num}-backup({date.today().strftime('%Y%m%d')})")
msg = QuestionAsker(title="Delete?", message=f"Are you sure you want to delete {self.rsl_plate_num}?\n")
fname = self.__backup_path__.joinpath(f"{self.rsl_plate_number}-backup({date.today().strftime('%Y%m%d')})")
msg = QuestionAsker(title="Delete?", message=f"Are you sure you want to delete {self.rsl_plate_number}?\n")
if msg.exec():
try:
# NOTE: backs up file as xlsx, same as export.
@@ -1187,17 +1211,17 @@ class Run(BaseClass, LogMixin):
except AttributeError:
logger.error("App will not refresh data at this time.")
def show_details(self, obj):
"""
Creates Widget for showing procedure details.
Args:
obj (Widget): Parent widget
"""
from frontend.widgets.submission_details import SubmissionDetails
dlg = SubmissionDetails(parent=obj, sub=self)
if dlg.exec():
pass
# def show_details(self, obj):
# """
# Creates Widget for showing procedure details.
#
# Args:
# obj (Widget): Parent widget
# """
# from frontend.widgets.submission_details import SubmissionDetails
# dlg = SubmissionDetails(parent=obj, sub=self)
# if dlg.exec():
# pass
def edit(self, obj):
"""
@@ -1641,12 +1665,12 @@ class ClientSubmissionSampleAssociation(BaseClass):
output = super().details_dict()
# NOTE: Figure out how to merge the misc_info if doing .update instead.
relevant = {k: v for k, v in output.items() if k not in ['sample']}
logger.debug(f"Relevant info from assoc output: {pformat(relevant)}")
# logger.debug(f"Relevant info from assoc output: {pformat(relevant)}")
output = output['sample'].details_dict()
misc = output['_misc_info']
logger.debug(f"Output from sample: {pformat(output)}")
misc = output['misc_info']
# logger.debug(f"Output from sample: {pformat(output)}")
output.update(relevant)
output['_misc_info'] = misc
output['misc_info'] = misc
# output['sample'] = temp
# output.update(output['sample'].details_dict())
return output
@@ -1815,7 +1839,7 @@ class ClientSubmissionSampleAssociation(BaseClass):
case ClientSubmission():
pass
case str():
clientsubmission = ClientSubmission.query(rsl_plate_num=clientsubmission)
clientsubmission = ClientSubmission.query(rsl_plate_number=clientsubmission)
case _:
raise ValueError()
match sample:
@@ -1879,7 +1903,7 @@ class RunSampleAssociation(BaseClass):
def __repr__(self) -> str:
try:
return f"<{self.__class__.__name__}({self.run.rsl_plate_num} & {self.sample.sample_id})"
return f"<{self.__class__.__name__}({self.run.rsl_plate_number} & {self.sample.sample_id})"
except AttributeError as e:
logger.error(f"Unable to construct __repr__ due to: {e}")
return super().__repr__()
@@ -1901,7 +1925,7 @@ class RunSampleAssociation(BaseClass):
except KeyError as e:
logger.error(f"Unable to find row {self.row} in row_map.")
sample['Well'] = None
sample['plate_name'] = self.run.rsl_plate_num
sample['plate_name'] = self.run.rsl_plate_number
sample['positive'] = False
return sample
@@ -1975,7 +1999,7 @@ class RunSampleAssociation(BaseClass):
case Run():
query = query.filter(cls.run == run)
case str():
query = query.join(Run).filter(Run.rsl_plate_num == run)
query = query.join(Run).filter(Run.rsl_plate_number == run)
case _:
pass
match sample:
@@ -2060,12 +2084,12 @@ class RunSampleAssociation(BaseClass):
output = super().details_dict()
# NOTE: Figure out how to merge the misc_info if doing .update instead.
relevant = {k: v for k, v in output.items() if k not in ['sample']}
logger.debug(f"Relevant info from assoc output: {pformat(relevant)}")
# logger.debug(f"Relevant info from assoc output: {pformat(relevant)}")
output = output['sample'].details_dict()
misc = output['_misc_info']
logger.debug(f"Output from sample: {pformat(output)}")
misc = output['misc_info']
# logger.debug(f"Output from sample: {pformat(output)}")
output.update(relevant)
output['_misc_info'] = misc
output['misc_info'] = misc
return output
class ProcedureSampleAssociation(BaseClass):
@@ -2132,9 +2156,9 @@ class ProcedureSampleAssociation(BaseClass):
# NOTE: Figure out how to merge the misc_info if doing .update instead.
relevant = {k: v for k, v in output.items() if k not in ['sample']}
output = output['sample'].details_dict()
misc = output['_misc_info']
misc = output['misc_info']
output.update(relevant)
output['_misc_info'] = misc
output['misc_info'] = misc
output['results'] = [result.details_dict() for result in output['results']]
return output