landon/Submissions-App
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added Postgres support.

Browse Source
This commit is contained in:
lwark
2024-07-25 08:41:44 -05:00
parent 54e1e55804
commit 4bc5e08ac6
32 changed files with 579 additions and 1030 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/submissions/backend/db/__init__.py
View File

@@ -17,7 +17,7 @@ def set_sqlite_pragma(dbapi_connection, connection_record):
connection_record (_type_): _description_
"""
cursor = dbapi_connection.cursor()
cursor.execute("PRAGMA foreign_keys=ON")
# cursor.execute("PRAGMA foreign_keys=ON")
cursor.close()

23
src/submissions/backend/db/models/__init__.py
View File

@@ -3,12 +3,13 @@ Contains all models for sqlalchemy
"""
from __future__ import annotations
import sys, logging
from sqlalchemy import Column, INTEGER, String, JSON
from sqlalchemy import Column, INTEGER, String, JSON, inspect
from sqlalchemy.orm import DeclarativeMeta, declarative_base, Query, Session
from sqlalchemy.ext.declarative import declared_attr
from sqlalchemy.exc import ArgumentError
from typing import Any, List
from pathlib import Path
from tools import report_result
# Load testing environment
if 'pytest' in sys.modules:
@@ -90,7 +91,7 @@ class BaseClass(Base):
Returns:
dict | list | str: Output of key:value dict or single (list, str) desired variable
"""
"""
dicto = dict(singles=['id'])
output = {}
for k, v in dicto.items():
@@ -110,7 +111,7 @@ class BaseClass(Base):
Returns:
Any | List[Any]: Result of query execution.
"""
"""
return cls.execute_query(**kwargs)
@classmethod
@@ -152,38 +153,43 @@ class BaseClass(Base):
case _:
return query.limit(limit).all()
@report_result
def save(self):
"""
Add the object to the database and commit
"""
# logger.debug(f"Saving object: {pformat(self.__dict__)}")
report = Report()
try:
self.__database_session__.add(self)
self.__database_session__.commit()
# self.__database_session__.merge(self)
except Exception as e:
logger.critical(f"Problem saving object: {e}")
self.__database_session__.rollback()
report.add_result(Result(msg=f"Problem saving object {e}", status="Critical"))
return report
class ConfigItem(BaseClass):
"""
Key:JSON objects to store config settings in database.
"""
"""
id = Column(INTEGER, primary_key=True)
key = Column(String(32)) #: Name of the configuration item.
value = Column(JSON) #: Value associated with the config item.
key = Column(String(32)) #: Name of the configuration item.
value = Column(JSON) #: Value associated with the config item.
def __repr__(self):
return f"ConfigItem({self.key} : {self.value})"
@classmethod
def get_config_items(cls, *args) -> ConfigItem|List[ConfigItem]:
def get_config_items(cls, *args) -> ConfigItem | List[ConfigItem]:
"""
Get desired config items from database
Returns:
ConfigItem|List[ConfigItem]: Config item(s)
"""
"""
config_items = cls.__database_session__.query(cls).all()
config_items = [item for item in config_items if item.key in args]
if len(args) == 1:
@@ -196,4 +202,5 @@ from .controls import *
from .organizations import *
from .kits import *
from .submissions import *
BasicSubmission.reagents.creator = lambda reg: SubmissionReagentAssociation(reagent=reg)

2
src/submissions/backend/db/models/kits.py
View File

@@ -1532,7 +1532,7 @@ class Process(BaseClass):
query = cls.__database_session__.query(cls)
match name:
case str():
logger.debug(f"Lookup Process with name str {name}")
# logger.debug(f"Lookup Process with name str {name}")
query = query.filter(cls.name == name)
limit = 1
case _:

50
src/submissions/backend/db/models/submissions.py
View File

@@ -23,7 +23,7 @@ import pandas as pd
from openpyxl import Workbook
from openpyxl.worksheet.worksheet import Worksheet
from openpyxl.drawing.image import Image as OpenpyxlImage
from tools import row_map, setup_lookup, jinja_template_loading, rreplace, row_keys, check_key_or_attr
from tools import row_map, setup_lookup, jinja_template_loading, rreplace, row_keys, check_key_or_attr, Result, Report
from datetime import datetime, date
from typing import List, Any, Tuple, Literal
from dateutil.parser import parse
@@ -691,7 +691,8 @@ class BasicSubmission(BaseClass):
Args:
input_dict (dict): Input sample dictionary
xl (pd.ExcelFile): original xl workbook, used for child classes mostly
xl (Workbook): original xl workbook, used for child classes mostly
custom_fields: Dictionary of locations, ranges, etc to be used by this function
Returns:
dict: Updated sample dictionary
@@ -739,6 +740,7 @@ class BasicSubmission(BaseClass):
input_excel (Workbook): initial workbook.
info (dict | None, optional): dictionary of additional info. Defaults to None.
backup (bool, optional): Whether this is part of a backup operation. Defaults to False.
custom_fields: Dictionary of locations, ranges, etc to be used by this function
Returns:
Workbook: Updated workbook
@@ -1046,14 +1048,16 @@ class BasicSubmission(BaseClass):
"""
code = 0
msg = ""
report = Report()
disallowed = ["id"]
if kwargs == {}:
raise ValueError("Need to narrow down query or the first available instance will be returned.")
for key in kwargs.keys():
if key in disallowed:
raise ValueError(
f"{key} is not allowed as a query argument as it could lead to creation of duplicate objects. Use .query() instead.")
instance = cls.query(submission_type=submission_type, limit=1, **kwargs)
sanitized_kwargs = {k: v for k, v in kwargs.items() if k not in disallowed}
# for key in kwargs.keys():
# if key in disallowed:
# raise ValueError(
# f"{key} is not allowed as a query argument as it could lead to creation of duplicate objects. Use .query() instead.")
instance = cls.query(submission_type=submission_type, limit=1, **sanitized_kwargs)
# logger.debug(f"Retrieved instance: {instance}")
if instance is None:
used_class = cls.find_polymorphic_subclass(attrs=kwargs, polymorphic_identity=submission_type)
@@ -1070,7 +1074,8 @@ class BasicSubmission(BaseClass):
else:
code = 1
msg = "This submission already exists.\nWould you like to overwrite?"
return instance, code, msg
report.add_result(Result(msg=msg, code=code))
return instance, report
# Custom context events for the ui
@@ -1135,7 +1140,7 @@ class BasicSubmission(BaseClass):
# logger.debug(widg)
widg.setParent(None)
pyd = self.to_pydantic(backup=True)
form = pyd.to_form(parent=obj)
form = pyd.to_form(parent=obj, disable=['rsl_plate_num'])
obj.app.table_widget.formwidget.layout().addWidget(form)
def add_comment(self, obj):
@@ -1352,13 +1357,29 @@ class Wastewater(BasicSubmission):
Args:
input_dict (dict): Input sample dictionary
xl (Workbook): xl (Workbook): original xl workbook, used for child classes mostly.
custom_fields: Dictionary of locations, ranges, etc to be used by this function
Returns:
dict: Updated sample dictionary
"""
input_dict = super().custom_info_parser(input_dict)
logger.debug(f"Input dict: {pformat(input_dict)}")
if xl is not None:
input_dict['csv'] = xl["Copy to import file"]
try:
input_dict['csv'] = xl["Copy to import file"]
except KeyError as e:
logger.error(e)
try:
match input_dict['rsl_plate_num']:
case dict():
input_dict['csv'] = xl[input_dict['rsl_plate_num']['value']]
case str():
input_dict['csv'] = xl[input_dict['rsl_plate_num']]
case _:
pass
except Exception as e:
logger.error(f"Error handling couldn't get csv due to: {e}")
return input_dict
@classmethod
@@ -1604,11 +1625,12 @@ class WastewaterArtic(BasicSubmission):
Args:
input_dict (dict): Input sample dictionary
xl (pd.ExcelFile): original xl workbook, used for child classes mostly
custom_fields: Dictionary of locations, ranges, etc to be used by this function
Returns:
dict: Updated sample dictionary
"""
# TODO: Clean up and move range start/stops to db somehow.
from backend.validators import RSLNamer
input_dict = super().custom_info_parser(input_dict)
egel_section = custom_fields['egel_results']
ws = xl[egel_section['sheet']]
@@ -1621,12 +1643,11 @@ class WastewaterArtic(BasicSubmission):
source_plates_section = custom_fields['source_plates']
ws = xl[source_plates_section['sheet']]
data = [dict(plate=ws.cell(row=ii, column=source_plates_section['plate_column']).value, starting_sample=ws.cell(row=ii, column=source_plates_section['starting_sample_column']).value) for ii in
range(source_plates_section['start_row'], source_plates_section['end_row'])]
range(source_plates_section['start_row'], source_plates_section['end_row']+1)]
for datum in data:
if datum['plate'] in ["None", None, ""]:
continue
else:
from backend.validators import RSLNamer
datum['plate'] = RSLNamer(filename=datum['plate'], sub_type="Wastewater").parsed_name
input_dict['source_plates'] = data
return input_dict
@@ -1820,6 +1841,7 @@ class WastewaterArtic(BasicSubmission):
input_excel (Workbook): initial workbook.
info (dict | None, optional): dictionary of additional info. Defaults to None.
backup (bool, optional): Whether this is part of a backup operation. Defaults to False.
custom_fields: Dictionary of locations, ranges, etc to be used by this function
Returns:
Workbook: Updated workbook
@@ -2798,7 +2820,7 @@ class WastewaterArticAssociation(SubmissionSampleAssociation):
DOC: https://docs.sqlalchemy.org/en/14/orm/extensions/associationproxy.html
"""
id = Column(INTEGER, ForeignKey("_submissionsampleassociation.id"), primary_key=True)
source_plate = Column(String(16))
source_plate = Column(String(32))
source_plate_number = Column(INTEGER)
source_well = Column(String(8))
ct = Column(String(8)) #: AKA ct for N1

92
src/submissions/backend/validators/pydant.py
View File

@@ -108,7 +108,7 @@ class PydReagent(BaseModel):
Returns:
dict: Information dictionary
"""
"""
try:
extras = list(self.model_extra.keys())
except AttributeError:
@@ -161,7 +161,7 @@ class PydReagent(BaseModel):
# reagent.reagent_submission_associations.append(assoc)
else:
assoc = None
report.add_result(Result(owner = __name__, code=0, msg="New reagent created.", status="Information"))
report.add_result(Result(owner=__name__, code=0, msg="New reagent created.", status="Information"))
else:
if submission is not None and reagent not in submission.reagents:
assoc = SubmissionReagentAssociation(reagent=reagent, submission=submission)
@@ -217,7 +217,7 @@ class PydSample(BaseModel, extra='allow'):
Returns:
dict: Information dictionary
"""
"""
fields = list(self.model_fields.keys()) + list(self.model_extra.keys())
return {k: getattr(self, k) for k in fields}
@@ -254,7 +254,8 @@ class PydSample(BaseModel, extra='allow'):
submission=submission,
sample=instance,
row=row, column=column, id=aid,
submission_rank=submission_rank, **self.model_extra)
submission_rank=submission_rank,
**self.model_extra)
# logger.debug(f"Using submission_sample_association: {association}")
try:
# instance.sample_submission_associations.append(association)
@@ -270,7 +271,7 @@ class PydSample(BaseModel, extra='allow'):
Returns:
dict: Information dictionary
"""
"""
try:
extras = list(self.model_extra.keys())
except AttributeError:
@@ -281,10 +282,10 @@ class PydSample(BaseModel, extra='allow'):
class PydTips(BaseModel):
name: str
lot: str|None = Field(default=None)
lot: str | None = Field(default=None)
role: str
def to_sql(self, submission:BasicSubmission) -> SubmissionTipsAssociation:
def to_sql(self, submission: BasicSubmission) -> SubmissionTipsAssociation:
"""
Con
@@ -293,7 +294,7 @@ class PydTips(BaseModel):
Returns:
SubmissionTipsAssociation: Association between queried tips and submission
"""
"""
tips = Tips.query(name=self.name, lot=self.lot, limit=1)
assoc = SubmissionTipsAssociation(submission=submission, tips=tips, role_name=self.role)
return assoc
@@ -305,7 +306,7 @@ class PydEquipment(BaseModel, extra='ignore'):
nickname: str | None
processes: List[str] | None
role: str | None
tips: List[PydTips]|None = Field(default=None)
tips: List[PydTips] | None = Field(default=None)
@field_validator('processes', mode='before')
@classmethod
@@ -338,23 +339,19 @@ class PydEquipment(BaseModel, extra='ignore'):
if equipment is None:
return
if submission is not None:
assoc = SubmissionEquipmentAssociation(submission=submission, equipment=equipment)
process = Process.query(name=self.processes[0])
if process is None:
logger.error(f"Found unknown process: {process}.")
# from frontend.widgets.pop_ups import QuestionAsker
# dlg = QuestionAsker(title="Add Process?",
# message=f"Unable to find {self.processes[0]} in the database.\nWould you like to add it?")
# if dlg.exec():
# kit = submission.extraction_kit
# submission_type = submission.submission_type
# process = Process(name=self.processes[0])
# process.kit_types.append(kit)
# process.submission_types.append(submission_type)
# process.equipment.append(equipment)
# process.save()
assoc.process = process
assoc.role = self.role
# NOTE: Need to make sure the same association is not added to the submission
assoc = SubmissionEquipmentAssociation.query(equipment_id=equipment.id, submission_id=submission.id,
role=self.role, limit=1)
if assoc is None:
assoc = SubmissionEquipmentAssociation(submission=submission, equipment=equipment)
process = Process.query(name=self.processes[0])
if process is None:
logger.error(f"Found unknown process: {process}.")
assoc.process = process
assoc.role = self.role
else:
assoc = None
else:
assoc = None
return equipment, assoc
@@ -365,7 +362,7 @@ class PydEquipment(BaseModel, extra='ignore'):
Returns:
dict: Information dictionary
"""
"""
try:
extras = list(self.model_extra.keys())
except AttributeError:
@@ -441,7 +438,7 @@ class PydSubmission(BaseModel, extra='allow'):
return value.date()
case int():
return dict(value=datetime.fromordinal(datetime(1900, 1, 1).toordinal() + value['value'] - 2).date(),
missing=True)
missing=True)
case str():
string = re.sub(r"(_|-)\d$", "", value['value'])
try:
@@ -508,7 +505,7 @@ class PydSubmission(BaseModel, extra='allow'):
output = "RSL-BS-Test001"
else:
output = RSLNamer(filename=values.data['filepath'].__str__(), sub_type=sub_type,
data=values.data).parsed_name
data=values.data).parsed_name
return dict(value=output, missing=True)
@field_validator("technician", mode="before")
@@ -637,14 +634,14 @@ class PydSubmission(BaseModel, extra='allow'):
self.submission_object = BasicSubmission.find_polymorphic_subclass(
polymorphic_identity=self.submission_type['value'])
def set_attribute(self, key:str, value):
def set_attribute(self, key: str, value):
"""
Better handling of attribute setting.
Args:
key (str): Name of field to set
value (_type_): Value to set field to.
"""
"""
self.__setattr__(name=key, value=value)
def handle_duplicate_samples(self):
@@ -710,7 +707,7 @@ class PydSubmission(BaseModel, extra='allow'):
missing_reagents = [reagent for reagent in self.reagents if reagent.missing]
return missing_info, missing_reagents
def to_sql(self) -> Tuple[BasicSubmission, Result]:
def to_sql(self) -> Tuple[BasicSubmission, Report]:
"""
Converts this instance into a backend.db.models.submissions.BasicSubmission instance
@@ -718,13 +715,13 @@ class PydSubmission(BaseModel, extra='allow'):
Tuple[BasicSubmission, Result]: BasicSubmission instance, result object
"""
# self.__dict__.update(self.model_extra)
report = Report()
dicto = self.improved_dict()
instance, code, msg = BasicSubmission.query_or_create(submission_type=self.submission_type['value'],
rsl_plate_num=self.rsl_plate_num['value'])
result = Result(msg=msg, code=code)
instance, result = BasicSubmission.query_or_create(submission_type=self.submission_type['value'],
rsl_plate_num=self.rsl_plate_num['value'])
report.add_result(result)
self.handle_duplicate_samples()
# logger.debug(f"Here's our list of duplicate removed samples: {self.samples}")
# for key, value in self.__dict__.items():
for key, value in dicto.items():
if isinstance(value, dict):
value = value['value']
@@ -733,13 +730,13 @@ class PydSubmission(BaseModel, extra='allow'):
# logger.debug(f"Setting {key} to {value}")
match key:
case "reagents":
if code == 1:
if report.results[0].code == 1:
instance.submission_reagent_associations = []
# logger.debug(f"Looking through {self.reagents}")
for reagent in self.reagents:
reagent, assoc, _ = reagent.toSQL(submission=instance)
# logger.debug(f"Association: {assoc}")
if assoc is not None:# and assoc not in instance.submission_reagent_associations:
if assoc is not None: # and assoc not in instance.submission_reagent_associations:
instance.submission_reagent_associations.append(assoc)
# instance.reagents.append(reagent)
case "samples":
@@ -755,10 +752,7 @@ class PydSubmission(BaseModel, extra='allow'):
if equip is None:
continue
equip, association = equip.toSQL(submission=instance)
if association is not None and association not in instance.submission_equipment_associations:
# association.save()
# logger.debug(
# f"Equipment association SQL object to be added to submission: {association.__dict__}")
if association is not None:
instance.submission_equipment_associations.append(association)
case "tips":
for tips in self.tips:
@@ -817,9 +811,9 @@ class PydSubmission(BaseModel, extra='allow'):
# except AttributeError as e:
# logger.debug(f"Something went wrong constructing instance {self.rsl_plate_num}: {e}")
# logger.debug(f"Constructed submissions message: {msg}")
return instance, result
return instance, report
def to_form(self, parent: QWidget):
def to_form(self, parent: QWidget, disable:list|None=None):
"""
Converts this instance into a frontend.widgets.submission_widget.SubmissionFormWidget
@@ -830,7 +824,8 @@ class PydSubmission(BaseModel, extra='allow'):
SubmissionFormWidget: Submission form widget
"""
from frontend.widgets.submission_widget import SubmissionFormWidget
return SubmissionFormWidget(parent=parent, submission=self)
logger.debug(f"Disbable: {disable}")
return SubmissionFormWidget(parent=parent, submission=self, disable=disable)
def to_writer(self) -> "SheetWriter":
"""
@@ -838,7 +833,7 @@ class PydSubmission(BaseModel, extra='allow'):
Returns:
SheetWriter: Sheetwriter object that will perform writing.
"""
"""
from backend.excel.writer import SheetWriter
return SheetWriter(self)
@@ -896,8 +891,8 @@ class PydSubmission(BaseModel, extra='allow'):
status="Warning")
report.add_result(result)
return output_reagents, report
def export_csv(self, filename:Path|str):
def export_csv(self, filename: Path | str):
try:
worksheet = self.csv
except AttributeError:
@@ -1024,4 +1019,3 @@ class PydEquipmentRole(BaseModel):
"""
from frontend.widgets.equipment_usage import RoleComboBox
return RoleComboBox(parent=parent, role=self, used=used)