From 4e099131028bdd00f74e275ebe61f72b638a50a0 Mon Sep 17 00:00:00 2001 From: Landon Wark Date: Mon, 15 Apr 2024 12:31:41 -0500 Subject: [PATCH] Updated to proper json-ing. --- TODO.md | 5 ++-- requirements.txt | Bin 4682 -> 4730 bytes src/submissions/backend/db/models/controls.py | 20 ++++++++------ .../backend/db/models/submissions.py | 25 ++++++++++-------- 4 files changed, 29 insertions(+), 21 deletions(-) diff --git a/TODO.md b/TODO.md index 95fb75d..1f9e608 100644 --- a/TODO.md +++ b/TODO.md @@ -1,7 +1,8 @@ -- [ ] Appending of qPCR results to WW not saving. Find out why. +- [x] Update controls to NestedMutableJson +- [x] Appending of qPCR results to WW not saving. Find out why. - Possibly due to immutable JSON? But... it's worked before... Right? - Based on research, if a top-level JSON field is not changed, SQLalchemy will not detect changes. - - May have to use a special class: [link](https://docs.sqlalchemy.org/en/14/orm/extensions/mutable.html#establishing-mutability-on-scalar-column-values) + - Using sqlalchemy-json module seems to have helped. - [ ] Add Bead basher and Assit to DB. - [x] Artic not creating right plate name. - [ ] Merge BasicSubmission.find_subclasses and BasicSubmission.find_polymorphic_subclass diff --git a/requirements.txt b/requirements.txt index d5a61cf36cc31b0e69ebc6fc9b3bd95a5cea8b34..b305c5b7c48e80b0802df304bab6aab28eb34b73 100644 GIT binary patch delta 54 zcmX@5@=ImI7Jkh_h8%`OAWmk;U`S=iWvFD(WyoSEX2@sAW3UB60|q??b09X@yq141 F69A8N49frj delta 12 UcmeyRa!O^x7XHmo_@^-e04b;iz5oCK diff --git a/src/submissions/backend/db/models/controls.py b/src/submissions/backend/db/models/controls.py index 46c6238..8646674 100644 --- a/src/submissions/backend/db/models/controls.py +++ b/src/submissions/backend/db/models/controls.py @@ -4,7 +4,8 @@ All control related models. from __future__ import annotations from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey from sqlalchemy.orm import relationship, Query -import logging, json +from sqlalchemy_json import NestedMutableJson +import logging from operator import itemgetter from . import BaseClass from tools import setup_lookup @@ -60,9 +61,10 @@ class ControlType(BaseClass): List[str]: list of subtypes available """ # Get first instance since all should have same subtypes - outs = self.instances[0] + # outs = self.instances[0] # Get mode of instance - jsoner = json.loads(getattr(outs, mode)) + # jsoner = json.loads(getattr(outs, mode)) + jsoner = getattr(self.instances[0], mode) logger.debug(f"JSON out: {jsoner.keys()}") try: # Pick genera (all should have same subtypes) @@ -82,9 +84,9 @@ class Control(BaseClass): controltype = relationship("ControlType", back_populates="instances", foreign_keys=[parent_id]) #: reference to parent control type name = Column(String(255), unique=True) #: Sample ID submitted_date = Column(TIMESTAMP) #: Date submitted to Robotics - contains = Column(JSON) #: unstructured hashes in contains.tsv for each organism - matches = Column(JSON) #: unstructured hashes in matches.tsv for each organism - kraken = Column(JSON) #: unstructured output from kraken_report + contains = Column(NestedMutableJson) #: unstructured hashes in contains.tsv for each organism + matches = Column(NestedMutableJson) #: unstructured hashes in matches.tsv for each organism + kraken = Column(NestedMutableJson) #: unstructured output from kraken_report submission_id = Column(INTEGER, ForeignKey("_basicsubmission.id")) #: parent submission id submission = relationship("BacterialCulture", back_populates="controls", foreign_keys=[submission_id]) #: parent submission refseq_version = Column(String(16)) #: version of refseq used in fastq parsing @@ -109,7 +111,8 @@ class Control(BaseClass): """ # logger.debug("loading json string into dict") try: - kraken = json.loads(self.kraken) + # kraken = json.loads(self.kraken) + kraken = self.kraken except TypeError: kraken = {} # logger.debug("calculating kraken count total to use in percentage") @@ -147,7 +150,8 @@ class Control(BaseClass): output = [] # logger.debug("load json string for mode (i.e. contains, matches, kraken2)") try: - data = json.loads(getattr(self, mode)) + # data = json.loads(getattr(self, mode)) + data = self.__getattribute__(mode) except TypeError: data = {} logger.debug(f"Length of data: {len(data)}") diff --git a/src/submissions/backend/db/models/submissions.py b/src/submissions/backend/db/models/submissions.py index fd6018b..d5dc74c 100644 --- a/src/submissions/backend/db/models/submissions.py +++ b/src/submissions/backend/db/models/submissions.py @@ -11,22 +11,24 @@ from reportlab.graphics.shapes import Drawing from reportlab.lib.units import mm from operator import attrgetter, itemgetter from pprint import pformat -from . import Reagent, SubmissionType, KitType, Organization +from . import BaseClass, Reagent, SubmissionType, KitType, Organization +# MutableDict and JSONEncodedDict are custom classes designed to get around JSON columns not being updated. +# See: https://docs.sqlalchemy.org/en/14/orm/extensions/mutable.html#establishing-mutability-on-scalar-column-values from sqlalchemy import Column, String, TIMESTAMP, INTEGER, ForeignKey, JSON, FLOAT, case from sqlalchemy.orm import relationship, validates, Query from sqlalchemy.ext.associationproxy import association_proxy +from sqlalchemy_json import NestedMutableJson +from sqlalchemy.exc import OperationalError as AlcOperationalError, IntegrityError as AlcIntegrityError, StatementError +from sqlite3 import OperationalError as SQLOperationalError, IntegrityError as SQLIntegrityError import pandas as pd from openpyxl import Workbook from openpyxl.worksheet.worksheet import Worksheet from openpyxl.drawing.image import Image as OpenpyxlImage -from . import BaseClass from tools import check_not_nan, row_map, setup_lookup, jinja_template_loading, rreplace from datetime import datetime, date from typing import List, Any, Tuple from dateutil.parser import parse from dateutil.parser._parser import ParserError -from sqlalchemy.exc import OperationalError as AlcOperationalError, IntegrityError as AlcIntegrityError, StatementError -from sqlite3 import OperationalError as SQLOperationalError, IntegrityError as SQLIntegrityError from pathlib import Path from jinja2.exceptions import TemplateNotFound from jinja2 import Template @@ -52,10 +54,10 @@ class BasicSubmission(BaseClass): # Move this into custom types? # reagents = relationship("Reagent", back_populates="submissions", secondary=reagents_submissions) #: relationship to reagents reagents_id = Column(String, ForeignKey("_reagent.id", ondelete="SET NULL", name="fk_BS_reagents_id")) #: id of used reagents - extraction_info = Column(JSON) #: unstructured output from the extraction table logger. + extraction_info = Column(NestedMutableJson) #: unstructured output from the extraction table logger. run_cost = Column(FLOAT(2)) #: total cost of running the plate. Set from constant and mutable kit costs at time of creation. uploaded_by = Column(String(32)) #: user name of person who submitted the submission to the database. - comment = Column(JSON) #: user notes + comment = Column(NestedMutableJson) #: user notes submission_category = Column(String(64)) #: ["Research", "Diagnostic", "Surveillance", "Validation"], else defaults to submission_type_name submission_sample_associations = relationship( @@ -1141,7 +1143,8 @@ class Wastewater(BasicSubmission): id = Column(INTEGER, ForeignKey('_basicsubmission.id'), primary_key=True) ext_technician = Column(String(64)) #: Name of technician doing extraction pcr_technician = Column(String(64)) #: Name of technician doing pcr - pcr_info = Column(JSON) #: unstructured output from pcr table logger or user(Artic) + # pcr_info = Column(JSON) #: unstructured output from pcr table logger or user(Artic) + pcr_info = Column(NestedMutableJson) __mapper_args__ = __mapper_args__ = dict(polymorphic_identity="Wastewater", polymorphic_load="inline", @@ -1331,10 +1334,10 @@ class WastewaterArtic(BasicSubmission): id = Column(INTEGER, ForeignKey('_basicsubmission.id'), primary_key=True) artic_technician = Column(String(64)) #: Name of technician performing artic dna_core_submission_number = Column(String(64)) #: Number used by core as id - pcr_info = Column(JSON) #: unstructured output from pcr table logger or user(Artic) + pcr_info = Column(NestedMutableJson) #: unstructured output from pcr table logger or user(Artic) gel_image = Column(String(64)) #: file name of gel image in zip file - gel_info = Column(JSON) #: unstructured data from gel. - source_plates = Column(JSON) #: wastewater plates that samples come from + gel_info = Column(NestedMutableJson) #: unstructured data from gel. + source_plates = Column(NestedMutableJson) #: wastewater plates that samples come from __mapper_args__ = dict(polymorphic_identity="Wastewater Artic", polymorphic_load="inline", @@ -2339,7 +2342,7 @@ class WastewaterAssociation(SubmissionSampleAssociation): ct_n2 = Column(FLOAT(2)) #: AKA ct for N2 n1_status = Column(String(32)) #: positive or negative for N1 n2_status = Column(String(32)) #: positive or negative for N2 - pcr_results = Column(JSON) #: imported PCR status from QuantStudio + pcr_results = Column(NestedMutableJson) #: imported PCR status from QuantStudio __mapper_args__ = dict(polymorphic_identity="Wastewater Association", polymorphic_load="inline",