Updated to proper json-ing.

TODO.md

@@ -1,7 +1,8 @@
-- [ ] Appending of qPCR results to WW not saving. Find out why.
+- [x] Update controls to NestedMutableJson
+- [x] Appending of qPCR results to WW not saving. Find out why.
     - Possibly due to immutable JSON? But... it's worked before... Right?
     - Based on research, if a top-level JSON field is not changed, SQLalchemy will not detect changes.
-    - May have to use a special class: [link](https://docs.sqlalchemy.org/en/14/orm/extensions/mutable.html#establishing-mutability-on-scalar-column-values)
+    - Using sqlalchemy-json module seems to have helped.
 - [ ] Add Bead basher and Assit to DB.
 - [x] Artic not creating right plate name.
 - [ ] Merge BasicSubmission.find_subclasses and BasicSubmission.find_polymorphic_subclass

src/submissions/backend/db/models/controls.py

@@ -4,7 +4,8 @@ All control related models.
 from __future__ import annotations
 from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey
 from sqlalchemy.orm import relationship, Query
-import logging, json
+from sqlalchemy_json import NestedMutableJson
+import logging
 from operator import itemgetter
 from . import BaseClass
 from tools import setup_lookup
@@ -60,9 +61,10 @@ class ControlType(BaseClass):
             List[str]: list of subtypes available
         """        
         # Get first instance since all should have same subtypes
-        outs = self.instances[0]
+        # outs = self.instances[0]
         # Get mode of instance
-        jsoner = json.loads(getattr(outs, mode))
+        # jsoner = json.loads(getattr(outs, mode))
+        jsoner = getattr(self.instances[0], mode)
         logger.debug(f"JSON out: {jsoner.keys()}")
         try:
             # Pick genera (all should have same subtypes)
@@ -82,9 +84,9 @@ class Control(BaseClass):
     controltype = relationship("ControlType", back_populates="instances", foreign_keys=[parent_id]) #: reference to parent control type
     name = Column(String(255), unique=True) #: Sample ID
     submitted_date = Column(TIMESTAMP) #: Date submitted to Robotics
-    contains = Column(JSON) #: unstructured hashes in contains.tsv for each organism
+    contains = Column(NestedMutableJson) #: unstructured hashes in contains.tsv for each organism
-    matches = Column(JSON) #: unstructured hashes in matches.tsv for each organism
+    matches = Column(NestedMutableJson) #: unstructured hashes in matches.tsv for each organism
-    kraken = Column(JSON) #: unstructured output from kraken_report
+    kraken = Column(NestedMutableJson) #: unstructured output from kraken_report
     submission_id = Column(INTEGER, ForeignKey("_basicsubmission.id")) #: parent submission id
     submission = relationship("BacterialCulture", back_populates="controls", foreign_keys=[submission_id]) #: parent submission
     refseq_version = Column(String(16)) #: version of refseq used in fastq parsing
@@ -109,7 +111,8 @@ class Control(BaseClass):
         """        
         # logger.debug("loading json string into dict")
         try:
-            kraken = json.loads(self.kraken)
+            # kraken = json.loads(self.kraken)
+            kraken = self.kraken
         except TypeError:
             kraken = {}
         # logger.debug("calculating kraken count total to use in percentage")
@@ -147,7 +150,8 @@ class Control(BaseClass):
         output = []
         # logger.debug("load json string for mode (i.e. contains, matches, kraken2)")
         try:
-            data = json.loads(getattr(self, mode))
+            # data = json.loads(getattr(self, mode))
+            data = self.__getattribute__(mode)
         except TypeError:
             data = {}
         logger.debug(f"Length of data: {len(data)}")

src/submissions/backend/db/models/submissions.py

@@ -11,22 +11,24 @@ from reportlab.graphics.shapes import Drawing
 from reportlab.lib.units import mm
 from operator import attrgetter, itemgetter
 from pprint import pformat
-from . import Reagent, SubmissionType, KitType, Organization
+from . import BaseClass, Reagent, SubmissionType, KitType, Organization
+# MutableDict and JSONEncodedDict are custom classes designed to get around JSON columns not being updated.
+# See: https://docs.sqlalchemy.org/en/14/orm/extensions/mutable.html#establishing-mutability-on-scalar-column-values
 from sqlalchemy import Column, String, TIMESTAMP, INTEGER, ForeignKey, JSON, FLOAT, case
 from sqlalchemy.orm import relationship, validates, Query
 from sqlalchemy.ext.associationproxy import association_proxy
+from sqlalchemy_json import NestedMutableJson
+from sqlalchemy.exc import OperationalError as AlcOperationalError, IntegrityError as AlcIntegrityError, StatementError
+from sqlite3 import OperationalError as SQLOperationalError, IntegrityError as SQLIntegrityError
 import pandas as pd
 from openpyxl import Workbook
 from openpyxl.worksheet.worksheet import Worksheet
 from openpyxl.drawing.image import Image as OpenpyxlImage
-from . import BaseClass
 from tools import check_not_nan, row_map, setup_lookup, jinja_template_loading, rreplace
 from datetime import datetime, date
 from typing import List, Any, Tuple
 from dateutil.parser import parse
 from dateutil.parser._parser import ParserError
-from sqlalchemy.exc import OperationalError as AlcOperationalError, IntegrityError as AlcIntegrityError, StatementError
-from sqlite3 import OperationalError as SQLOperationalError, IntegrityError as SQLIntegrityError
 from pathlib import Path
 from jinja2.exceptions import TemplateNotFound
 from jinja2 import Template
@@ -52,10 +54,10 @@ class BasicSubmission(BaseClass):
     # Move this into custom types?
     # reagents = relationship("Reagent", back_populates="submissions", secondary=reagents_submissions) #: relationship to reagents
     reagents_id = Column(String, ForeignKey("_reagent.id", ondelete="SET NULL", name="fk_BS_reagents_id")) #: id of used reagents
-    extraction_info = Column(JSON) #: unstructured output from the extraction table logger.
+    extraction_info = Column(NestedMutableJson) #: unstructured output from the extraction table logger.
     run_cost = Column(FLOAT(2)) #: total cost of running the plate. Set from constant and mutable kit costs at time of creation.
     uploaded_by = Column(String(32)) #: user name of person who submitted the submission to the database.
-    comment = Column(JSON) #: user notes
+    comment = Column(NestedMutableJson) #: user notes
     submission_category = Column(String(64)) #: ["Research", "Diagnostic", "Surveillance", "Validation"], else defaults to submission_type_name
 
     submission_sample_associations = relationship(
@@ -1141,7 +1143,8 @@ class Wastewater(BasicSubmission):
     id = Column(INTEGER, ForeignKey('_basicsubmission.id'), primary_key=True)
     ext_technician = Column(String(64)) #: Name of technician doing extraction
     pcr_technician = Column(String(64)) #: Name of technician doing pcr
-    pcr_info = Column(JSON) #: unstructured output from pcr table logger or user(Artic)
+    # pcr_info = Column(JSON) #: unstructured output from pcr table logger or user(Artic)
+    pcr_info = Column(NestedMutableJson)
 
     __mapper_args__ = __mapper_args__ = dict(polymorphic_identity="Wastewater", 
                            polymorphic_load="inline", 
@@ -1331,10 +1334,10 @@ class WastewaterArtic(BasicSubmission):
     id = Column(INTEGER, ForeignKey('_basicsubmission.id'), primary_key=True)
     artic_technician = Column(String(64)) #: Name of technician performing artic
     dna_core_submission_number = Column(String(64)) #: Number used by core as id
-    pcr_info = Column(JSON) #: unstructured output from pcr table logger or user(Artic)
+    pcr_info = Column(NestedMutableJson) #: unstructured output from pcr table logger or user(Artic)
     gel_image = Column(String(64)) #: file name of gel image in zip file
-    gel_info = Column(JSON) #: unstructured data from gel.
+    gel_info = Column(NestedMutableJson) #: unstructured data from gel.
-    source_plates = Column(JSON) #: wastewater plates that samples come from
+    source_plates = Column(NestedMutableJson) #: wastewater plates that samples come from
 
     __mapper_args__ = dict(polymorphic_identity="Wastewater Artic", 
                            polymorphic_load="inline", 
@@ -2339,7 +2342,7 @@ class WastewaterAssociation(SubmissionSampleAssociation):
     ct_n2 = Column(FLOAT(2)) #: AKA ct for N2
     n1_status = Column(String(32)) #: positive or negative for N1
     n2_status = Column(String(32)) #: positive or negative for N2
-    pcr_results = Column(JSON) #: imported PCR status from QuantStudio
+    pcr_results = Column(NestedMutableJson) #: imported PCR status from QuantStudio
 
     __mapper_args__ = dict(polymorphic_identity="Wastewater Association", 
                            polymorphic_load="inline",