Improvements to JSON updaters.

TODO.md

@@ -1,3 +1,4 @@
+- [ ] Critical: Convert Json lits to dicts so I can have them update properly without using crashy Sqlalchemy-json
 - [ ] Fix Parsed/Missing mix ups.
 - [x] Have sample parser check for controls and add to reagents?
 - [x] Update controls to NestedMutableJson

alembic.ini

@@ -57,7 +57,7 @@ version_path_separator = os  # Use os.pathsep. Default configuration used for ne
 
 ; sqlalchemy.url = sqlite:///L:\Robotics Laboratory Support\Submissions\submissions.db
 ; sqlalchemy.url = sqlite:///C:\Users\lwark\Documents\Archives\Submissions_app_backups\DB_backups\submissions-demo.db
-sqlalchemy.url = sqlite:///C:\Users\lwark\Documents\python\submissions\tests\test_assets\submissions-test.db
+sqlalchemy.url = sqlite:///C:\Users\lwark\Documents\python\submissions\mytests\test_assets\submissions-test.db
 
 
 [post_write_hooks]

alembic/versions/f18487b41f45_adding_source_plates_to_artic_.py

@@ -0,0 +1,51 @@
+"""adding source plates to Artic submission...again
+
+Revision ID: f18487b41f45
+Revises: fabf697c721d
+Create Date: 2024-04-17 10:42:30.508213
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'f18487b41f45'
+down_revision = 'fabf697c721d'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    # op.drop_table('_alembic_tmp__submissionsampleassociation')
+    # with op.batch_alter_table('_submissionsampleassociation', schema=None) as batch_op:
+    #     batch_op.create_unique_constraint("ssa_id", ['id'])
+
+    with op.batch_alter_table('_wastewaterartic', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('source_plates', sa.JSON(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('_wastewaterartic', schema=None) as batch_op:
+        batch_op.drop_column('source_plates')
+
+    # with op.batch_alter_table('_submissionsampleassociation', schema=None) as batch_op:
+    #     batch_op.drop_constraint(None, type_='unique')
+
+    # op.create_table('_alembic_tmp__submissionsampleassociation',
+    # sa.Column('sample_id', sa.INTEGER(), nullable=False),
+    # sa.Column('submission_id', sa.INTEGER(), nullable=False),
+    # sa.Column('row', sa.INTEGER(), nullable=False),
+    # sa.Column('column', sa.INTEGER(), nullable=False),
+    # sa.Column('base_sub_type', sa.VARCHAR(), nullable=True),
+    # sa.Column('id', sa.INTEGER(), server_default=sa.text('1'), nullable=False),
+    # sa.ForeignKeyConstraint(['sample_id'], ['_basicsample.id'], ),
+    # sa.ForeignKeyConstraint(['submission_id'], ['_basicsubmission.id'], ),
+    # sa.PrimaryKeyConstraint('submission_id', 'row', 'column'),
+    # sa.UniqueConstraint('id', name='ssa_unique')
+    # )
+    # ### end Alembic commands ###

src/submissions/backend/db/models/__init__.py

@@ -86,7 +86,7 @@ class BaseClass(Base):
         """
         Add the object to the database and commit
         """        
-        logger.debug(f"Saving object: {pformat(self.__dict__)}")
+        # logger.debug(f"Saving object: {pformat(self.__dict__)}")
         try:
             self.__database_session__.add(self)
             self.__database_session__.commit()

src/submissions/backend/db/models/controls.py

@@ -4,8 +4,7 @@ All control related models.
 from __future__ import annotations
 from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey
 from sqlalchemy.orm import relationship, Query
-from sqlalchemy_json import NestedMutableJson
-import logging, re
+import logging, re, sys
 from operator import itemgetter
 from . import BaseClass
 from tools import setup_lookup
@@ -13,6 +12,7 @@ from datetime import date, datetime
 from typing import List
 from dateutil.parser import parse
 
+
 logger = logging.getLogger(f"submissions.{__name__}")
 
 class ControlType(BaseClass):
@@ -86,7 +86,6 @@ class ControlType(BaseClass):
         strings = list(set([item.name.split("-")[0] for item in cls.get_positive_control_types()]))
         return re.compile(rf"(^{'|^'.join(strings)})-.*", flags=re.IGNORECASE)
 
-
 class Control(BaseClass):
     """
     Base class of a control sample.
@@ -97,9 +96,9 @@ class Control(BaseClass):
     controltype = relationship("ControlType", back_populates="instances", foreign_keys=[parent_id]) #: reference to parent control type
     name = Column(String(255), unique=True) #: Sample ID
     submitted_date = Column(TIMESTAMP) #: Date submitted to Robotics
-    contains = Column(NestedMutableJson) #: unstructured hashes in contains.tsv for each organism
-    matches = Column(NestedMutableJson) #: unstructured hashes in matches.tsv for each organism
-    kraken = Column(NestedMutableJson) #: unstructured output from kraken_report
+    contains = Column(JSON) #: unstructured hashes in contains.tsv for each organism
+    matches = Column(JSON) #: unstructured hashes in matches.tsv for each organism
+    kraken = Column(JSON) #: unstructured output from kraken_report
     submission_id = Column(INTEGER, ForeignKey("_basicsubmission.id")) #: parent submission id
     submission = relationship("BacterialCulture", back_populates="controls", foreign_keys=[submission_id]) #: parent submission
     refseq_version = Column(String(16)) #: version of refseq used in fastq parsing

src/submissions/backend/db/models/submissions.py

@@ -16,12 +16,15 @@ from . import BaseClass, Reagent, SubmissionType, KitType, Organization
 # See: https://docs.sqlalchemy.org/en/14/orm/extensions/mutable.html#establishing-mutability-on-scalar-column-values
 from sqlalchemy import Column, String, TIMESTAMP, INTEGER, ForeignKey, JSON, FLOAT, case
 from sqlalchemy.orm import relationship, validates, Query
+from sqlalchemy.orm.attributes import flag_modified
 from sqlalchemy.ext.associationproxy import association_proxy
-from sqlalchemy_json import NestedMutableJson
+# from sqlalchemy.ext.declarative import declared_attr
+# from sqlalchemy_json import NestedMutableJson
+# from sqlalchemy.ext.mutable import MutableDict
 from sqlalchemy.exc import OperationalError as AlcOperationalError, IntegrityError as AlcIntegrityError, StatementError
 from sqlite3 import OperationalError as SQLOperationalError, IntegrityError as SQLIntegrityError
 import pandas as pd
-from openpyxl import Workbook
+from openpyxl import Workbook, load_workbook
 from openpyxl.worksheet.worksheet import Worksheet
 from openpyxl.drawing.image import Image as OpenpyxlImage
 from tools import check_not_nan, row_map, setup_lookup, jinja_template_loading, rreplace
@@ -54,10 +57,10 @@ class BasicSubmission(BaseClass):
     # Move this into custom types?
     # reagents = relationship("Reagent", back_populates="submissions", secondary=reagents_submissions) #: relationship to reagents
     reagents_id = Column(String, ForeignKey("_reagent.id", ondelete="SET NULL", name="fk_BS_reagents_id")) #: id of used reagents
-    extraction_info = Column(NestedMutableJson) #: unstructured output from the extraction table logger.
+    extraction_info = Column(JSON) #: unstructured output from the extraction table logger.
     run_cost = Column(FLOAT(2)) #: total cost of running the plate. Set from constant and mutable kit costs at time of creation.
     uploaded_by = Column(String(32)) #: user name of person who submitted the submission to the database.
-    comment = Column(NestedMutableJson) #: user notes
+    comment = Column(JSON) #: user notes
     submission_category = Column(String(64)) #: ["Research", "Diagnostic", "Surveillance", "Validation"], else defaults to submission_type_name
 
     submission_sample_associations = relationship(
@@ -99,6 +102,13 @@ class BasicSubmission(BaseClass):
         submission_type = self.submission_type or "Basic"
         return f"{submission_type}Submission({self.rsl_plate_num})"
 
+    @classmethod
+    def jsons(cls):
+        output = [item.name for item in cls.__table__.columns if isinstance(item.type, JSON)]
+        if issubclass(cls, BasicSubmission) and not cls.__name__ == "BasicSubmission":
+            output += BasicSubmission.jsons()
+        return output
+
     def to_dict(self, full_data:bool=False, backup:bool=False, report:bool=False) -> dict:
         """
         Constructs dictionary used in submissions summary
@@ -315,7 +325,7 @@ class BasicSubmission(BaseClass):
         logger.debug(f"Got {len(subs)} submissions.")
         df = pd.DataFrame.from_records(subs)
         # Exclude sub information
-        for item in ['controls', 'extraction_info', 'pcr_info', 'comment', 'comments', 'samples', 'reagents', 'equipment', 'gel_info', 'gel_image', 'dna_core_submission_number', 'source_plates']:
+        for item in ['controls', 'extraction_info', 'pcr_info', 'comment', 'comments', 'samples', 'reagents', 'equipment', 'gel_info', 'gel_image', 'dna_core_submission_number', 'gel_controls']:
             try:
                 df = df.drop(item, axis=1)
             except:
@@ -360,15 +370,29 @@ class BasicSubmission(BaseClass):
                     field_value = value
             case "ctx" | "csv" | "filepath" | "equipment":
                 return
-            case "comment":
-                if value == "" or value == None or value == 'null':
-                    field_value = None
+            # case "comment":
+            #     if value == "" or value == None or value == 'null':
+            #         field_value = None
+            #     else:
+            #         field_value = dict(name=getuser(), text=value, time=datetime.now())
+            #         # if self.comment is None:
+            #         #     self.comment = [field_value]
+            #         # else:
+            #         #     self.comment.append(field_value)
+            #         self.update_json(field=key, value=field_value)
+            #     return
+            case item if item in self.jsons():
+                logger.debug(f"Setting JSON attribute.")
+                existing = self.__getattribute__(key)
+                if existing is None:
+                    existing = []
+                if value in existing:
+                    logger.warning("Value already exists. Preventing duplicate addition.")
+                    return
                 else:
-                    field_value = dict(name=getuser(), text=value, time=datetime.now())
-                    if self.comment is None:
-                        self.comment = [field_value]
-                    else:
-                        self.comment.append(field_value)
+                    existing.append(value)
+                self.__setattr__(key, existing)
+                flag_modified(self, key)
                 return
             case _:
                 field_value = value
@@ -957,12 +981,13 @@ class BasicSubmission(BaseClass):
         dlg = SubmissionComment(parent=obj, submission=self)
         if dlg.exec():
             comment = dlg.parse_form()
-            try:
-                # For some reason .append results in new comment being ignored, so have to concatenate lists.
-                self.comment = self.comment + comment
-            except (AttributeError, TypeError) as e:
-                logger.error(f"Hit error ({e}) creating comment")
-                self.comment = comment
+            # try:
+            #     # For some reason .append results in new comment being ignored, so have to concatenate lists.
+            #     self.comment = self.comment + comment
+            # except (AttributeError, TypeError) as e:
+            #     logger.error(f"Hit error ({e}) creating comment")
+            #     self.comment = comment
+            self.set_attribute(key='comment', value=comment)
             logger.debug(self.comment)
             self.save(original=False)
 
@@ -1108,15 +1133,6 @@ class BacterialCulture(BasicSubmission):
         template += "_{{ submitting_lab }}_{{ submitter_plate_num }}"
         return template
     
-    # @classmethod
-    # def parse_info(cls, input_dict: dict, xl: pd.ExcelFile | None = None) -> dict:
-    #     """
-    #     Extends parent
-    #     """        
-    #     input_dict = super().parse_info(input_dict, xl)
-    #     input_dict['submitted_date']['missing'] = True
-    #     return input_dict
-
     @classmethod
     def finalize_parse(cls, input_dict: dict, xl: pd.ExcelFile | None = None, info_map: dict | None = None, plate_map: dict | None = None) -> dict:
         """
@@ -1177,7 +1193,7 @@ class Wastewater(BasicSubmission):
     ext_technician = Column(String(64)) #: Name of technician doing extraction
     pcr_technician = Column(String(64)) #: Name of technician doing pcr
     # pcr_info = Column(JSON) 
-    pcr_info = Column(NestedMutableJson)#: unstructured output from pcr table logger or user(Artic)
+    pcr_info = Column(JSON)#: unstructured output from pcr table logger or user(Artic)
 
     __mapper_args__ = __mapper_args__ = dict(polymorphic_identity="Wastewater", 
                            polymorphic_load="inline", 
@@ -1319,35 +1335,36 @@ class Wastewater(BasicSubmission):
         fname = select_open_file(obj=obj, file_extension="xlsx")
         parser = PCRParser(filepath=fname)
         # Check if PCR info already exists
-        if hasattr(self, 'pcr_info') and self.pcr_info != None:
-            # existing = json.loads(sub.pcr_info)
-            existing = self.pcr_info
-            logger.debug(f"Found existing pcr info: {pformat(self.pcr_info)}")
-        else:
-            existing = None
-        if existing != None:
-            # update pcr_info
-            try:
-                logger.debug(f"Updating {type(existing)}:\n {pformat(existing)} with {type(parser.pcr)}:\n {pformat(parser.pcr)}")
-                # if json.dumps(parser.pcr) not in sub.pcr_info:
-                if parser.pcr not in self.pcr_info:
-                    logger.debug(f"This is new pcr info, appending to existing")
-                    existing.append(parser.pcr)
-                else:
-                    logger.debug("This info already exists, skipping.")
-                # logger.debug(f"Setting {self.rsl_plate_num} PCR to:\n {pformat(existing)}")
-                # sub.pcr_info = json.dumps(existing)
-                self.pcr_info = existing
-            except TypeError:
-                logger.error(f"Error updating!")
-                # sub.pcr_info = json.dumps([parser.pcr])
-                self.pcr_info = [parser.pcr]
-            logger.debug(f"Final pcr info for {self.rsl_plate_num}:\n {pformat(self.pcr_info)}")
-        else:
-            # sub.pcr_info = json.dumps([parser.pcr])
-            self.pcr_info = [parser.pcr]
-        # logger.debug(f"Existing {type(self.pcr_info)}: {self.pcr_info}")
-        # logger.debug(f"Inserting {type(parser.pcr)}: {parser.pcr}")
+        # if hasattr(self, 'pcr_info') and self.pcr_info != None:
+        #     # existing = json.loads(sub.pcr_info)
+        #     existing = self.pcr_info
+        #     logger.debug(f"Found existing pcr info: {pformat(self.pcr_info)}")
+        # else:
+        #     existing = None
+        # if existing != None:
+        #     # update pcr_info
+        #     try:
+        #         logger.debug(f"Updating {type(existing)}:\n {pformat(existing)} with {type(parser.pcr)}:\n {pformat(parser.pcr)}")
+        #         # if json.dumps(parser.pcr) not in sub.pcr_info:
+        #         if parser.pcr not in self.pcr_info:
+        #             logger.debug(f"This is new pcr info, appending to existing")
+        #             existing.append(parser.pcr)
+        #         else:
+        #             logger.debug("This info already exists, skipping.")
+        #         # logger.debug(f"Setting {self.rsl_plate_num} PCR to:\n {pformat(existing)}")
+        #         # sub.pcr_info = json.dumps(existing)
+        #         self.pcr_info = existing
+        #     except TypeError:
+        #         logger.error(f"Error updating!")
+        #         # sub.pcr_info = json.dumps([parser.pcr])
+        #         self.pcr_info = [parser.pcr]
+        #     logger.debug(f"Final pcr info for {self.rsl_plate_num}:\n {pformat(self.pcr_info)}")
+        # else:
+        #     # sub.pcr_info = json.dumps([parser.pcr])
+        #     self.pcr_info = [parser.pcr]
+        # # logger.debug(f"Existing {type(self.pcr_info)}: {self.pcr_info}")
+        # # logger.debug(f"Inserting {type(parser.pcr)}: {parser.pcr}")
+        self.set_attribute("pcr_info", parser.pcr)
         self.save(original=False)
         logger.debug(f"Got {len(parser.samples)} samples to update!")
         logger.debug(f"Parser samples: {parser.samples}")
@@ -1367,10 +1384,12 @@ class WastewaterArtic(BasicSubmission):
     id = Column(INTEGER, ForeignKey('_basicsubmission.id'), primary_key=True)
     artic_technician = Column(String(64)) #: Name of technician performing artic
     dna_core_submission_number = Column(String(64)) #: Number used by core as id
-    pcr_info = Column(NestedMutableJson) #: unstructured output from pcr table logger or user(Artic)
+    pcr_info = Column(JSON) #: unstructured output from pcr table logger or user(Artic)
     gel_image = Column(String(64)) #: file name of gel image in zip file
-    gel_info = Column(NestedMutableJson) #: unstructured data from gel.
-    source_plates = Column(NestedMutableJson) #: wastewater plates that samples come from
+    gel_info = Column(JSON) #: unstructured data from gel.
+    gel_controls = Column(JSON) #: locations of controls on the gel
+    source_plates = Column(JSON) #: wastewater plates that samples come from
+
 
     __mapper_args__ = dict(polymorphic_identity="Wastewater Artic", 
                            polymorphic_load="inline", 
@@ -1408,14 +1427,14 @@ class WastewaterArtic(BasicSubmission):
         Returns:
             dict: Updated sample dictionary
         """        
-        from backend.validators import RSLNamer
+        # from backend.validators import RSLNamer
         input_dict = super().parse_info(input_dict)
-        df = xl.parse("First Strand List", header=None)
-        plates = []
-        for row in [8,9,10]:
-            plate_name = RSLNamer(df.iat[row-1, 2]).parsed_name
-            plates.append(dict(plate=plate_name, start_sample=df.iat[row-1, 3]))
-        input_dict['source_plates'] = plates
+        ws = load_workbook(xl.io, data_only=True)['Egel results']
+        data = [ws.cell(row=jj,column=ii) for ii in range(15,27) for jj in range(10,18)]
+        data = [cell for cell in data if cell.value is not None and "NTC" in cell.value]
+        input_dict['gel_controls'] = [dict(sample_id=cell.value, location=f"{row_map[cell.row-9]}{str(cell.column-14).zfill(2)}") for cell in data]
+        # df = xl.parse("Egel results").iloc[7:16, 13:26]
+        # df = df.set_index(df.columns[0])
         return input_dict
 
     @classmethod
@@ -1535,57 +1554,6 @@ class WastewaterArtic(BasicSubmission):
             dict: Updated parser product.
         """        
         input_dict = super().finalize_parse(input_dict, xl, info_map, plate_map)
-        # logger.debug(pformat(input_dict))
-        # logger.debug(pformat(info_map))
-        # logger.debug(pformat(plate_map))
-        # samples = []
-        # for sample in input_dict['samples']:
-        #     logger.debug(f"Input sample: {pformat(sample.__dict__)}")
-        #     if sample.submitter_id == "NTC1":
-        #         samples.append(dict(sample=sample.submitter_id, destination_row=8, destination_column=2, source_row=0, source_column=0, plate_number='control', plate=None))
-        #         continue
-        #     elif sample.submitter_id == "NTC2":
-        #         samples.append(dict(sample=sample.submitter_id, destination_row=8, destination_column=5, source_row=0, source_column=0, plate_number='control', plate=None))
-        #         continue
-        #     destination_row = sample.row[0]
-        #     destination_column = sample.column[0]
-        #     # logger.debug(f"Looking up: {sample.submitter_id} friend.")
-        #     lookup_sample = BasicSample.query(submitter_id=sample.submitter_id)
-        #     lookup_ssa = SubmissionSampleAssociation.query(sample=lookup_sample, exclude_submission_type=cls.__mapper_args__['polymorphic_identity'] , chronologic=True, reverse=True, limit=1)
-        #     try:
-        #         plate = lookup_ssa.submission.rsl_plate_num
-        #         source_row = lookup_ssa.row
-        #         source_column = lookup_ssa.column
-        #     except AttributeError as e:
-        #         logger.error(f"Problem with lookup: {e}")
-        #         plate = "Error"
-        #         source_row = 0
-        #         source_column = 0
-        #         # continue
-        #     output_sample = dict(
-        #         sample=sample.submitter_id,
-        #         destination_column=destination_column, 
-        #         destination_row=destination_row,
-        #         plate=plate,
-        #         source_column=source_column,
-        #         source_row = source_row
-        #         )
-        #     logger.debug(f"output sample: {pformat(output_sample)}")
-        #     samples.append(output_sample)
-        # plates = sorted(list(set([sample['plate'] for sample in samples if sample['plate'] != None and sample['plate'] != "Error"])))
-        # logger.debug(f"Here's what I got for plates: {plates}")
-        # for iii, plate in enumerate(plates):
-        #     for sample in samples:
-        #         if sample['plate'] == plate:
-        #             sample['plate_number'] = iii + 1
-        # df = pd.DataFrame.from_records(samples).fillna(value="")
-        # try:
-        #     df.source_row = df.source_row.astype(int)
-        #     df.source_column = df.source_column.astype(int)
-        #     df.sort_values(by=['destination_column', 'destination_row'], inplace=True)
-        # except AttributeError as e:
-        #     logger.error(f"Couldn't construct df due to {e}")
-        # input_dict['csv'] = df
         input_dict['csv'] = xl.parse("hitpicks_csv_to_export")
         return input_dict
 
@@ -1606,30 +1574,30 @@ class WastewaterArtic(BasicSubmission):
         worksheet = input_excel["First Strand List"]
         samples = cls.query(rsl_number=info['rsl_plate_num']['value']).submission_sample_associations
         samples = sorted(samples, key=attrgetter('column', 'row'))
-        try:
-            source_plates = [item['plate'] for item in info['source_plates']]
-            first_samples = [item['start_sample'] for item in info['source_plates']]
-        except:
-            source_plates = []
-            first_samples = []
-            for sample in samples:
-                sample = sample.sample
-                try:
-                    assoc = [item.submission.rsl_plate_num for item in sample.sample_submission_associations if item.submission.submission_type_name=="Wastewater"][-1]
-                except IndexError:
-                    logger.error(f"Association not found for {sample}")
-                    continue
-                if assoc not in source_plates:
-                    source_plates.append(assoc)
-                    first_samples.append(sample.ww_processing_num)
-        # Pad list to length of 3
-        source_plates += ['None'] * (3 - len(source_plates))
-        first_samples += [''] * (3 - len(first_samples))
-        source_plates = zip(source_plates, first_samples, strict=False)
-        for iii, plate in enumerate(source_plates, start=8):
-            logger.debug(f"Plate: {plate}")
-            for jjj, value in enumerate(plate, start=3):
-                worksheet.cell(row=iii, column=jjj, value=value)
+        # try:
+        #     source_plates = [item['plate'] for item in info['source_plates']]
+        #     first_samples = [item['start_sample'] for item in info['source_plates']]
+        # except:
+        #     source_plates = []
+        #     first_samples = []
+        #     for sample in samples:
+        #         sample = sample.sample
+        #         try:
+        #             assoc = [item.submission.rsl_plate_num for item in sample.sample_submission_associations if item.submission.submission_type_name=="Wastewater"][-1]
+        #         except IndexError:
+        #             logger.error(f"Association not found for {sample}")
+        #             continue
+        #         if assoc not in source_plates:
+        #             source_plates.append(assoc)
+        #             first_samples.append(sample.ww_processing_num)
+        # # Pad list to length of 3
+        # source_plates += ['None'] * (3 - len(source_plates))
+        # first_samples += [''] * (3 - len(first_samples))
+        # source_plates = zip(source_plates, first_samples, strict=False)
+        # for iii, plate in enumerate(source_plates, start=8):
+        #     logger.debug(f"Plate: {plate}")
+        #     for jjj, value in enumerate(plate, start=3):
+        #         worksheet.cell(row=iii, column=jjj, value=value)
         logger.debug(f"Info:\n{pformat(info)}")
         check = 'gel_info' in info.keys() and info['gel_info']['value'] != None
         if check:
@@ -1676,7 +1644,7 @@ class WastewaterArtic(BasicSubmission):
             Tuple[dict, Template]: (Updated dictionary, Template to be rendered)
         """        
         base_dict, template = super().get_details_template(base_dict=base_dict)
-        base_dict['excluded'] += ['gel_info', 'gel_image', 'headers', "dna_core_submission_number", "source_plates"]
+        base_dict['excluded'] += ['gel_info', 'gel_image', 'headers', "dna_core_submission_number", "source_plates", "gel_controls"]
         base_dict['DNA Core ID'] = base_dict['dna_core_submission_number']
         check = 'gel_info' in base_dict.keys() and base_dict['gel_info'] != None
         if check:
@@ -1739,7 +1707,7 @@ class WastewaterArtic(BasicSubmission):
         from frontend.widgets.gel_checker import GelBox
         from frontend.widgets import select_open_file
         fname = select_open_file(obj=obj, file_extension="jpg")
-        dlg = GelBox(parent=obj, img_path=fname)
+        dlg = GelBox(parent=obj, img_path=fname, submission=self)
         if dlg.exec():
             self.dna_core_submission_number, img_path, output, comment = dlg.parse_form()
             self.gel_image = img_path.name
@@ -2375,7 +2343,7 @@ class WastewaterAssociation(SubmissionSampleAssociation):
     ct_n2 = Column(FLOAT(2)) #: AKA ct for N2
     n1_status = Column(String(32)) #: positive or negative for N1
     n2_status = Column(String(32)) #: positive or negative for N2
-    pcr_results = Column(NestedMutableJson) #: imported PCR status from QuantStudio
+    pcr_results = Column(JSON) #: imported PCR status from QuantStudio
 
     __mapper_args__ = dict(polymorphic_identity="Wastewater Association", 
                            polymorphic_load="inline", 

src/submissions/frontend/widgets/gel_checker.py

@@ -7,24 +7,26 @@ from PyQt6.QtWidgets import (QWidget, QDialog, QGridLayout,
                              )
 import numpy as np
 import pyqtgraph as pg
-from PyQt6.QtGui import QIcon
+from PyQt6.QtGui import QIcon, QFont
 from PIL import Image
 import numpy as np
 import logging
 from pprint import pformat
 from typing import Tuple, List
 from pathlib import Path
+from backend.db.models import WastewaterArtic
 
 logger = logging.getLogger(f"submissions.{__name__}")
 
 # Main window class
 class GelBox(QDialog):
 
-    def __init__(self, parent, img_path:str|Path):
+    def __init__(self, parent, img_path:str|Path, submission:WastewaterArtic):
         super().__init__(parent)
         # setting title
         self.setWindowTitle("PyQtGraph")
         self.img_path = img_path
+        self.submission = submission
         # setting geometry
         self.setGeometry(50, 50, 1200, 900)
         # icon
@@ -57,7 +59,11 @@ class GelBox(QDialog):
         # plot window goes on right side, spanning 3 rows
         layout.addWidget(self.imv, 1, 1,20,20)
         # setting this widget as central widget of the main window
-        self.form = ControlsForm(parent=self)
+        try:
+            control_info = sorted(self.submission.gel_controls, key=lambda d: d['location'])
+        except KeyError:
+            control_info = None
+        self.form = ControlsForm(parent=self, control_info=control_info)
         layout.addWidget(self.form,22,1,1,4)
         QBtn = QDialogButtonBox.StandardButton.Ok | QDialogButtonBox.StandardButton.Cancel
         self.buttonBox = QDialogButtonBox(QBtn)
@@ -79,16 +85,24 @@ class GelBox(QDialog):
         
 class ControlsForm(QWidget):
 
-    def __init__(self, parent) -> None:
+    def __init__(self, parent, control_info:List=None) -> None:
         super().__init__(parent)
         self.layout = QGridLayout()
         columns = []
         rows = []
+        try:
+            tt_text = "\n".join([f"{item['sample_id']} - CELL {item['location']}" for item in control_info])
+        except TypeError:
+            tt_text = None
         for iii, item in enumerate(["Negative Control Key", "Description", "Results - 65 C", "Results - 63 C", "Results - Spike"]):
             label = QLabel(item)
             self.layout.addWidget(label, 0, iii,1,1)
             if iii > 1:
                 columns.append(item)
+            elif iii == 0:
+                if tt_text:
+                    label.setStyleSheet("font-weight: bold; color: blue; text-decoration: underline;")
+                    label.setToolTip(tt_text)
         for iii, item in enumerate(["RSL-NTC", "ENC-NTC", "NTC"], start=1):
             label = QLabel(item)
             self.layout.addWidget(label, iii, 0, 1, 1)
@@ -102,6 +116,11 @@ class ControlsForm(QWidget):
                 widge.setText("Neg")
                 widge.setObjectName(f"{rows[iii]} : {columns[jjj]}")
                 self.layout.addWidget(widge, iii+1, jjj+2, 1, 1)
+        # try:
+        #     for iii, item in enumerate(control_info, start=1):
+        #         self.layout.addWidget(QLabel(f"{item['sample_id']} - {item['location']}"), iii+4, 1)
+        # except TypeError:
+        #     pass
         self.layout.addWidget(QLabel("Comments:"), 0,5,1,1)
         self.comment_field = QTextEdit(self)
         self.comment_field.setFixedHeight(50)

src/submissions/frontend/widgets/submission_details.py

@@ -95,8 +95,8 @@ class SubmissionDetails(QDialog):
         self.html = self.template.render(sub=self.base_dict, signing_permission=is_power_user())
         self.webview.setHtml(self.html)
         self.setWindowTitle(f"Submission Details - {submission.rsl_plate_num}")
-        with open("details.html", "w") as f:
-            f.write(self.html)
+        # with open("details.html", "w") as f:
+        #     f.write(self.html)
 
     @pyqtSlot(str)
     def sign_off(self, submission:str|BasicSubmission):
@@ -171,7 +171,7 @@ class SubmissionComment(QDialog):
         commenter = getuser()
         comment = self.txt_editor.toPlainText()
         dt = datetime.strftime(datetime.now(), "%Y-%m-%d %H:%M:%S")
-        full_comment = [{"name":commenter, "time": dt, "text": comment}]
+        full_comment = {"name":commenter, "time": dt, "text": comment}
         logger.debug(f"Full comment: {full_comment}")
         return full_comment
 

src/submissions/frontend/widgets/submission_table.py

@@ -1,7 +1,7 @@
 '''
 Contains widgets specific to the submission summary and submission details.
 '''
-import logging, json
+import logging
 from pprint import pformat
 from PyQt6.QtWidgets import QTableView, QMenu
 from PyQt6.QtCore import Qt, QAbstractTableModel, QSortFilterProxyModel
@@ -177,35 +177,36 @@ class SubmissionsSheet(QTableView):
                 count += 1
             except AttributeError:
                 continue
-            if sub.extraction_info != None:
-                # existing = json.loads(sub.extraction_info)
-                existing = sub.extraction_info
-            else:
-                existing = None
-            # Check if the new info already exists in the imported submission
-            try:
-                # if json.dumps(new_run) in sub.extraction_info:
-                if new_run in sub.extraction_info:
-                    logger.debug(f"Looks like we already have that info.")
-                    continue
-            except TypeError:
-                pass
-            # Update or create the extraction info
-            if existing != None:
-                try:
-                    logger.debug(f"Updating {type(existing)}: {existing} with {type(new_run)}: {new_run}")
-                    existing.append(new_run)
-                    logger.debug(f"Setting: {existing}")
-                    # sub.extraction_info = json.dumps(existing)
-                    sub.extraction_info = existing
-                except TypeError:
-                    logger.error(f"Error updating!")
-                    # sub.extraction_info = json.dumps([new_run])
-                    sub.extraction_info = [new_run]
-                logger.debug(f"Final ext info for {sub.rsl_plate_num}: {sub.extraction_info}")
-            else:
-                # sub.extraction_info = json.dumps([new_run])
-                sub.extraction_info = [new_run]
+            sub.set_attribute('extraction_info', new_run)
+            # if sub.extraction_info != None:
+            #     # existing = json.loads(sub.extraction_info)
+            #     existing = sub.extraction_info
+            # else:
+            #     existing = None
+            # # Check if the new info already exists in the imported submission
+            # try:
+            #     # if json.dumps(new_run) in sub.extraction_info:
+            #     if new_run in sub.extraction_info:
+            #         logger.debug(f"Looks like we already have that info.")
+            #         continue
+            # except TypeError:
+            #     pass
+            # # Update or create the extraction info
+            # if existing != None:
+            #     try:
+            #         logger.debug(f"Updating {type(existing)}: {existing} with {type(new_run)}: {new_run}")
+            #         existing.append(new_run)
+            #         logger.debug(f"Setting: {existing}")
+            #         # sub.extraction_info = json.dumps(existing)
+            #         sub.extraction_info = existing
+            #     except TypeError:
+            #         logger.error(f"Error updating!")
+            #         # sub.extraction_info = json.dumps([new_run])
+            #         sub.extraction_info = [new_run]
+            #     logger.debug(f"Final ext info for {sub.rsl_plate_num}: {sub.extraction_info}")
+            # else:
+            #     # sub.extraction_info = json.dumps([new_run])
+            #     sub.extraction_info = [new_run]
             sub.save()
         self.report.add_result(Result(msg=f"We added {count} logs to the database.", status='Information'))
 
@@ -253,37 +254,38 @@ class SubmissionsSheet(QTableView):
                 logger.debug(f"Found submission: {sub.rsl_plate_num}")
             except AttributeError:
                 continue
-            # check if pcr_info already exists
-            if hasattr(sub, 'pcr_info') and sub.pcr_info != None:
-                # existing = json.loads(sub.pcr_info)
-                existing = sub.pcr_info
-            else:
-                existing = None
-            # check if this entry already exists in imported submission
-            try:
-                # if json.dumps(new_run) in sub.pcr_info:
-                if new_run in sub.pcr_info:
-                    logger.debug(f"Looks like we already have that info.")
-                    continue
-                else:
-                    count += 1
-            except TypeError:
-                logger.error(f"No json to dump")
-            if existing is not None:
-                try:
-                    logger.debug(f"Updating {type(existing)}: {existing} with {type(new_run)}: {new_run}")
-                    existing.append(new_run)
-                    logger.debug(f"Setting: {existing}")
-                    # sub.pcr_info = json.dumps(existing)
-                    sub.pcr_info = existing
-                except TypeError:
-                    logger.error(f"Error updating!")
-                    # sub.pcr_info = json.dumps([new_run])
-                    sub.pcr_info = [new_run]
-                logger.debug(f"Final ext info for {sub.rsl_plate_num}: {sub.pcr_info}")
-            else:
-                # sub.pcr_info = json.dumps([new_run])
-                sub.pcr_info = [new_run]
+            sub.set_attribute('pcr_info', new_run)
+            # # check if pcr_info already exists
+            # if hasattr(sub, 'pcr_info') and sub.pcr_info != None:
+            #     # existing = json.loads(sub.pcr_info)
+            #     existing = sub.pcr_info
+            # else:
+            #     existing = None
+            # # check if this entry already exists in imported submission
+            # try:
+            #     # if json.dumps(new_run) in sub.pcr_info:
+            #     if new_run in sub.pcr_info:
+            #         logger.debug(f"Looks like we already have that info.")
+            #         continue
+            #     else:
+            #         count += 1
+            # except TypeError:
+            #     logger.error(f"No json to dump")
+            # if existing is not None:
+            #     try:
+            #         logger.debug(f"Updating {type(existing)}: {existing} with {type(new_run)}: {new_run}")
+            #         existing.append(new_run)
+            #         logger.debug(f"Setting: {existing}")
+            #         # sub.pcr_info = json.dumps(existing)
+            #         sub.pcr_info = existing
+            #     except TypeError:
+            #         logger.error(f"Error updating!")
+            #         # sub.pcr_info = json.dumps([new_run])
+            #         sub.pcr_info = [new_run]
+            #     logger.debug(f"Final ext info for {sub.rsl_plate_num}: {sub.pcr_info}")
+            # else:
+            #     # sub.pcr_info = json.dumps([new_run])
+            #     sub.pcr_info = [new_run]
             sub.save()
         self.report.add_result(Result(msg=f"We added {count} logs to the database.", status='Information'))
         

src/submissions/frontend/widgets/submission_widget.py

@@ -5,14 +5,14 @@ from PyQt6.QtWidgets import (
 from PyQt6.QtCore import pyqtSignal
 from pathlib import Path
 from . import select_open_file, select_save_file
-import logging, difflib, inspect, pickle
+import logging, difflib, inspect
 from pathlib import Path
 from tools import Report, Result, check_not_nan
-from backend.excel.parser import SheetParser, PCRParser
+from backend.excel.parser import SheetParser
 from backend.validators import PydSubmission, PydReagent
 from backend.db import (
     KitType, Organization, SubmissionType, Reagent, 
-    ReagentType, KitTypeReagentTypeAssociation, BasicSubmission
+    ReagentType, KitTypeReagentTypeAssociation
 )
 from pprint import pformat
 from .pop_ups import QuestionAsker, AlertPop
@@ -154,7 +154,7 @@ class SubmissionFormWidget(QWidget):
         # self.samples = []
         self.missing_info = []
         self.ignore = ['filepath', 'samples', 'reagents', 'csv', 'ctx', 'comment', 
-                       'equipment', 'source_plates', 'id', 'cost', 'extraction_info', 
+                       'equipment', 'gel_controls', 'id', 'cost', 'extraction_info', 
                        'controls', 'pcr_info', 'gel_info', 'gel_image']
         self.recover = ['filepath', 'samples', 'csv', 'comment', 'equipment']
         self.layout = QVBoxLayout()