new parsers/DB objects, pre code cleanup

alembic/versions/3d9a88bd4ecd_added_in_other_ww_techs.py

@@ -0,0 +1,34 @@
+"""added in other ww techs
+
+Revision ID: 3d9a88bd4ecd
+Revises: f7f46e72f057
+Create Date: 2023-08-30 11:03:41.575219
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '3d9a88bd4ecd'
+down_revision = 'f7f46e72f057'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('_submissions', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('ext_technician', sa.String(length=64), nullable=True))
+        batch_op.add_column(sa.Column('pcr_technician', sa.String(length=64), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('_submissions', schema=None) as batch_op:
+        batch_op.drop_column('pcr_technician')
+        batch_op.drop_column('ext_technician')
+
+    # ### end Alembic commands ###

alembic/versions/9a133efb3ffd_adjusting_reagents_reagenttypes_to_many_.py

@@ -0,0 +1,33 @@
+"""adjusting reagents/reagenttypes to many-to-many
+
+Revision ID: 9a133efb3ffd
+Revises: 3d9a88bd4ecd
+Create Date: 2023-09-01 10:28:22.335890
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '9a133efb3ffd'
+down_revision = '3d9a88bd4ecd'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('_reagenttypes_reagents',
+    sa.Column('reagent_id', sa.INTEGER(), nullable=True),
+    sa.Column('reagenttype_id', sa.INTEGER(), nullable=True),
+    sa.ForeignKeyConstraint(['reagent_id'], ['_reagents.id'], ),
+    sa.ForeignKeyConstraint(['reagenttype_id'], ['_reagent_types.id'], )
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('_reagenttypes_reagents')
+    # ### end Alembic commands ###

alembic/versions/f7f46e72f057_rebuild_database.py

@@ -1,8 +1,8 @@
 """rebuild database
 
-Revision ID: cac89ced412b
+Revision ID: f7f46e72f057
 Revises: 
-Create Date: 2023-08-25 14:03:48.883090
+Create Date: 2023-08-30 09:47:18.071070
 
 """
 from alembic import op
@@ -10,7 +10,7 @@ import sqlalchemy as sa
 
 
 # revision identifiers, used by Alembic.
-revision = 'cac89ced412b'
+revision = 'f7f46e72f057'
 down_revision = None
 branch_labels = None
 depends_on = None
@@ -168,8 +168,8 @@ def upgrade() -> None:
     op.create_table('_submission_sample',
     sa.Column('sample_id', sa.INTEGER(), nullable=False),
     sa.Column('submission_id', sa.INTEGER(), nullable=False),
-    sa.Column('row', sa.INTEGER(), nullable=True),
+    sa.Column('row', sa.INTEGER(), nullable=False),
-    sa.Column('column', sa.INTEGER(), nullable=True),
+    sa.Column('column', sa.INTEGER(), nullable=False),
     sa.Column('base_sub_type', sa.String(), nullable=True),
     sa.Column('ct_n1', sa.FLOAT(precision=2), nullable=True),
     sa.Column('ct_n2', sa.FLOAT(precision=2), nullable=True),
@@ -178,7 +178,7 @@ def upgrade() -> None:
     sa.Column('pcr_results', sa.JSON(), nullable=True),
     sa.ForeignKeyConstraint(['sample_id'], ['_samples.id'], ),
     sa.ForeignKeyConstraint(['submission_id'], ['_submissions.id'], ),
-    sa.PrimaryKeyConstraint('sample_id', 'submission_id')
+    sa.PrimaryKeyConstraint('submission_id', 'row', 'column')
     )
     # ### end Alembic commands ###
 

src/submissions/backend/db/functions.py

@@ -2,6 +2,7 @@
 Convenience functions for interacting with the database.
 '''
 
+import pprint
 from . import models
 # from .models.kits import KitType
 # from .models.submissions import BasicSample, reagents_submissions, BasicSubmission, SubmissionSampleAssociation
@@ -33,7 +34,6 @@ def set_sqlite_pragma(dbapi_connection, connection_record):
     cursor.execute("PRAGMA foreign_keys=ON")
     cursor.close()
 
-
 def store_submission(ctx:Settings, base_submission:models.BasicSubmission, samples:List[dict]=[]) -> None|dict:
     """
     Upserts submissions into database
@@ -206,10 +206,12 @@ def construct_submission_info(ctx:Settings, info_dict:dict) -> models.BasicSubmi
                     try:
                         with ctx.database_session.no_autoflush:
                             try:
-                                logger.debug(f"Here is the sample instance type: {sample_instance.sample_type}")
+                                sample_query = sample_instance.sample_type.replace('Sample', '').strip()
+                                logger.debug(f"Here is the sample instance type: {sample_query}")
                                 try:
-                                    assoc = getattr(models, f"{sample_instance.sample_type.replace('_sample', '').replace('_', ' ').title().replace(' ', '')}Association")
+                                    assoc = getattr(models, f"{sample_query}Association")
                                 except AttributeError as e:
+                                    logger.error(f"Couldn't get type specific association. Getting generic.")
                                     assoc = models.SubmissionSampleAssociation
                                 # assoc = models.SubmissionSampleAssociation(submission=instance, sample=sample_instance, row=sample['row'], column=sample['column'])
                                 assoc = assoc(submission=instance, sample=sample_instance, row=sample['row'], column=sample['column'])
@@ -287,7 +289,9 @@ def construct_reagent(ctx:Settings, info_dict:dict) -> models.Reagent:
             case "expiry":
                 reagent.expiry = info_dict[item]
             case "type":
-                reagent.type = lookup_reagenttype_by_name(ctx=ctx, rt_name=info_dict[item])
+                reagent_type = lookup_reagenttype_by_name(ctx=ctx, rt_name=info_dict[item])
+                if reagent_type != None:
+                    reagent.type.append(reagent_type)
             case "name":
                 if item == None:
                     reagent.name = reagent.type.name
@@ -420,7 +424,7 @@ def lookup_regent_by_type_name_and_kit_name(ctx:Settings, type_name:str, kit_nam
     output = rt_types.instances
     return output
 
-def lookup_all_submissions_by_type(ctx:Settings, sub_type:str|None=None) -> list[models.BasicSubmission]:
+def lookup_all_submissions_by_type(ctx:Settings, sub_type:str|None=None, chronologic:bool=False) -> list[models.BasicSubmission]:
     """
     Get all submissions, filtering by type if given
 
@@ -433,11 +437,13 @@ def lookup_all_submissions_by_type(ctx:Settings, sub_type:str|None=None) -> list
     """
     if sub_type == None:
         # subs = ctx['database_session'].query(models.BasicSubmission).all()
-        subs = ctx.database_session.query(models.BasicSubmission).all()
+        subs = ctx.database_session.query(models.BasicSubmission)
     else:
         # subs = ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.submission_type==sub_type.lower().replace(" ", "_")).all()
-        subs = ctx.database_session.query(models.BasicSubmission).filter(models.BasicSubmission.submission_type==sub_type.lower().replace(" ", "_")).all()
+        subs = ctx.database_session.query(models.BasicSubmission).filter(models.BasicSubmission.submission_type==sub_type.lower().replace(" ", "_"))
-    return subs
+    if chronologic:
+        subs.order_by(models.BasicSubmission.submitted_date)
+    return subs.all()
 
 def lookup_all_orgs(ctx:Settings) -> list[models.Organization]:
     """
@@ -480,7 +486,7 @@ def submissions_to_df(ctx:Settings, sub_type:str|None=None) -> pd.DataFrame:
     """    
     logger.debug(f"Type: {sub_type}")
     # use lookup function to create list of dicts
-    subs = [item.to_dict() for item in lookup_all_submissions_by_type(ctx=ctx, sub_type=sub_type)]
+    subs = [item.to_dict() for item in lookup_all_submissions_by_type(ctx=ctx, sub_type=sub_type, chronologic=True)]
     # make df from dicts (records) in list
     df = pd.DataFrame.from_records(subs)
     # Exclude sub information
@@ -569,7 +575,9 @@ def create_kit_from_yaml(ctx:Settings, exp:dict) -> dict:
         #     continue
         # A submission type may use multiple kits.
         for kt in exp[type]['kits']:
+            logger.debug(f"Looking up submission type: {type}")
             submission_type = lookup_submissiontype_by_name(ctx=ctx, type_name=type)
+            logger.debug(f"Looked up submission type: {submission_type}")
             kit = models.KitType(name=kt, 
                                 #  constant_cost=exp[type]["kits"][kt]["constant_cost"], 
                                 #  mutable_cost_column=exp[type]["kits"][kt]["mutable_cost_column"],
@@ -588,7 +596,7 @@ def create_kit_from_yaml(ctx:Settings, exp:dict) -> dict:
                 look_up = ctx.database_session.query(models.ReagentType).filter(models.ReagentType.name==r).first()
                 if look_up == None:
                     # rt = models.ReagentType(name=r.replace(" ", "_").lower(), eol_ext=timedelta(30*exp[type]['kits'][kt]['reagenttypes'][r]['eol_ext']), kits=[kit], required=1)
-                    rt = models.ReagentType(name=r.replace(" ", "_").lower().strip(), eol_ext=timedelta(30*exp[type]['kits'][kt]['reagenttypes'][r]['eol_ext']), last_used="")
+                    rt = models.ReagentType(name=r.strip(), eol_ext=timedelta(30*exp[type]['kits'][kt]['reagenttypes'][r]['eol_ext']), last_used="")
                 else:
                     rt = look_up
                     # rt.kits.append(kit)
@@ -893,9 +901,10 @@ def update_ww_sample(ctx:Settings, sample_obj:dict):
         sample_obj (dict): dictionary representing new values for database object
     """    
     # ww_samp = lookup_ww_sample_by_rsl_sample_number(ctx=ctx, rsl_number=sample_obj['sample'])
+    logger.debug(f"dictionary to use for update: {pprint.pformat(sample_obj)}")
     logger.debug(f"Looking up {sample_obj['sample']} in plate {sample_obj['plate_rsl']}")
     # ww_samp = lookup_ww_sample_by_sub_sample_rsl(ctx=ctx, sample_rsl=sample_obj['sample'], plate_rsl=sample_obj['plate_rsl'])
-    assoc = lookup_ww_association_by_plate_sample(ctx=ctx, rsl_plate_num=sample_obj['plate_rsl'], rsl_sample_num=sample_obj['sample'])
+    assoc = lookup_subsamp_association_by_plate_sample(ctx=ctx, rsl_plate_num=sample_obj['plate_rsl'], rsl_sample_num=sample_obj['sample'])
     # ww_samp = lookup_ww_sample_by_sub_sample_well(ctx=ctx, sample_rsl=sample_obj['sample'], well_num=sample_obj['well_num'], plate_rsl=sample_obj['plate_rsl'])
     if assoc != None:
         # del sample_obj['well_number']
@@ -903,11 +912,16 @@ def update_ww_sample(ctx:Settings, sample_obj:dict):
             # set attribute 'key' to 'value'
             try:
                 check = getattr(assoc, key)
-            except AttributeError:
+            except AttributeError as e:
+                logger.error(f"Item doesn't have field {key} due to {e}")
                 continue
-            if check == None:
+            if check != value:
-                logger.debug(f"Setting {key} to {value}")
+                logger.debug(f"Setting association key: {key} to {value}")
-                setattr(assoc, key, value)
+                try:
+                    setattr(assoc, key, value)
+                except AttributeError as e:
+                    logger.error(f"Can't set field {key} to {value} due to {e}")
+                    continue
     else:
         logger.error(f"Unable to find sample {sample_obj['sample']}")
         return
@@ -1059,16 +1073,22 @@ def check_kit_integrity(sub:models.BasicSubmission|models.KitType, reagenttypes:
     """    
     logger.debug(type(sub))
     # What type is sub?
+    reagenttypes = []
     match sub:
         case models.BasicSubmission():
             # Get all required reagent types for this kit.
             # ext_kit_rtypes = [reagenttype.name for reagenttype in sub.extraction_kit.reagent_types if reagenttype.required == 1]
             ext_kit_rtypes = [item.name for item in sub.extraction_kit.get_reagents(required=True)]
             # Overwrite function parameter reagenttypes
-            try:
+            for reagent in sub.reagents:
-                reagenttypes = [reagent.type.name for reagent in sub.reagents]
+                try:
-            except AttributeError as e:
+                    # reagenttypes = [reagent.type.name for reagent in sub.reagents]
-                logger.error(f"Problem parsing reagents: {[f'{reagent.lot}, {reagent.type}' for reagent in sub.reagents]}")
+                    rt = list(set(reagent.type).intersection(sub.extraction_kit.reagent_types))[0].name
+                    logger.debug(f"Got reagent type: {rt}")
+                    reagenttypes.append(rt)
+                except AttributeError as e:
+                    logger.error(f"Problem parsing reagents: {[f'{reagent.lot}, {reagent.type}' for reagent in sub.reagents]}")
+                    reagenttypes.append(reagent.type[0].name)
         case models.KitType():
             # ext_kit_rtypes = [reagenttype.name for reagenttype in sub.reagent_types if reagenttype.required == 1]
             ext_kit_rtypes = [item.name for item in sub.get_reagents(required=True)]
@@ -1133,7 +1153,7 @@ def get_reagents_in_extkit(ctx:Settings, kit_name:str) -> List[str]:
     kit = lookup_kittype_by_name(ctx=ctx, name=kit_name)
     return kit.get_reagents(required=False)
 
-def lookup_ww_association_by_plate_sample(ctx:Settings, rsl_plate_num:str, rsl_sample_num:str) -> models.SubmissionSampleAssociation:
+def lookup_subsamp_association_by_plate_sample(ctx:Settings, rsl_plate_num:str, rsl_sample_num:str) -> models.SubmissionSampleAssociation:
     """
     _summary_
 
@@ -1147,9 +1167,9 @@ def lookup_ww_association_by_plate_sample(ctx:Settings, rsl_plate_num:str, rsl_s
     """    
     return ctx.database_session.query(models.SubmissionSampleAssociation)\
                 .join(models.BasicSubmission)\
-                .join(models.WastewaterSample)\
+                .join(models.BasicSample)\
                 .filter(models.BasicSubmission.rsl_plate_num==rsl_plate_num)\
-                .filter(models.WastewaterSample.rsl_number==rsl_sample_num)\
+                .filter(models.BasicSample.submitter_id==rsl_sample_num)\
                 .first()
 
 def lookup_all_reagent_names_by_role(ctx:Settings, role_name:str) -> List[str]:
@@ -1182,3 +1202,24 @@ def lookup_submissiontype_by_name(ctx:Settings, type_name:str) -> models.Submiss
     """    
 
     return ctx.database_session.query(models.SubmissionType).filter(models.SubmissionType.name==type_name).first()
+
+def add_reagenttype_to_kit(ctx:Settings, rt_name:str, kit_name:str, eol:int=0):
+    """
+    Mostly commandline procedure to add missing reagenttypes to kits
+
+    Args:
+        ctx (Settings): _description_
+        rt_name (str): _description_
+        kit_name (str): _description_
+        eol (int, optional): _description_. Defaults to 0.
+    """    
+    kit = lookup_kittype_by_name(ctx=ctx, name=kit_name)
+    rt = lookup_reagenttype_by_name(ctx=ctx, rt_name=rt_name)
+    if rt == None:
+        rt = models.ReagentType(name=rt_name.strip(), eol_ext=timedelta(30*eol), last_used="")
+        ctx.database_session.add(rt)
+    assoc = models.KitTypeReagentTypeAssociation(kit_type=kit, reagent_type=rt, uses={})
+    kit.kit_reagenttype_associations.append(assoc)
+    ctx.database_session.add(kit)
+    ctx.database_session.commit()
+    

src/submissions/backend/db/models/kits.py

@@ -2,7 +2,7 @@
 All kit and reagent related models
 '''
 from . import Base
-from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey, Interval, Table, FLOAT, CheckConstraint
+from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey, Interval, Table, FLOAT
 from sqlalchemy.orm import relationship, validates
 from sqlalchemy.ext.associationproxy import association_proxy
 
@@ -22,6 +22,8 @@ logger = logging.getLogger(f'submissions.{__name__}')
 #                               Column("required", INTEGER)
 #                               )
 
+reagenttypes_reagents = Table("_reagenttypes_reagents", Base.metadata, Column("reagent_id", INTEGER, ForeignKey("_reagents.id")), Column("reagenttype_id", INTEGER, ForeignKey("_reagent_types.id")))
+
 
 class KitType(Base):
     """
@@ -47,7 +49,7 @@ class KitType(Base):
 
     # association proxy of "user_keyword_associations" collection
     # to "keyword" attribute
-    reagent_types = association_proxy("kit_reagenttype_associations", "reagenttype")
+    reagent_types = association_proxy("kit_reagenttype_associations", "reagent_type")
 
 
     kit_submissiontype_associations = relationship(
@@ -139,7 +141,7 @@ class ReagentType(Base):
     name = Column(String(64)) #: name of reagent type
     # kit_id = Column(INTEGER, ForeignKey("_kits.id", ondelete="SET NULL", use_alter=True, name="fk_RT_kits_id")) #: id of joined kit type
     # kits = relationship("KitType", back_populates="reagent_types", uselist=True, foreign_keys=[kit_id]) #: kits this reagent is used in
-    instances = relationship("Reagent", back_populates="type") #: concrete instances of this reagent type
+    instances = relationship("Reagent", back_populates="type", secondary=reagenttypes_reagents) #: concrete instances of this reagent type
     eol_ext = Column(Interval()) #: extension of life interval
     # required = Column(INTEGER, server_default="1") #: sqlite boolean to determine if reagent type is essential for the kit
     last_used = Column(String(32)) #: last used lot number of this type of reagent
@@ -169,7 +171,7 @@ class Reagent(Base):
     __tablename__ = "_reagents"
 
     id = Column(INTEGER, primary_key=True) #: primary key
-    type = relationship("ReagentType", back_populates="instances") #: joined parent reagent type
+    type = relationship("ReagentType", back_populates="instances", secondary=reagenttypes_reagents) #: joined parent reagent type
     type_id = Column(INTEGER, ForeignKey("_reagent_types.id", ondelete='SET NULL', name="fk_reagent_type_id")) #: id of parent reagent type
     name = Column(String(64)) #: reagent name
     lot = Column(String(64)) #: lot number of reagent
@@ -192,19 +194,26 @@ class Reagent(Base):
         """    
         return str(self.lot)
 
-    def to_sub_dict(self) -> dict:
+    def to_sub_dict(self, extraction_kit:KitType=None) -> dict:
         """
         dictionary containing values necessary for gui
 
         Returns:
             dict: gui friendly dictionary
         """        
+        if extraction_kit != None:
+            try:
+                reagent_role = list(set(self.type).intersection(extraction_kit.reagent_types))[0]
+            except:
+                reagent_role = self.type[0]
+        else:
+            reagent_role = self.type[0]
         try:
-            type = self.type.name.replace("_", " ").title()
+            rtype = reagent_role.name.replace("_", " ").title()
         except AttributeError:
-            type = "Unknown"
+            rtype = "Unknown"
         try:
-            place_holder = self.expiry + self.type.eol_ext
+            place_holder = self.expiry + reagent_role.eol_ext
             # logger.debug(f"EOL_ext for {self.lot} -- {self.expiry} + {self.type.eol_ext} = {place_holder}")
         except TypeError as e:
             place_holder = date.today()
@@ -213,14 +222,14 @@ class Reagent(Base):
             place_holder = date.today()
             logger.debug(f"We got an attribute error setting {self.lot} expiry: {e}. Setting to today for testing")
         return {
-            "type": type,
+            "type": rtype,
             "lot": self.lot,
             "expiry": place_holder.strftime("%Y-%m-%d")
         }
     
     def to_reagent_dict(self) -> dict:
         return {
-            "type": self.type.name,
+            "type": type,
             "lot": self.lot,
             "expiry": self.expiry.strftime("%Y-%m-%d")
         }
@@ -280,3 +289,6 @@ class SubmissionTypeKitTypeAssociation(Base):
         self.mutable_cost_column = 0.00
         self.mutable_cost_sample = 0.00
         self.constant_cost = 0.00
+
+    def __repr__(self) -> str:
+        return f"<SubmissionTypeKitTypeAssociation({self.submission_type.name})"

src/submissions/backend/db/models/submissions.py

@@ -3,7 +3,7 @@ Models for the main submission types.
 '''
 import math
 from . import Base
-from sqlalchemy import Column, String, TIMESTAMP, INTEGER, ForeignKey, Table, JSON, FLOAT, BOOLEAN
+from sqlalchemy import Column, String, TIMESTAMP, INTEGER, ForeignKey, Table, JSON, FLOAT
 from sqlalchemy.orm import relationship, validates
 import logging
 import json
@@ -106,7 +106,7 @@ class BasicSubmission(Base):
             ext_info = None
             logger.debug(f"Json error in {self.rsl_plate_num}: {e}")
         try:
-            reagents = [item.to_sub_dict() for item in self.reagents]
+            reagents = [item.to_sub_dict(extraction_kit=self.extraction_kit) for item in self.reagents]
         except Exception as e:
             logger.error(f"We got an error retrieving reagents: {e}")
             reagents = None
@@ -252,6 +252,8 @@ class Wastewater(BasicSubmission):
     """    
     # samples = relationship("WWSample", back_populates="rsl_plate", uselist=True)
     pcr_info = Column(JSON)
+    ext_technician = Column(String(64))
+    pcr_technician = Column(String(64))
     # ww_sample_id = Column(String, ForeignKey("_ww_samples.id", ondelete="SET NULL", name="fk_WW_sample_id"))
     __mapper_args__ = {"polymorphic_identity": "Wastewater", "polymorphic_load": "inline"}
 
@@ -267,6 +269,7 @@ class Wastewater(BasicSubmission):
             output['pcr_info'] = json.loads(self.pcr_info)
         except TypeError as e:
             pass
+        output['Technician'] = f"Enr: {self.technician}, Ext: {self.ext_technician}, PCR: {self.pcr_technician}"
         return output
     
 class WastewaterArtic(BasicSubmission):
@@ -460,10 +463,10 @@ class WastewaterSample(BasicSample):
         except AttributeError as e:
             check = False
         if check:
-            logger.debug(f"Using well info in name.")
+            # logger.debug(f"Using well info in name.")
             sample['name'] = f"{self.submitter_id}\n\t- ct N1: {'{:.2f}'.format(self.assoc.ct_n1)} ({self.assoc.n1_status})\n\t- ct N2: {'{:.2f}'.format(self.assoc.ct_n2)} ({self.assoc.n2_status})"
-        else:
+        # else:
-            logger.error(f"Couldn't get the pcr info")
+            # logger.error(f"Couldn't get the pcr info")
         return sample
     
     def to_hitpick(self, submission_rsl:str) -> dict|None:
@@ -511,7 +514,7 @@ class BacterialCultureSample(BasicSample):
     # rsl_plate_id = Column(INTEGER, ForeignKey("_submissions.id", ondelete="SET NULL", name="fk_BCS_sample_id")) #: id of parent plate
     # rsl_plate = relationship("BacterialCulture", back_populates="samples") #: relationship to parent plate
 
-    __mapper_args__ = {"polymorphic_identity": "bacterial_culture_sample", "polymorphic_load": "inline"}
+    __mapper_args__ = {"polymorphic_identity": "Bacterial Culture Sample", "polymorphic_load": "inline"}
 
     # def to_string(self) -> str:
     #     """
@@ -543,10 +546,10 @@ class SubmissionSampleAssociation(Base):
     DOC: https://docs.sqlalchemy.org/en/14/orm/extensions/associationproxy.html
     """    
     __tablename__ = "_submission_sample"
-    sample_id = Column(INTEGER, ForeignKey("_samples.id"), primary_key=True)
+    sample_id = Column(INTEGER, ForeignKey("_samples.id"), nullable=False)
     submission_id = Column(INTEGER, ForeignKey("_submissions.id"), primary_key=True)
-    row = Column(INTEGER)
+    row = Column(INTEGER, primary_key=True)
-    column = Column(INTEGER)
+    column = Column(INTEGER, primary_key=True)
 
     submission = relationship(BasicSubmission, back_populates="submission_sample_associations")
 
@@ -569,6 +572,9 @@ class SubmissionSampleAssociation(Base):
         self.row = row
         self.column = column
 
+    def __repr__(self) -> str:
+        return f"<SubmissionSampleAssociation({self.submission.rsl_plate_num} & {self.sample.submitter_id})"
+
 class WastewaterAssociation(SubmissionSampleAssociation):
 
     ct_n1 = Column(FLOAT(2)) #: AKA ct for N1

src/submissions/backend/excel/parser.py

@@ -3,7 +3,7 @@ contains parser object for pulling values from client generated submission sheet
 '''
 from getpass import getuser
 import pprint
-from typing import List, Tuple
+from typing import List
 import pandas as pd
 from pathlib import Path
 from backend.db import lookup_ww_sample_by_ww_sample_num, lookup_sample_by_submitter_id, get_reagents_in_extkit, lookup_kittype_by_name, lookup_submissiontype_by_name, models
@@ -12,11 +12,11 @@ import logging
 from collections import OrderedDict
 import re
 import numpy as np
-from datetime import date, datetime
+from datetime import date
 from dateutil.parser import parse, ParserError
 import uuid
 # from submissions.backend.db.functions import 
-from tools import check_not_nan, RSLNamer, massage_common_reagents, convert_nans_to_nones, Settings
+from tools import check_not_nan, RSLNamer, convert_nans_to_nones, Settings
 from frontend.custom_widgets.pop_ups import SubmissionTypeSelector, KitSelector
 
 logger = logging.getLogger(f"submissions.{__name__}")
@@ -56,6 +56,7 @@ class SheetParser(object):
         self.parse_reagents()
         self.import_reagent_validation_check()
         self.parse_samples()
+        # self.sub['sample_count'] = len(self.sub['samples'])
         
 
     def type_decider(self) -> str:
@@ -448,10 +449,10 @@ class InfoParser(object):
         logger.debug(f"Looking up submission type: {submission_type['value']}")
         submission_type = lookup_submissiontype_by_name(ctx=self.ctx, type_name=submission_type['value'])
         info_map = submission_type.info_map
-        try:
+        # try:
-            del info_map['samples']
+        #     del info_map['samples']
-        except KeyError:
+        # except KeyError:
-            pass
+        #     pass
         return info_map
 
     def parse_info(self) -> dict:
@@ -472,14 +473,20 @@ class InfoParser(object):
                 value = df.iat[relevant[item]['row']-1, relevant[item]['column']-1]
                 logger.debug(f"Setting {item} on {sheet} to {value}")
                 if check_not_nan(value):
-                    try:
+                    if value != "None":
-                        dicto[item] = dict(value=value, parsed=True)
+                        try:
-                    except (KeyError, IndexError):
+                            dicto[item] = dict(value=value, parsed=True)
-                        continue
+                        except (KeyError, IndexError):
+                            continue
+                    else:
+                        try:
+                            dicto[item] = dict(value=value, parsed=False)
+                        except (KeyError, IndexError):
+                            continue
                 else:
                     dicto[item] = dict(value=convert_nans_to_nones(value), parsed=False)
-        if "submitter_plate_num" not in dicto.keys():
+        # if "submitter_plate_num" not in dicto.keys():
-            dicto['submitter_plate_num'] = dict(value=None, parsed=False)
+        #     dicto['submitter_plate_num'] = dict(value=None, parsed=False)
         return dicto
                 
 class ReagentParser(object):
@@ -554,6 +561,7 @@ class SampleParser(object):
     def fetch_sample_info_map(self, submission_type:dict) -> dict:
         logger.debug(f"Looking up submission type: {submission_type}")
         submission_type = lookup_submissiontype_by_name(ctx=self.ctx, type_name=submission_type)
+        logger.debug(f"info_map: {pprint.pformat(submission_type.info_map)}")
         sample_info_map = submission_type.info_map['samples']
         return sample_info_map
 
@@ -620,7 +628,13 @@ class SampleParser(object):
     def parse_samples(self) -> List[dict]:
         result = None
         new_samples = []
-        for sample in self.samples:
+        for ii, sample in enumerate(self.samples):
+            # logger.debug(f"\n\n{new_samples}\n\n")
+            try:
+                if sample['submitter_id'] in [check_sample['sample'].submitter_id for check_sample in new_samples]:
+                    sample['submitter_id'] = f"{sample['submitter_id']}-{ii}"
+            except KeyError as e:
+                logger.error(f"Sample obj: {sample}, error: {e}")
             translated_dict = {}
             for k, v in sample.items():
                 match v:
@@ -647,11 +661,13 @@ class SampleParser(object):
         instance = lookup_sample_by_submitter_id(ctx=self.ctx, submitter_id=input_dict['submitter_id'])
         if instance == None:
             instance = database_obj()
-        for k,v in input_dict.items():
+            for k,v in input_dict.items():
-            try:
+                try:
-                setattr(instance, k, v)
+                    setattr(instance, k, v)
-            except Exception as e:
+                except Exception as e:
-                logger.error(f"Failed to set {k} due to {type(e).__name__}: {e}")
+                    logger.error(f"Failed to set {k} due to {type(e).__name__}: {e}")
+        else:
+            logger.debug(f"Sample already exists, will run update.")
         return dict(sample=instance, row=input_dict['row'], column=input_dict['column'])
 
 

src/submissions/backend/pydant/__init__.py

@@ -1,5 +1,5 @@
 import uuid
-from pydantic import BaseModel, field_validator, model_validator, Extra
+from pydantic import BaseModel, field_validator, Extra
 from datetime import date, datetime
 from dateutil.parser import parse
 from dateutil.parser._parser import ParserError
@@ -9,7 +9,6 @@ from pathlib import Path
 import re
 import logging
 from tools import check_not_nan, convert_nans_to_nones, Settings
-import numpy as np
 from backend.db.functions import lookup_submission_by_rsl_num
 
 
@@ -46,7 +45,11 @@ class PydReagent(BaseModel):
         # else:
         #     return value
         if value != None:
+            if isinstance(value, int):
+                return datetime.fromordinal(datetime(1900, 1, 1).toordinal() + value - 2).date()
             return convert_nans_to_nones(str(value))
+        if value == None:
+            value = date.today()
         return value
     
     @field_validator("name", mode="before")
@@ -85,7 +88,7 @@ class PydSubmission(BaseModel, extra=Extra.allow):
     @classmethod
     def enforce_with_uuid(cls, value):
         logger.debug(f"submitter plate id: {value}")
-        if value['value'] == None:
+        if value['value'] == None or value['value'] == "None":
             return dict(value=uuid.uuid4().hex.upper(), parsed=False)
         else:
             return value

src/submissions/frontend/custom_widgets/misc.py

@@ -89,7 +89,6 @@ class AddReagentForm(QDialog):
         self.name_input.clear()
         self.name_input.addItems(item for item in lookup_all_reagent_names_by_role(ctx=self.ctx, role_name=self.type_input.currentText().replace(" ", "_").lower()))
 
-
 class ReportDatePicker(QDialog):
     """
     custom dialog to ask for report start/stop dates
@@ -118,7 +117,6 @@ class ReportDatePicker(QDialog):
         self.layout.addWidget(self.buttonBox)
         self.setLayout(self.layout)
 
-
 class KitAdder(QWidget):
     """
     dialog to get information to add kit
@@ -195,7 +193,7 @@ class KitAdder(QWidget):
             yml_type['password'] = info['password']
         except KeyError:
             pass
-        used = info['used_for'].replace(" ", "_").lower()
+        used = info['used_for']
         yml_type[used] = {}
         yml_type[used]['kits'] = {}
         yml_type[used]['kits'][info['kit_name']] = {}
@@ -210,7 +208,6 @@ class KitAdder(QWidget):
         msg.exec()
         self.__init__(self.ctx)
 
-
 class ReagentTypeForm(QWidget):
     """
     custom widget to add information about a new reagenttype
@@ -234,7 +231,6 @@ class ReagentTypeForm(QWidget):
         eol.setMinimum(0)
         grid.addWidget(eol, 0,3)
 
-
 class ControlsDatePicker(QWidget):
     """
     custom widget to pick start and end dates for controls graphs
@@ -259,7 +255,6 @@ class ControlsDatePicker(QWidget):
     def sizeHint(self) -> QSize:
         return QSize(80,20)  
 
-
 class ImportReagent(QComboBox):
 
     def __init__(self, ctx:dict, reagent:PydReagent):

src/submissions/frontend/main_window_functions.py

@@ -26,13 +26,13 @@ from backend.db.functions import (
     construct_submission_info, lookup_reagent, store_submission, lookup_submissions_by_date_range, 
     create_kit_from_yaml, create_org_from_yaml, get_control_subtypes, get_all_controls_by_type,
     lookup_all_submissions_by_type, get_all_controls, lookup_submission_by_rsl_num, update_ww_sample,
-    check_kit_integrity, get_reagents_in_extkit
+    check_kit_integrity
 )
 from backend.excel.parser import SheetParser, PCRParser
 from backend.excel.reports import make_report_html, make_report_xlsx, convert_data_list_to_df
 from backend.pydant import PydReagent
 from tools import check_not_nan
-from .custom_widgets.pop_ups import AlertPop, KitSelector, QuestionAsker
+from .custom_widgets.pop_ups import AlertPop, QuestionAsker
 from .custom_widgets import ReportDatePicker
 from .custom_widgets.misc import ImportReagent, ParsedQLabel
 from .visualizations.control_charts import create_charts, construct_html
@@ -92,7 +92,7 @@ def import_submission_function(obj:QMainWindow) -> Tuple[QMainWindow, dict|None]
     for item in obj.table_widget.formlayout.parentWidget().findChildren(QWidget):
         item.setParent(None)
     # Get list of fields from pydantic model.
-    fields = list(pyd.model_fields.keys())
+    fields = list(pyd.model_fields.keys()) + list(pyd.model_extra.keys())
     fields.remove('filepath')
     logger.debug(f"pydantic fields: {fields}")
     for field in fields:
@@ -175,7 +175,7 @@ def import_submission_function(obj:QMainWindow) -> Tuple[QMainWindow, dict|None]
                     else:
                         # try:
                         #     reg_label = QLabel(f"MISSING Lot: {reagent['value'].type}")
-                        obj.missing_reagents.append(reagent['value'].type)
+                        obj.missing_reagents.append(reagent['value'])
                         continue
                         # except AttributeError:
                         #     continue
@@ -273,10 +273,10 @@ def kit_integrity_completion_function(obj:QMainWindow) -> Tuple[QMainWindow, dic
         # obj.missing_reagents = kit_integrity['missing']
     # for item in kit_integrity['missing']:
     if len(obj.missing_reagents) > 0:
-        result = dict(message=f"The submission you are importing is missing some reagents expected by the kit.\n\nIt looks like you are missing: {[item.upper() for item in obj.missing_reagents]}\n\nAlternatively, you may have set the wrong extraction kit.\n\nThe program will populate lists using existing reagents.\n\nPlease make sure you check the lots carefully!", status="Warning")
+        result = dict(message=f"The submission you are importing is missing some reagents expected by the kit.\n\nIt looks like you are missing: {[item.type.upper() for item in obj.missing_reagents]}\n\nAlternatively, you may have set the wrong extraction kit.\n\nThe program will populate lists using existing reagents.\n\nPlease make sure you check the lots carefully!", status="Warning")
     for item in obj.missing_reagents:
-        obj.table_widget.formlayout.addWidget(ParsedQLabel({'parsed':False}, item, title=False))
+        obj.table_widget.formlayout.addWidget(ParsedQLabel({'parsed':False}, item.type, title=False))
-        reagent = dict(type=item, lot=None, exp=None, name=None)
+        reagent = dict(type=item.type, lot=None, exp=date.today(), name=None)
         add_widget = ImportReagent(ctx=obj.ctx, reagent=PydReagent(**reagent))#item=item)
         obj.table_widget.formlayout.addWidget(add_widget)
     submit_btn = QPushButton("Submit")
@@ -316,8 +316,12 @@ def submit_new_sample_function(obj:QMainWindow) -> Tuple[QMainWindow, dict]:
             r_lot = reagents[reagent]
             dlg = QuestionAsker(title=f"Add {r_lot}?", message=f"Couldn't find reagent type {reagent.strip('Lot')}: {r_lot} in the database.\n\nWould you like to add it?")
             if dlg.exec():
-                logger.debug(f"Looking through {obj.reagents} for reagent {reagent}")
+                logger.debug(f"Looking through {pprint.pformat(obj.reagents)} for reagent {reagent}")
-                picked_reagent = [item for item in obj.reagents if item.type == reagent][0]
+                try:
+                    picked_reagent = [item for item in obj.reagents if item.type == reagent][0]
+                except IndexError:
+                    logger.error(f"Couldn't find {reagent} in obj.reagents. Checking missing reagents {pprint.pformat(obj.missing_reagents)}")
+                    picked_reagent = [item for item in obj.missing_reagents if item.type == reagent][0]
                 logger.debug(f"checking reagent: {reagent} in obj.reagents. Result: {picked_reagent}")
                 expiry_date = picked_reagent.exp
                 wanted_reagent = obj.add_reagent(reagent_lot=r_lot, reagent_type=reagent.replace("lot_", ""), expiry=expiry_date, name=picked_reagent.name)
@@ -369,14 +373,15 @@ def submit_new_sample_function(obj:QMainWindow) -> Tuple[QMainWindow, dict]:
     for item in obj.table_widget.formlayout.parentWidget().findChildren(QWidget):
         item.setParent(None)
     logger.debug(f"All attributes of obj: {pprint.pformat(obj.__dict__)}")
-    if len(obj.missing_reagents) > 0:
+    if len(obj.missing_reagents + obj.missing_info) > 0:
         logger.debug(f"We have blank reagents in the excel sheet.\n\tLet's try to fill them in.") 
         extraction_kit = lookup_kittype_by_name(obj.ctx, name=obj.ext_kit)
         logger.debug(f"We have the extraction kit: {extraction_kit.name}")
-        logger.debug(f"Extraction kit map:\n\n{extraction_kit.used_for[obj.current_submission_type.replace('_', ' ')]}")
+        
         # TODO replace below with function in KitType object. Update Kittype associations.
         # excel_map = extraction_kit.used_for[obj.current_submission_type.replace('_', ' ')]
         excel_map = extraction_kit.construct_xl_map_for_use(obj.current_submission_type)
+        logger.debug(f"Extraction kit map:\n\n{pprint.pformat(excel_map)}")
         # excel_map.update(extraction_kit.used_for[obj.current_submission_type.replace('_', ' ').title()])
         input_reagents = [item.to_reagent_dict() for item in parsed_reagents]
         autofill_excel(obj=obj, xl_map=excel_map, reagents=input_reagents, missing_reagents=obj.missing_reagents, info=info, missing_info=obj.missing_info)
@@ -863,41 +868,47 @@ def autofill_excel(obj:QMainWindow, xl_map:dict, reagents:List[dict], missing_re
     logger.debug(f"Here is the info dict coming in:\n{pprint.pformat(info)}")
     logger.debug(f"Here are the missing reagents:\n{missing_reagents}")
     logger.debug(f"Here are the missing info:\n{missing_info}")
+    logger.debug(f"Here is the xl map: {pprint.pformat(xl_map)}")
     # pare down the xl map to only the missing data.
-    relevant_map = {k:v for k,v in xl_map.items() if k in missing_reagents}
+    relevant_reagent_map = {k:v for k,v in xl_map.items() if k in [reagent.type for reagent in missing_reagents]}
     # pare down reagents to only what's missing
-    relevant_reagents = [item for item in reagents if item['type'] in missing_reagents]
+    relevant_reagents = [item for item in reagents if item['type'] in [reagent.type for reagent in missing_reagents]]
+    logger.debug(f"Here are the relevant reagents: {pprint.pformat(relevant_reagents)}")
     # hacky manipulation of submission type so it looks better.
     # info['submission_type'] = info['submission_type'].replace("_", " ").title()
     # pare down info to just what's missing
+    relevant_info_map = {k:v for k,v in xl_map['info'].items() if k in missing_info and k != 'samples'}
     relevant_info = {k:v for k,v in info.items() if k in missing_info}
     logger.debug(f"Here is the relevant info: {pprint.pformat(relevant_info)}")
     # construct new objects to put into excel sheets:
     new_reagents = []
+    logger.debug(f"Parsing from relevant reagent map: {pprint.pformat(relevant_reagent_map)}")
     for reagent in relevant_reagents:
         new_reagent = {}
         new_reagent['type'] = reagent['type']
-        new_reagent['lot'] = relevant_map[new_reagent['type']]['lot']
+        new_reagent['lot'] = relevant_reagent_map[new_reagent['type']]['lot']
         new_reagent['lot']['value'] = reagent['lot']
-        new_reagent['expiry'] = relevant_map[new_reagent['type']]['expiry']
+        new_reagent['expiry'] = relevant_reagent_map[new_reagent['type']]['expiry']
         new_reagent['expiry']['value'] = reagent['expiry']
-        new_reagent['sheet'] = relevant_map[new_reagent['type']]['sheet']
+        new_reagent['sheet'] = relevant_reagent_map[new_reagent['type']]['sheet']
         # name is only present for Bacterial Culture
         try:
-            new_reagent['name'] = relevant_map[new_reagent['type']]['name']
+            new_reagent['name'] = relevant_reagent_map[new_reagent['type']]['name']
             new_reagent['name']['value'] = reagent['type']
         except:
             pass
         new_reagents.append(new_reagent)
     # construct new info objects to put into excel sheets
     new_info = []
+    logger.debug(f"Parsing from relevant info map: {pprint.pformat(relevant_info_map)}")
     for item in relevant_info:
         new_item = {}
         new_item['type'] = item
-        new_item['location'] = relevant_map[item]
+        new_item['location'] = relevant_info_map[item]
         new_item['value'] = relevant_info[item]
         new_info.append(new_item)
     logger.debug(f"New reagents: {new_reagents}")
+    logger.debug(f"New info: {new_info}")
     # open the workbook using openpyxl
     workbook = load_workbook(obj.xl)
     # get list of sheet names

src/submissions/templates/submission_details.html

@@ -7,28 +7,16 @@
     <body>
         <h2><u>Submission Details for {{ sub['Plate Number'] }}</u></h2>&nbsp;&nbsp;&nbsp;<img align='right' height="30px" width="120px" src="data:image/jpeg;base64,{{ sub['barcode'] | safe }}">
         <p>{% for key, value in sub.items() if key not in excluded %}
-            <!-- {% if loop.index == 1 %} -->
+            &nbsp;&nbsp;&nbsp;&nbsp;<b>{{ key }}: </b>{% if key=='Cost' %} {{ "${:,.2f}".format(value) }}{% else %}{{ value }}{% endif %}<br>
-                <!-- &nbsp;&nbsp;&nbsp;<b>{{ key }}:</b> {% if key=='Cost' %}{{ "${:,.2f}".format(value) }}{% else %}{{ value }}{% endif %}<br> -->
-            <!-- {% else %} -->
-                &nbsp;&nbsp;&nbsp;&nbsp;<b>{{ key }}: </b>{% if key=='Cost' %} {{ "${:,.2f}".format(value) }}{% else %}{{ value }}{% endif %}<br>
-            <!-- {% endif %} -->
         {% endfor %}</p>
         <h3><u>Reagents:</u></h3>
         <p>{% for item in sub['reagents'] %}
-            <!-- {% if loop.index == 1%} -->
+            &nbsp;&nbsp;&nbsp;&nbsp;<b>{{ item['type'] }}</b>: {{ item['lot'] }} (EXP: {{ item['expiry'] }})<br>
-                <!-- &nbsp;&nbsp;&nbsp;<b>{{ item['type'] }}:</b> {{ item['lot'] }} (EXP: {{ item['expiry'] }})<br> -->
-            <!-- {% else %} -->
-                &nbsp;&nbsp;&nbsp;&nbsp;<b>{{ item['type'] }}</b>: {{ item['lot'] }} (EXP: {{ item['expiry'] }})<br>
-            <!-- {% endif %} -->
         {% endfor %}</p>
         {% if sub['samples'] %}
         <h3><u>Samples:</u></h3>
         <p>{% for item in sub['samples'] %}
-            <!-- {% if loop.index == 1 %} -->
+            &nbsp;&nbsp;&nbsp;&nbsp;<b>{{ item['well'] }}:</b> {{ item['name']|replace('\n\t', '<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;') }}<br>
-                <!-- &nbsp;&nbsp;&nbsp;<b>{{ item['well'] }}:</b> {{ item['name']|replace('\n\t', '<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;') }}<br> -->
-            <!-- {% else %} -->
-                &nbsp;&nbsp;&nbsp;&nbsp;<b>{{ item['well'] }}:</b> {{ item['name']|replace('\n\t', '<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;') }}<br>
-            <!-- {% endif %} -->
         {% endfor %}</p>
         {% endif %}
         {% if sub['controls'] %}
@@ -38,11 +26,7 @@
             {% if item['kraken'] %}
             <p>&nbsp;&nbsp;&nbsp;{{ item['name'] }} Top 5 Kraken Results:</p>
             <p>{% for genera in item['kraken'] %}
-                <!-- {% if loop.index == 1 %} -->
-                <!-- &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;{{ genera['name'] }}: {{ genera['kraken_count'] }} ({{ genera['kraken_percent'] }})<br> -->
-                <!-- {% else %} -->
                 &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;{{ genera['name'] }}: {{ genera['kraken_count'] }} ({{ genera['kraken_percent'] }})<br>
-                <!-- {% endif %} -->
             {% endfor %}</p>
             {% endif %}
         {% endfor %}
@@ -51,15 +35,11 @@
         {% for entry in sub['ext_info'] %}
         <h3><u>Extraction Status:</u></h3>
         <p>{% for key, value in entry.items() %}
-            <!-- {% if loop.index == 1 %} -->
+            {% if "column" in key %}
-                <!-- &nbsp;&nbsp;&nbsp;<b>{{ key|replace('_', ' ')|title() }}:</b> {{ value }}<br> -->
+                &nbsp;&nbsp;&nbsp;&nbsp;<b>{{ key|replace('_', ' ')|title() }}:</b> {{ value }}uL<br>
-            <!-- {% else %} -->
+            {% else %}
-                {% if "column" in key %}
+                &nbsp;&nbsp;&nbsp;&nbsp;<b>{{ key|replace('_', ' ')|title() }}:</b> {{ value }}<br>
-                    &nbsp;&nbsp;&nbsp;&nbsp;<b>{{ key|replace('_', ' ')|title() }}:</b> {{ value }}uL<br>
+            {% endif %}
-                {% else %}
-                    &nbsp;&nbsp;&nbsp;&nbsp;<b>{{ key|replace('_', ' ')|title() }}:</b> {{ value }}<br>
-                {% endif %}
-            <!-- {% endif %} -->
         {% endfor %}</p>
         {% endfor %}
         {% endif %}
@@ -71,26 +51,18 @@
             <h3><u>qPCR Status:</u></h3>
         {% endif %}
         <p>{% for key, value in entry.items() if key != 'imported_by'%}
-            <!-- {% if loop.index == 1 %} -->
+            {% if "column" in key %}
-                <!-- &nbsp;&nbsp;&nbsp;<b>{{ key|replace('_', ' ')|title() }}:</b> {{ value }}<br> -->
+                &nbsp;&nbsp;&nbsp;&nbsp;<b>{{ key|replace('_', ' ')|title() }}:</b> {{ value }}uL<br>
-            <!-- {% else %} -->
+            {% else %}
-                {% if "column" in key %}
+                &nbsp;&nbsp;&nbsp;&nbsp;<b>{{ key|replace('_', ' ')|title() }}:</b> {{ value }}<br>
-                    &nbsp;&nbsp;&nbsp;&nbsp;<b>{{ key|replace('_', ' ')|title() }}:</b> {{ value }}uL<br>
+            {% endif %}
-                {% else %}
-                    &nbsp;&nbsp;&nbsp;&nbsp;<b>{{ key|replace('_', ' ')|title() }}:</b> {{ value }}<br>
-                {% endif %}
-            <!-- {% endif %} -->
         {% endfor %}</p>
         {% endfor %}
         {% endif %}
         {% if sub['comments'] %}
         <h3><u>Comments:</u></h3>
         <p>{% for entry in sub['comments'] %}
-            <!-- {% if loop.index == 1 %} -->
+            &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<b>{{ entry['name'] }}:</b><br> {{ entry['text'] }}<br>- {{ entry['time'] }}<br>
-                <!-- &nbsp;&nbsp;&nbsp;<b>{{ entry['name'] }}:</b><br> {{ entry['text'] }}<br>- {{ entry['time'] }}<br> -->
-            <!-- {% else %} -->
-                &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<b>{{ entry['name'] }}:</b><br> {{ entry['text'] }}<br>- {{ entry['time'] }}<br>
-            <!-- {% endif %} -->
         {% endfor %}</p>
         {% endif %}
         {% if sub['platemap'] %}

src/submissions/tools/__init__.py

@@ -53,6 +53,8 @@ def check_not_nan(cell_contents) -> bool:
         cell_contents = cell_contents.lower()
     except (TypeError, AttributeError):
         pass
+    if cell_contents == "nat":
+        cell_contents = np.nan
     if cell_contents == 'nan':
         cell_contents = np.nan
     if cell_contents == None:
@@ -80,6 +82,7 @@ def convert_nans_to_nones(input_str) -> str|None:
     Returns:
         str: _description_
     """    
+    # logger.debug(f"Input value of: {input_str}")
     if check_not_nan(input_str):
         return input_str
     return None