Pending large code cleanup

src/submissions/backend/db/models/__init__.py

@@ -50,7 +50,7 @@ class BaseClass(Base):
         return ctx.backup_path
     
     def save(self):
-        logger.debug(f"Saving {self}")
+        # logger.debug(f"Saving {self}")
         try:
             self.__database_session__.add(self)
             self.__database_session__.commit()

src/submissions/backend/db/models/controls.py

@@ -78,20 +78,20 @@ class Control(BaseClass):
     # __tablename__ = '_control_samples'
     
     id = Column(INTEGER, primary_key=True) #: primary key
-    parent_id = Column(String, ForeignKey("_control_types.id", name="fk_control_parent_id")) #: primary key of control type
+    parent_id = Column(String, ForeignKey("_controltype.id", name="fk_control_parent_id")) #: primary key of control type
     controltype = relationship("ControlType", back_populates="instances", foreign_keys=[parent_id]) #: reference to parent control type
     name = Column(String(255), unique=True) #: Sample ID
     submitted_date = Column(TIMESTAMP) #: Date submitted to Robotics
     contains = Column(JSON) #: unstructured hashes in contains.tsv for each organism
     matches = Column(JSON) #: unstructured hashes in matches.tsv for each organism
     kraken = Column(JSON) #: unstructured output from kraken_report
-    submission_id = Column(INTEGER, ForeignKey("_submissions.id")) #: parent submission id
+    submission_id = Column(INTEGER, ForeignKey("_basicsubmission.id")) #: parent submission id
     submission = relationship("BacterialCulture", back_populates="controls", foreign_keys=[submission_id]) #: parent submission
     refseq_version = Column(String(16)) #: version of refseq used in fastq parsing
     kraken2_version = Column(String(16)) #: version of kraken2 used in fastq parsing
     kraken2_db_version = Column(String(32)) #: folder name of kraken2 db
     sample = relationship("BacterialCultureSample", back_populates="control")
-    sample_id = Column(INTEGER, ForeignKey("_samples.id", ondelete="SET NULL", name="cont_BCS_id"))
+    sample_id = Column(INTEGER, ForeignKey("_basicsample.id", ondelete="SET NULL", name="cont_BCS_id"))
 
     def __repr__(self) -> str:
         return f"<Control({self.name})>"

src/submissions/backend/db/models/kits.py

@@ -18,8 +18,8 @@ logger = logging.getLogger(f'submissions.{__name__}')
 reagenttypes_reagents = Table(
                                 "_reagenttypes_reagents", 
                                 Base.metadata, 
-                                Column("reagent_id", INTEGER, ForeignKey("_reagents.id")), 
-                                Column("reagenttype_id", INTEGER, ForeignKey("_reagent_types.id")),
+                                Column("reagent_id", INTEGER, ForeignKey("_reagent.id")), 
+                                Column("reagenttype_id", INTEGER, ForeignKey("_reagenttype.id")),
                                 extend_existing = True
                                 )
 
@@ -27,7 +27,7 @@ equipmentroles_equipment = Table(
     "_equipmentroles_equipment",
     Base.metadata,
     Column("equipment_id", INTEGER, ForeignKey("_equipment.id")),
-    Column("equipmentroles_id", INTEGER, ForeignKey("_equipment_roles.id")),
+    Column("equipmentroles_id", INTEGER, ForeignKey("_equipmentrole.id")),
     extend_existing=True
 )
 
@@ -43,7 +43,7 @@ equipmentroles_processes = Table(
     "_equipmentroles_processes",
     Base.metadata,
     Column("process_id", INTEGER, ForeignKey("_process.id")),
-    Column("equipmentrole_id", INTEGER, ForeignKey("_equipment_roles.id")),
+    Column("equipmentrole_id", INTEGER, ForeignKey("_equipmentrole.id")),
     extend_existing=True
 )
 
@@ -51,7 +51,7 @@ submissiontypes_processes = Table(
     "_submissiontypes_processes",
     Base.metadata,
     Column("process_id", INTEGER, ForeignKey("_process.id")),
-    Column("equipmentroles_id", INTEGER, ForeignKey("_submission_types.id")),
+    Column("equipmentroles_id", INTEGER, ForeignKey("_submissiontype.id")),
     extend_existing=True
 )
 
@@ -59,7 +59,7 @@ kittypes_processes = Table(
     "_kittypes_processes",
     Base.metadata,
     Column("process_id", INTEGER, ForeignKey("_process.id")),
-    Column("kit_id", INTEGER, ForeignKey("_kits.id")),
+    Column("kit_id", INTEGER, ForeignKey("_kittype.id")),
     extend_existing=True
 )
 
@@ -304,7 +304,7 @@ class Reagent(BaseClass):
 
     id = Column(INTEGER, primary_key=True) #: primary key
     type = relationship("ReagentType", back_populates="instances", secondary=reagenttypes_reagents) #: joined parent reagent type
-    type_id = Column(INTEGER, ForeignKey("_reagent_types.id", ondelete='SET NULL', name="fk_reagent_type_id")) #: id of parent reagent type
+    type_id = Column(INTEGER, ForeignKey("_reagenttype.id", ondelete='SET NULL', name="fk_reagent_type_id")) #: id of parent reagent type
     name = Column(String(64)) #: reagent name
     lot = Column(String(64)) #: lot number of reagent
     expiry = Column(TIMESTAMP) #: expiry date - extended by eol_ext of parent programmatically
@@ -442,9 +442,9 @@ class Discount(BaseClass):
 
     id = Column(INTEGER, primary_key=True) #: primary key
     kit = relationship("KitType") #: joined parent reagent type
-    kit_id = Column(INTEGER, ForeignKey("_kits.id", ondelete='SET NULL', name="fk_kit_type_id")) #: id of joined kit
+    kit_id = Column(INTEGER, ForeignKey("_kittype.id", ondelete='SET NULL', name="fk_kit_type_id")) #: id of joined kit
     client = relationship("Organization") #: joined client lab
-    client_id = Column(INTEGER, ForeignKey("_organizations.id", ondelete='SET NULL', name="fk_org_id")) #: id of joined client
+    client_id = Column(INTEGER, ForeignKey("_organization.id", ondelete='SET NULL', name="fk_org_id")) #: id of joined client
     name = Column(String(128)) #: Short description 
     amount = Column(FLOAT(2)) #: Dollar amount of discount
 
@@ -625,8 +625,9 @@ class SubmissionType(BaseClass):
         """
         Adds this instances to the database and commits.
         """        
-        self.__database_session__.add(self)
-        self.__database_session__.commit()
+        # self.__database_session__.add(self)
+        # self.__database_session__.commit()
+        super().save()
   
 class SubmissionTypeKitTypeAssociation(BaseClass):
     """
@@ -634,8 +635,8 @@ class SubmissionTypeKitTypeAssociation(BaseClass):
     """    
     # __tablename__ = "_submissiontypes_kittypes"
 
-    submission_types_id = Column(INTEGER, ForeignKey("_submission_types.id"), primary_key=True) #: id of joined submission type
-    kits_id = Column(INTEGER, ForeignKey("_kits.id"), primary_key=True) #: id of joined kit
+    submission_types_id = Column(INTEGER, ForeignKey("_submissiontype.id"), primary_key=True) #: id of joined submission type
+    kits_id = Column(INTEGER, ForeignKey("_kittype.id"), primary_key=True) #: id of joined kit
     mutable_cost_column = Column(FLOAT(2)) #: dollar amount per 96 well plate that can change with number of columns (reagents, tips, etc)
     mutable_cost_sample = Column(FLOAT(2)) #: dollar amount that can change with number of samples (reagents, tips, etc)
     constant_cost = Column(FLOAT(2)) #: dollar amount per plate that will remain constant (plates, man hours, etc)
@@ -707,9 +708,9 @@ class KitTypeReagentTypeAssociation(BaseClass):
     """    
     # __tablename__ = "_reagenttypes_kittypes"
 
-    reagent_types_id = Column(INTEGER, ForeignKey("_reagent_types.id"), primary_key=True) #: id of associated reagent type
-    kits_id = Column(INTEGER, ForeignKey("_kits.id"), primary_key=True) #: id of associated reagent type
-    submission_type_id = Column(INTEGER, ForeignKey("_submission_types.id"), primary_key=True)
+    reagent_types_id = Column(INTEGER, ForeignKey("_reagenttype.id"), primary_key=True) #: id of associated reagent type
+    kits_id = Column(INTEGER, ForeignKey("_kittype.id"), primary_key=True) #: id of associated reagent type
+    submission_type_id = Column(INTEGER, ForeignKey("_submissiontype.id"), primary_key=True)
     uses = Column(JSON) #: map to location on excel sheets of different submission types
     required = Column(INTEGER) #: whether the reagent type is required for the kit (Boolean 1 or 0)
     last_used = Column(String(32)) #: last used lot number of this type of reagent
@@ -810,8 +811,8 @@ class SubmissionReagentAssociation(BaseClass):
 
     # __tablename__ = "_reagents_submissions"
 
-    reagent_id = Column(INTEGER, ForeignKey("_reagents.id"), primary_key=True) #: id of associated sample
-    submission_id = Column(INTEGER, ForeignKey("_submissions.id"), primary_key=True)
+    reagent_id = Column(INTEGER, ForeignKey("_reagent.id"), primary_key=True) #: id of associated sample
+    submission_id = Column(INTEGER, ForeignKey("_basicsubmission.id"), primary_key=True)
     comments = Column(String(1024))
 
     submission = relationship("BasicSubmission", back_populates="submission_reagent_associations") #: associated submission
@@ -1060,7 +1061,7 @@ class SubmissionEquipmentAssociation(BaseClass):
     # __tablename__ = "_equipment_submissions"
 
     equipment_id = Column(INTEGER, ForeignKey("_equipment.id"), primary_key=True) #: id of associated equipment
-    submission_id = Column(INTEGER, ForeignKey("_submissions.id"), primary_key=True) #: id of associated submission
+    submission_id = Column(INTEGER, ForeignKey("_basicsubmission.id"), primary_key=True) #: id of associated submission
     role = Column(String(64), primary_key=True) #: name of the role the equipment fills
     # process = Column(String(64)) #: name of the process run on this equipment
     process_id = Column(INTEGER, ForeignKey("_process.id",ondelete="SET NULL", name="SEA_Process_id"))
@@ -1090,8 +1091,8 @@ class SubmissionTypeEquipmentRoleAssociation(BaseClass):
 
     # __tablename__ = "_submissiontype_equipmentrole"
 
-    equipmentrole_id = Column(INTEGER, ForeignKey("_equipment_roles.id"), primary_key=True) #: id of associated equipment
-    submissiontype_id = Column(INTEGER, ForeignKey("_submission_types.id"), primary_key=True) #: id of associated submission
+    equipmentrole_id = Column(INTEGER, ForeignKey("_equipmentrole.id"), primary_key=True) #: id of associated equipment
+    submissiontype_id = Column(INTEGER, ForeignKey("_submissiontype.id"), primary_key=True) #: id of associated submission
     uses = Column(JSON) #: locations of equipment on the submission type excel sheet.
     static = Column(INTEGER, default=1) #: if 1 this piece of equipment will always be used, otherwise it will need to be selected from list?
 
@@ -1156,7 +1157,7 @@ class Process(BaseClass):
     
     @classmethod
     @setup_lookup
-    def query(cls, name:str|None, limit:int=0):
+    def query(cls, name:str|None=None, limit:int=0):
         query = cls.__database_session__.query(cls)
         match name:
             case str():

src/submissions/backend/db/models/organizations.py

@@ -15,8 +15,8 @@ logger = logging.getLogger(f"submissions.{__name__}")
 orgs_contacts = Table(
                         "_orgs_contacts", 
                         Base.metadata, 
-                        Column("org_id", INTEGER, ForeignKey("_organizations.id")), 
-                        Column("contact_id", INTEGER, ForeignKey("_contacts.id")), 
+                        Column("org_id", INTEGER, ForeignKey("_organization.id")), 
+                        Column("contact_id", INTEGER, ForeignKey("_contact.id")), 
                         # __table_args__ = {'extend_existing': True} 
                         extend_existing = True
                     )

src/submissions/backend/db/models/submissions.py

@@ -3,7 +3,8 @@ Models for the main submission types.
 '''
 from __future__ import annotations
 from getpass import getuser
-import math, json, logging, uuid, tempfile, re, yaml
+import math, json, logging, uuid, tempfile, re, yaml, zipfile
+import sys
 from operator import attrgetter
 from pprint import pformat
 from . import Reagent, SubmissionType, KitType, Organization
@@ -13,9 +14,10 @@ from json.decoder import JSONDecodeError
 from sqlalchemy.ext.associationproxy import association_proxy
 import pandas as pd
 from openpyxl import Workbook
-from . import BaseClass, Equipment
+from openpyxl.worksheet.worksheet import Worksheet
+from . import BaseClass
 from tools import check_not_nan, row_map, query_return, setup_lookup, jinja_template_loading
-from datetime import datetime, date, time
+from datetime import datetime, date
 from typing import List, Any
 from dateutil.parser import parse
 from dateutil.parser._parser import ParserError
@@ -37,17 +39,16 @@ class BasicSubmission(BaseClass):
     submitter_plate_num = Column(String(127), unique=True) #: The number given to the submission by the submitting lab
     submitted_date = Column(TIMESTAMP) #: Date submission received
     submitting_lab = relationship("Organization", back_populates="submissions") #: client org
-    submitting_lab_id = Column(INTEGER, ForeignKey("_organizations.id", ondelete="SET NULL", name="fk_BS_sublab_id")) #: client lab id from _organizations
+    submitting_lab_id = Column(INTEGER, ForeignKey("_organization.id", ondelete="SET NULL", name="fk_BS_sublab_id")) #: client lab id from _organizations
     sample_count = Column(INTEGER) #: Number of samples in the submission
     extraction_kit = relationship("KitType", back_populates="submissions") #: The extraction kit used
-    extraction_kit_id = Column(INTEGER, ForeignKey("_kits.id", ondelete="SET NULL", name="fk_BS_extkit_id")) #: id of joined extraction kit
-    submission_type_name = Column(String, ForeignKey("_submission_types.name", ondelete="SET NULL", name="fk_BS_subtype_name")) #: name of joined submission type
+    extraction_kit_id = Column(INTEGER, ForeignKey("_kittype.id", ondelete="SET NULL", name="fk_BS_extkit_id")) #: id of joined extraction kit
+    submission_type_name = Column(String, ForeignKey("_submissiontype.name", ondelete="SET NULL", name="fk_BS_subtype_name")) #: name of joined submission type
     technician = Column(String(64)) #: initials of processing tech(s)
     # Move this into custom types?
     # reagents = relationship("Reagent", back_populates="submissions", secondary=reagents_submissions) #: relationship to reagents
-    reagents_id = Column(String, ForeignKey("_reagents.id", ondelete="SET NULL", name="fk_BS_reagents_id")) #: id of used reagents
+    reagents_id = Column(String, ForeignKey("_reagent.id", ondelete="SET NULL", name="fk_BS_reagents_id")) #: id of used reagents
     extraction_info = Column(JSON) #: unstructured output from the extraction table logger.
-    pcr_info = Column(JSON) #: unstructured output from pcr table logger or user(Artic)
     run_cost = Column(FLOAT(2)) #: total cost of running the plate. Set from constant and mutable kit costs at time of creation.
     uploaded_by = Column(String(32)) #: user name of person who submitted the submission to the database.
     comment = Column(JSON) #: user notes
@@ -132,19 +133,21 @@ class BasicSubmission(BaseClass):
             logger.error(f"Json error in {self.rsl_plate_num}: {e}")
         # Updated 2023-09 to use the extraction kit to pull reagents.
         if full_data:
+            logger.debug(f"Attempting reagents.")
             try:
                 reagents = [item.to_sub_dict(extraction_kit=self.extraction_kit) for item in self.submission_reagent_associations]
             except Exception as e:
                 logger.error(f"We got an error retrieving reagents: {e}")
                 reagents = None
             # samples = [item.sample.to_sub_dict(submission_rsl=self.rsl_plate_num) for item in self.submission_sample_associations]
+            logger.debug(f"Running samples.")
             samples = self.adjust_to_dict_samples(backup=backup)
             try:
                 equipment = [item.to_sub_dict() for item in self.submission_equipment_associations]
                 if len(equipment) == 0:
                     equipment = None
             except Exception as e:
-                logger.error(f"Error setting equipment: {self.equipment}")
+                logger.error(f"Error setting equipment: {e}")
                 equipment = None
         else:
             reagents = None
@@ -155,7 +158,6 @@ class BasicSubmission(BaseClass):
         except Exception as e:
             logger.error(f"Error setting comment: {self.comment}")
             comments = None
-        
         output = {
             "id": self.id,
             "Plate Number": self.rsl_plate_num,
@@ -440,6 +442,7 @@ class BasicSubmission(BaseClass):
     def filename_template(cls) -> str:
         """
         Constructs template for filename of this class.
+        Note: This is meant to be used with the dictionary constructed in self.to_dict(). Keys need to have spaces removed
 
         Returns:
             str: filename template in jinja friendly format.
@@ -462,6 +465,20 @@ class BasicSubmission(BaseClass):
                 logger.warning(f"Couldn't drop '{item}' column from submissionsheet df.")
         return df
 
+    @classmethod
+    def custom_sample_autofill_row(cls, sample, worksheet:Worksheet) -> int:
+        """
+        _summary_
+
+        Args:
+            sample (_type_): _description_
+            worksheet (Workbook): _description_
+
+        Returns:
+            int: _description_
+        """      
+        return None  
+
     def set_attribute(self, key:str, value):
         """
         Performs custom attribute setting based on values.
@@ -547,7 +564,7 @@ class BasicSubmission(BaseClass):
         """        
         from backend.validators import PydSubmission, PydSample, PydReagent, PydEquipment
         dicto = self.to_dict(full_data=True, backup=backup)
-        logger.debug(f"Backup dictionary: {pformat(dicto)}")
+        # logger.debug(f"Backup dictionary: {pformat(dicto)}")
         # dicto['filepath'] = Path(tempfile.TemporaryFile().name)
         new_dict = {}
         for key, value in dicto.items():
@@ -572,7 +589,7 @@ class BasicSubmission(BaseClass):
                     # new_dict[key.lower().replace(" ", "_")]['value'] = value
                     # new_dict[key.lower().replace(" ", "_")]['missing'] = True
         new_dict['filepath'] = Path(tempfile.TemporaryFile().name)
-        logger.debug(f"Dictionary coming into PydSubmission: {pformat(new_dict)}")
+        # logger.debug(f"Dictionary coming into PydSubmission: {pformat(new_dict)}")
         # sys.exit()
         return PydSubmission(**new_dict)
 
@@ -797,22 +814,15 @@ class BasicSubmission(BaseClass):
             # logger.debug(f"Save result: {result}")
 
     def add_equipment(self, obj):
-        # submission_type = submission.submission_type_name
         from frontend.widgets.equipment_usage import EquipmentUsage
-        dlg = EquipmentUsage(parent=obj, submission_type=self.submission_type_name, submission=self)
+        dlg = EquipmentUsage(parent=obj, submission=self)
         if dlg.exec():
             equipment = dlg.parse_form()
             logger.debug(f"We've got equipment: {equipment}")
             for equip in equipment:
-                # e = Equipment.query(name=equip.name)
-                # assoc = SubmissionEquipmentAssociation(submission=submission, equipment=e)
-                # process = Process.query(name=equip.processes)
-                # assoc.process = process
-                # assoc.role = equip.role
+                logger.debug(f"Processing: {equip}")
                 _, assoc = equip.toSQL(submission=self)
-                # submission.submission_equipment_associations.append(assoc)
                 logger.debug(f"Appending SubmissionEquipmentAssociation: {assoc}")
-                # submission.save()
                 assoc.save()
         else:
             pass
@@ -825,12 +835,14 @@ class BasicSubmission(BaseClass):
             fname (Path): Filename of xlsx file.
         """        
         logger.debug("Hello from backup.")
+        pyd = self.to_pydantic(backup=True)
         if fname == None:
             from frontend.widgets.functions import select_save_file
-            from backend.validators import RSLNamer
-            abbreviation = self.get_abbreviation()
-            file_data = dict(rsl_plate_num=self.rsl_plate_num, submission_type=self.submission_type_name, submitted_date=self.submitted_date, abbreviation=abbreviation)
-            fname = select_save_file(default_name=RSLNamer.construct_new_plate_name(data=file_data), extension="xlsx", obj=obj)
+            fname = select_save_file(default_name=pyd.construct_filename(), extension="xlsx", obj=obj)
+        logger.debug(fname.name)
+        if fname.name == "":
+            logger.debug(f"export cancelled.")
+            return
         if full_backup:
             backup = self.to_dict(full_data=True)
             try:
@@ -838,7 +850,6 @@ class BasicSubmission(BaseClass):
                     yaml.dump(backup, f)
             except KeyError as e:
                 logger.error(f"Problem saving yml backup file: {e}")
-        pyd = self.to_pydantic(backup=True)
         wb = pyd.autofill_excel()
         wb = pyd.autofill_samples(wb)
         wb = pyd.autofill_equipment(wb)
@@ -856,14 +867,14 @@ class BacterialCulture(BasicSubmission):
                            polymorphic_load="inline", 
                            inherit_condition=(id == BasicSubmission.id))
 
-    def to_dict(self, full_data:bool=False) -> dict:
+    def to_dict(self, full_data:bool=False, backup:bool=False) -> dict:
         """
         Extends parent class method to add controls to dict
 
         Returns:
             dict: dictionary used in submissions summary
         """        
-        output = super().to_dict(full_data=full_data)
+        output = super().to_dict(full_data=full_data, backup=backup)
         if full_data:
             output['controls'] = [item.to_sub_dict() for item in self.controls]
         return output
@@ -996,6 +1007,22 @@ class BacterialCulture(BasicSubmission):
         input_dict['submitted_date']['missing'] = True
         return input_dict
 
+    @classmethod
+    def custom_sample_autofill_row(cls, sample, worksheet: Worksheet) -> int:
+        logger.debug(f"Checking {sample.well}")
+        logger.debug(f"here's the worksheet: {worksheet}")
+        row = super().custom_sample_autofill_row(sample, worksheet)
+        df = pd.DataFrame(list(worksheet.values))
+        # logger.debug(f"Here's the dataframe: {df}")
+        idx = df[df[0]==sample.well]
+        if idx.empty:
+            new = f"{sample.well[0]}{sample.well[1:].zfill(2)}"
+            logger.debug(f"Checking: {new}")
+            idx = df[df[0]==new]
+        logger.debug(f"Here is the row: {idx}")
+        row = idx.index.to_list()[0]
+        return row + 1
+
 class Wastewater(BasicSubmission):
     """
     derivative submission type from BasicSubmission
@@ -1003,11 +1030,13 @@ class Wastewater(BasicSubmission):
     id = Column(INTEGER, ForeignKey('_basicsubmission.id'), primary_key=True)
     ext_technician = Column(String(64))
     pcr_technician = Column(String(64))
+    pcr_info = Column(JSON) #: unstructured output from pcr table logger or user(Artic)
+
     __mapper_args__ = __mapper_args__ = dict(polymorphic_identity="Wastewater", 
                            polymorphic_load="inline", 
                            inherit_condition=(id == BasicSubmission.id))
 
-    def to_dict(self, full_data:bool=False) -> dict:
+    def to_dict(self, full_data:bool=False, backup:bool=False) -> dict:
         """
         Extends parent class method to add controls to dict
 
@@ -1020,6 +1049,7 @@ class Wastewater(BasicSubmission):
         except TypeError as e:
             pass
         output['Technician'] = f"Enr: {self.technician}, Ext: {self.ext_technician}, PCR: {self.pcr_technician}"
+        
         return output
     
     @classmethod
@@ -1144,6 +1174,18 @@ class Wastewater(BasicSubmission):
     def adjust_autofill_samples(cls, samples: List[Any]) -> List[Any]:
         samples = super().adjust_autofill_samples(samples)
         return [item for item in samples if not item.submitter_id.startswith("EN")]
+    
+    @classmethod
+    def custom_sample_autofill_row(cls, sample, worksheet: Worksheet) -> int:
+        logger.debug(f"Checking {sample.well}")
+        logger.debug(f"here's the worksheet: {worksheet}")
+        row = super().custom_sample_autofill_row(sample, worksheet)
+        df = pd.DataFrame(list(worksheet.values))
+        logger.debug(f"Here's the dataframe: {df}")
+        idx = df[df[1]==sample.sample_location]
+        logger.debug(f"Here is the row: {idx}")
+        row = idx.index.to_list()[0]
+        return row + 1
 
 class WastewaterArtic(BasicSubmission):
     """
@@ -1155,6 +1197,9 @@ class WastewaterArtic(BasicSubmission):
                            inherit_condition=(id == BasicSubmission.id))
     artic_technician = Column(String(64))
     dna_core_submission_number = Column(String(64))
+    pcr_info = Column(JSON) #: unstructured output from pcr table logger or user(Artic)
+    gel_image = Column(String(64))
+    gel_info = Column(JSON)
 
     def calculate_base_cost(self):
         """
@@ -1381,10 +1426,19 @@ class WastewaterArtic(BasicSubmission):
     
     def gel_box(self, obj):
         from frontend.widgets.gel_checker import GelBox
-        dlg = GelBox(parent=obj)
+        from frontend.widgets import select_open_file
+        fname = select_open_file(obj=obj, file_extension="jpg")
+        dlg = GelBox(parent=obj, img_path=fname)
         if dlg.exec():
-            output = dlg.parse_form()
-        print(output)
+            img_path, output = dlg.parse_form()
+            self.gel_image = img_path.name
+            self.gel_info = output
+            with zipfile.ZipFile(self.__directory_path__.joinpath("submission_imgs.zip"), 'a') as zipf:
+                # Add a file located at the source_path to the destination within the zip
+                # file. It will overwrite existing files if the names collide, but it
+                # will give a warning
+                zipf.write(img_path, self.gel_image)
+            self.save()
 
 # Sample Classes
 
@@ -1439,7 +1493,10 @@ class BasicSample(BaseClass):
             return value
         
     def __repr__(self) -> str:
-        return f"<{self.sample_type.replace('_', ' ').title().replace(' ', '')}({self.submitter_id})>"
+        try:
+            return f"<{self.sample_type.replace('_', ' ').title().replace(' ', '')}({self.submitter_id})>"
+        except AttributeError:
+            return f"<Sample({self.submitter_id})"
     
     def to_sub_dict(self, submission_rsl:str) -> dict:
         """
@@ -1448,6 +1505,7 @@ class BasicSample(BaseClass):
         Returns:
             dict: well location and name (sample id, organism) NOTE: keys must sync with WWSample to_sub_dict above
         """
+        # logger.debug(f"Converting {self} to dict.")
         sample = {}
         sample['submitter_id'] = self.submitter_id
         sample['sample_type'] = self.sample_type
@@ -1642,6 +1700,9 @@ class WastewaterSample(BasicSample):
         """
         sample = super().to_sub_dict(submission_rsl=submission_rsl)
         sample['ww_processing_num'] = self.ww_processing_num
+        sample['sample_location'] = self.sample_location
+        sample['received_date'] = self.received_date
+        sample['collection_date'] = self.collection_date
         return sample
         
     @classmethod
@@ -1721,9 +1782,10 @@ class SubmissionSampleAssociation(BaseClass):
     """    
 
     # __tablename__ = "_submission_sample"
-
-    sample_id = Column(INTEGER, ForeignKey("_samples.id"), nullable=False) #: id of associated sample
-    submission_id = Column(INTEGER, ForeignKey("_submissions.id"), primary_key=True) #: id of associated submission
+    
+    id = Column(INTEGER, unique=True, nullable=False)
+    sample_id = Column(INTEGER, ForeignKey("_basicsample.id"), nullable=False) #: id of associated sample
+    submission_id = Column(INTEGER, ForeignKey("_basicsubmission.id"), primary_key=True) #: id of associated submission
     row = Column(INTEGER, primary_key=True) #: row on the 96 well plate
     column = Column(INTEGER, primary_key=True) #: column on the 96 well plate
 
@@ -1743,14 +1805,23 @@ class SubmissionSampleAssociation(BaseClass):
         "with_polymorphic": "*",
     }
 
-    def __init__(self, submission:BasicSubmission=None, sample:BasicSample=None, row:int=1, column:int=1):
+    def __init__(self, submission:BasicSubmission=None, sample:BasicSample=None, row:int=1, column:int=1, id:int|None=None):
         self.submission = submission
         self.sample = sample
         self.row = row
         self.column = column
+        if id != None:
+            self.id = id
+        else:
+            self.id = self.__class__.autoincrement_id()
+        logger.debug(f"Using id: {self.id}")
 
     def __repr__(self) -> str:
-        return f"<SubmissionSampleAssociation({self.submission.rsl_plate_num} & {self.sample.submitter_id})"
+        try:
+            return f"<{self.__class__.__name__}({self.submission.rsl_plate_num} & {self.sample.submitter_id})"
+        except AttributeError as e:
+            logger.error(f"Unable to construct __repr__ due to: {e}")
+            return super().__repr__()
     
     def to_sub_dict(self) -> dict:
         """
@@ -1760,6 +1831,7 @@ class SubmissionSampleAssociation(BaseClass):
             dict: Updated dictionary with row, column and well updated
         """        
         # Get sample info
+        # logger.debug(f"Running {self.__repr__()}")
         sample = self.sample.to_sub_dict(submission_rsl=self.submission)
         # sample = {}
         sample['name'] = self.sample.submitter_id
@@ -1787,6 +1859,7 @@ class SubmissionSampleAssociation(BaseClass):
         # Since there is no PCR, negliable result is necessary.
         # assoc = [item for item in self.sample_submission_associations if item.submission.rsl_plate_num==submission_rsl][0]
         sample = self.to_sub_dict()
+        logger.debug(f"Sample dict to hitpick: {sample}")
         env = jinja_template_loading()
         template = env.get_template("tooltip.html")
         tooltip_text = template.render(fields=sample)
@@ -1801,6 +1874,14 @@ class SubmissionSampleAssociation(BaseClass):
         sample.update(dict(name=self.sample.submitter_id[:10], tooltip=tooltip_text))
         return sample
 
+    @classmethod
+    def autoincrement_id(cls):
+        try:
+            return max([item.id for item in cls.query()]) + 1
+        except ValueError as e:
+            logger.error(f"Problem incrementing id: {e}")
+            return 1
+
     @classmethod
     def find_polymorphic_subclass(cls, polymorphic_identity:str|None=None) -> SubmissionSampleAssociation:   
         """
@@ -1890,6 +1971,7 @@ class SubmissionSampleAssociation(BaseClass):
                 association_type:str="Basic Association", 
                 submission:BasicSubmission|str|None=None,
                 sample:BasicSample|str|None=None,
+                id:int|None=None,
                 **kwargs) -> SubmissionSampleAssociation:
         """
         Queries for an association, if none exists creates a new one.
@@ -1931,7 +2013,7 @@ class SubmissionSampleAssociation(BaseClass):
             instance = None
         if instance == None:
             used_cls = cls.find_polymorphic_subclass(polymorphic_identity=association_type)
-            instance = used_cls(submission=submission, sample=sample, **kwargs)
+            instance = used_cls(submission=submission, sample=sample, id=id, **kwargs)
         return instance
 
     def delete(self):
@@ -1941,8 +2023,8 @@ class WastewaterAssociation(SubmissionSampleAssociation):
     """
     Derivative custom Wastewater/Submission Association... fancy.
     """    
-    sample_id = Column(INTEGER, ForeignKey('_submissionsampleassociation.sample_id'), primary_key=True)
-    submission_id = Column(INTEGER, ForeignKey('_submissionsampleassociation.submission_id'), primary_key=True)
+    # sample_id = Column(INTEGER, ForeignKey('_submissionsampleassociation.sample_id'), primary_key=True)
+    id = Column(INTEGER, ForeignKey("_submissionsampleassociation.id"), primary_key=True)
     ct_n1 = Column(FLOAT(2)) #: AKA ct for N1
     ct_n2 = Column(FLOAT(2)) #: AKA ct for N2
     n1_status = Column(String(32)) #: positive or negative for N1
@@ -1952,7 +2034,11 @@ class WastewaterAssociation(SubmissionSampleAssociation):
     # __mapper_args__ = {"polymorphic_identity": "Wastewater Association", "polymorphic_load": "inline"}
     __mapper_args__ = dict(polymorphic_identity="Wastewater Association", 
                            polymorphic_load="inline", 
-                           inherit_condition=(sample_id == SubmissionSampleAssociation.sample_id))
+                        #    inherit_condition=(submission_id==SubmissionSampleAssociation.submission_id and 
+                        #                       row==SubmissionSampleAssociation.row and 
+                        #                       column==SubmissionSampleAssociation.column))
+                        inherit_condition=(id==SubmissionSampleAssociation.id))
+                        # inherit_foreign_keys=(sample_id == SubmissionSampleAssociation.sample_id, submission_id == SubmissionSampleAssociation.submission_id))
 
     def to_sub_dict(self) -> dict:
         sample = super().to_sub_dict()
@@ -1970,3 +2056,12 @@ class WastewaterAssociation(SubmissionSampleAssociation):
         except (TypeError, AttributeError) as e:
             logger.error(f"Couldn't set tooltip for {self.sample.rsl_number}. Looks like there isn't PCR data.")
         return sample
+
+    @classmethod
+    def autoincrement_id(cls):
+        try:
+            parent = [base for base in cls.__bases__ if base.__name__=="SubmissionSampleAssociation"][0]
+            return max([item.id for item in parent.query()]) + 1
+        except ValueError as e:
+            logger.error(f"Problem incrementing id: {e}")
+            return 1       

src/submissions/backend/validators/__init__.py

@@ -3,6 +3,7 @@ from pathlib import Path
 from openpyxl import load_workbook
 from backend.db.models import BasicSubmission, SubmissionType
 from datetime import date
+from tools import jinja_template_loading
 
 logger = logging.getLogger(f"submissions.{__name__}")
 
@@ -126,9 +127,20 @@ class RSLNamer(object):
                 today = parse(today.group())
             except AttributeError:
                 today = datetime.now()
-        previous = BasicSubmission.query(start_date=today, end_date=today, submission_type=data['submission_type'])
-        plate_number = len(previous) + 1
+        if "rsl_plate_num" in data.keys():
+            plate_number = data['rsl_plate_num'].split("-")[-1][0]
+        else:
+            previous = BasicSubmission.query(start_date=today, end_date=today, submission_type=data['submission_type'])
+            plate_number = len(previous) + 1
         return f"RSL-{data['abbreviation']}-{today.year}{str(today.month).zfill(2)}{str(today.day).zfill(2)}-{plate_number}"
+    
+    @classmethod
+    def construct_export_name(cls, template, **kwargs):
+        logger.debug(f"Kwargs: {kwargs}")
+        logger.debug(f"Template: {template}")
+        environment = jinja_template_loading()
+        template = environment.from_string(template)
+        return template.render(**kwargs)
 
         
 from .pydant import *

src/submissions/backend/validators/pydant.py

@@ -8,7 +8,7 @@ from pydantic import BaseModel, field_validator, Field
 from datetime import date, datetime, timedelta
 from dateutil.parser import parse
 from dateutil.parser._parser import ParserError
-from typing import List, Any, Tuple
+from typing import List, Tuple
 from . import RSLNamer
 from pathlib import Path
 from tools import check_not_nan, convert_nans_to_nones, jinja_template_loading, Report, Result, row_map
@@ -156,8 +156,9 @@ class PydSample(BaseModel, extra='allow'):
     sample_type: str
     row: int|List[int]|None
     column: int|List[int]|None
+    assoc_id: int|List[int]|None = Field(default=None)
 
-    @field_validator("row", "column")
+    @field_validator("row", "column", "assoc_id")
     @classmethod
     def row_int_to_list(cls, value):
         if isinstance(value, int):
@@ -193,14 +194,14 @@ class PydSample(BaseModel, extra='allow'):
         out_associations = []
         if submission != None:
             assoc_type = self.sample_type.replace("Sample", "").strip()
-            for row, column in zip(self.row, self.column):
-                # logger.debug(f"Looking up association with identity: ({submission.submission_type_name} Association)")
+            for row, column, id in zip(self.row, self.column, self.assoc_id):
+                logger.debug(f"Looking up association with identity: ({submission.submission_type_name} Association)")
                 logger.debug(f"Looking up association with identity: ({assoc_type} Association)")
                 association = SubmissionSampleAssociation.query_or_create(association_type=f"{assoc_type} Association", 
-                                                                          submission=submission, 
-                                                                          sample=instance, 
-                                                                          row=row, column=column)
-                logger.debug(f"Using submission_sample_association: {association}")
+                                                                            submission=submission, 
+                                                                            sample=instance, 
+                                                                            row=row, column=column, id=id)
+                # logger.debug(f"Using submission_sample_association: {association}")
                 try:
                     instance.sample_submission_associations.append(association)
                     out_associations.append(association)
@@ -254,7 +255,7 @@ class PydEquipment(BaseModel, extra='ignore'):
             assoc.process = process
             assoc.role = self.role
             # equipment.equipment_submission_associations.append(assoc)
-            equipment.equipment_submission_associations.append(assoc)
+            # equipment.equipment_submission_associations.append(assoc)
         else:
             assoc = None
         return equipment, assoc
@@ -275,7 +276,7 @@ class PydSubmission(BaseModel, extra='allow'):
     comment: dict|None = Field(default=dict(value="", missing=True), validate_default=True)
     reagents: List[dict]|List[PydReagent] = []
     samples: List[PydSample]
-    equipment: List[PydEquipment]|None
+    equipment: List[PydEquipment]|None =[]
 
     @field_validator('equipment', mode='before')
     @classmethod
@@ -421,6 +422,16 @@ class PydSubmission(BaseModel, extra='allow'):
             value['value'] = values.data['submission_type']['value']
         return value
 
+    @field_validator("samples")
+    def assign_ids(cls, value, values):
+        starting_id = SubmissionSampleAssociation.autoincrement_id()
+        output = []
+        for iii, sample in enumerate(value, start=starting_id):
+            sample.assoc_id = [iii]
+            output.append(sample)
+        return output
+
+
     def handle_duplicate_samples(self):
         """
         Collapses multiple samples with same submitter id into one with lists for rows, columns.
@@ -428,14 +439,19 @@ class PydSubmission(BaseModel, extra='allow'):
         """        
         submitter_ids = list(set([sample.submitter_id for sample in self.samples]))
         output = []
-        for id in submitter_ids:
+        for iii, id in enumerate(submitter_ids, start=1):
             relevants = [item for item in self.samples if item.submitter_id==id]
             if len(relevants) <= 1:
                 output += relevants
             else:
                 rows = [item.row[0] for item in relevants]
                 columns = [item.column[0] for item in relevants]
+                ids = [item.assoc_id[0] for item in relevants]
+                # for jjj, rel in enumerate(relevants, start=1):
+                #     starting_id += jjj  
+                #     ids.append(starting_id)                  
                 dummy = relevants[0]
+                dummy.assoc_id = ids
                 dummy.row = rows
                 dummy.column = columns
                 output.append(dummy)
@@ -663,14 +679,17 @@ class PydSubmission(BaseModel, extra='allow'):
         logger.debug(f"Workbook sheets: {workbook.sheetnames}")
         worksheet = workbook[sample_info["lookup_table"]['sheet']]
         samples = sorted(self.samples, key=attrgetter('column', 'row'))
-        custom_sampler = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=self.submission_type).adjust_autofill_samples
-        samples = custom_sampler(samples=samples)
+        submission_obj = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=self.submission_type)
+        samples = submission_obj.adjust_autofill_samples(samples=samples)
         logger.debug(f"Samples: {pformat(samples)}")
         # Fail safe against multiple instances of the same sample
         for iii, sample in enumerate(samples, start=1):
-            row = sample_info['lookup_table']['start_row'] + iii
+            logger.debug(f"Sample: {sample}")
+            row = submission_obj.custom_sample_autofill_row(sample, worksheet=worksheet)
+            logger.debug(f"Writing to {row}")
+            if row == None:
+                row = sample_info['lookup_table']['start_row'] + iii
             fields = [field for field in list(sample.model_fields.keys()) + list(sample.model_extra.keys()) if field in sample_info['sample_columns'].keys()]
-            
             for field in fields:
                 column = sample_info['sample_columns'][field]
                 value = getattr(sample, field)