Various bug fixes for new forms.

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 202404.04
+
+- Storing of default values in db rather than hardcoded.
+
 ## 202404.03
 
 - Package updates.

TODO.md

@@ -1,4 +1,7 @@
-- [ ] Critical: Convert Json lits to dicts so I can have them update properly without using crashy Sqlalchemy-json
+- [ ] Put "Not applicable" reagents in to_dict() method.
+    - Currently in to_pydantic().
+- [x] Critical: Convert Json lits to dicts so I can have them update properly without using crashy Sqlalchemy-json
+    - Was actually not necessary.
 - [ ] Fix Parsed/Missing mix ups.
 - [x] Have sample parser check for controls and add to reagents?
 - [x] Update controls to NestedMutableJson
@@ -6,7 +9,7 @@
     - Possibly due to immutable JSON? But... it's worked before... Right?
     - Based on research, if a top-level JSON field is not changed, SQLalchemy will not detect changes.
     - Using sqlalchemy-json module seems to have helped.
-- [ ] Add Bead basher and Assit to DB.
+- [ ] Add Bead basher and Assist to DB.
 - [x] Artic not creating right plate name.
 - [ ] Merge BasicSubmission.find_subclasses and BasicSubmission.find_polymorphic_subclass
 - [x] Fix updating of Extraction Kit in submission form widget.

alembic.ini

@@ -55,9 +55,9 @@ version_path_separator = os  # Use os.pathsep. Default configuration used for ne
 # are written from script.py.mako
 # output_encoding = utf-8
 
-; sqlalchemy.url = sqlite:///L:\Robotics Laboratory Support\Submissions\submissions.db
+sqlalchemy.url = sqlite:///L:\Robotics Laboratory Support\Submissions\submissions.db
 ; sqlalchemy.url = sqlite:///C:\Users\lwark\Documents\Archives\Submissions_app_backups\DB_backups\submissions-demo.db
-sqlalchemy.url = sqlite:///C:\Users\lwark\Documents\python\submissions\mytests\test_assets\submissions-test.db
+; sqlalchemy.url = sqlite:///C:\Users\lwark\Documents\python\submissions\mytests\test_assets\submissions-test.db
 
 
 [post_write_hooks]

alembic/versions/6d2a357860ef_adding_cost_centre_storage_to_.py

@@ -0,0 +1,51 @@
+"""adding cost centre storage to basicsubmission
+
+Revision ID: 6d2a357860ef
+Revises: e6647bd661d9
+Create Date: 2024-04-24 13:01:14.923814
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '6d2a357860ef'
+down_revision = 'e6647bd661d9'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    # op.drop_table('_alembic_tmp__submissionsampleassociation')
+    with op.batch_alter_table('_basicsubmission', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('cost_centre', sa.String(length=64), nullable=True))
+
+    # with op.batch_alter_table('_submissionsampleassociation', schema=None) as batch_op:
+    #     batch_op.create_unique_constraint(None, ['id'])
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    # with op.batch_alter_table('_submissionsampleassociation', schema=None) as batch_op:
+    #     batch_op.drop_constraint(None, type_='unique')
+
+    with op.batch_alter_table('_basicsubmission', schema=None) as batch_op:
+        batch_op.drop_column('used_cost_centre')
+
+    op.create_table('_alembic_tmp__submissionsampleassociation',
+    sa.Column('sample_id', sa.INTEGER(), nullable=False),
+    sa.Column('submission_id', sa.INTEGER(), nullable=False),
+    sa.Column('row', sa.INTEGER(), nullable=False),
+    sa.Column('column', sa.INTEGER(), nullable=False),
+    sa.Column('base_sub_type', sa.VARCHAR(), nullable=True),
+    sa.Column('id', sa.INTEGER(), server_default=sa.text('1'), nullable=False),
+    sa.ForeignKeyConstraint(['sample_id'], ['_basicsample.id'], ),
+    sa.ForeignKeyConstraint(['submission_id'], ['_basicsubmission.id'], ),
+    sa.PrimaryKeyConstraint('submission_id', 'row', 'column'),
+    sa.UniqueConstraint('id', name='ssa_id')
+    )
+    # ### end Alembic commands ###

alembic/versions/e6647bd661d9_adding_default_info_to_submissiontype.py

@@ -0,0 +1,33 @@
+"""adding default info to submissiontype
+
+Revision ID: e6647bd661d9
+Revises: f18487b41f45
+Create Date: 2024-04-22 12:02:21.512781
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'e6647bd661d9'
+down_revision = 'f18487b41f45'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('_submissiontype', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('defaults', sa.JSON(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('_submissiontype', schema=None) as batch_op:
+        batch_op.drop_column('defaults')
+
+    
+    # ### end Alembic commands ###

src/submissions/backend/db/models/kits.py

@@ -374,11 +374,15 @@ class Reagent(BaseClass):
         except (TypeError, AttributeError) as e:
             place_holder = date.today()
             logger.debug(f"We got a type error setting {self.lot} expiry: {e}. setting to today for testing")
+        if self.expiry.year == 1970:
+            place_holder = "NA"
+        else:
+            place_holder = place_holder.strftime("%Y-%m-%d")
         return dict(
             name=self.name,
             type=rtype,
             lot=self.lot,
-            expiry=place_holder.strftime("%Y-%m-%d")
+            expiry=place_holder
         )
     
     def update_last_used(self, kit:KitType) -> Report:
@@ -410,6 +414,7 @@ class Reagent(BaseClass):
     @classmethod
     @setup_lookup
     def query(cls, 
+                id:int|None=None,
                 reagent_type:str|ReagentType|None=None,
                 lot_number:str|None=None,
                 name:str|None=None,
@@ -428,6 +433,12 @@ class Reagent(BaseClass):
             models.Reagent | List[models.Reagent]: reagent or list of reagents matching filter.
         """    
         query: Query = cls.__database_session__.query(cls)
+        match id:
+            case int():
+                query = query.filter(cls.id==id)
+                limit = 1
+            case _:
+                pass
         match reagent_type:
             case str():
                 # logger.debug(f"Looking up reagents by reagent type str: {reagent_type}")
@@ -535,6 +546,7 @@ class SubmissionType(BaseClass):
     id = Column(INTEGER, primary_key=True) #: primary key
     name = Column(String(128), unique=True) #: name of submission type
     info_map = Column(JSON) #: Where basic information is found in the excel workbook corresponding to this type.
+    defaults = Column(JSON) #: Basic information about this submission type
     instances = relationship("BasicSubmission", backref="submission_type") #: Concrete instances of this type.
     template_file = Column(BLOB) #: Blank form for this type stored as binary.
     processes = relationship("Process", back_populates="submission_types", secondary=submissiontypes_processes) #: Relation to equipment processes used for this type.
@@ -653,6 +665,10 @@ class SubmissionType(BaseClass):
                 raise TypeError(f"Type {type(equipment_role)} is not allowed")
         return list(set([item for items in relevant for item in items if item != None ]))
 
+    def get_submission_class(self):
+        from .submissions import BasicSubmission
+        return BasicSubmission.find_polymorphic_subclass(polymorphic_identity=self.name)
+
     @classmethod
     @setup_lookup
     def query(cls, 

src/submissions/backend/db/models/organizations.py

@@ -65,8 +65,8 @@ class Organization(BaseClass):
                 pass
         match name:
             case str():
-                # logger.debug(f"Looking up organization with name: {name}")
-                query = query.filter(cls.name==name)
+                # logger.debug(f"Looking up organization with name starting with: {name}")
+                query = query.filter(cls.name.startswith(name))
                 limit = 1
             case _:
                 pass

src/submissions/backend/db/models/submissions.py

@@ -3,12 +3,12 @@ Models for the main submission types.
 '''
 from __future__ import annotations
 from getpass import getuser
-import logging, uuid, tempfile, re, yaml, base64
+import logging, uuid, tempfile, re, yaml, base64, sys
 from zipfile import ZipFile
 from tempfile import TemporaryDirectory
-from reportlab.graphics.barcode import createBarcodeImageInMemory
-from reportlab.graphics.shapes import Drawing
-from reportlab.lib.units import mm
+# from reportlab.graphics.barcode import createBarcodeImageInMemory
+# from reportlab.graphics.shapes import Drawing
+# from reportlab.lib.units import mm
 from operator import attrgetter, itemgetter
 from pprint import pformat
 from . import BaseClass, Reagent, SubmissionType, KitType, Organization
@@ -18,9 +18,6 @@ from sqlalchemy import Column, String, TIMESTAMP, INTEGER, ForeignKey, JSON, FLO
 from sqlalchemy.orm import relationship, validates, Query
 from sqlalchemy.orm.attributes import flag_modified
 from sqlalchemy.ext.associationproxy import association_proxy
-# from sqlalchemy.ext.declarative import declared_attr
-# from sqlalchemy_json import NestedMutableJson
-# from sqlalchemy.ext.mutable import MutableDict
 from sqlalchemy.exc import OperationalError as AlcOperationalError, IntegrityError as AlcIntegrityError, StatementError
 from sqlite3 import OperationalError as SQLOperationalError, IntegrityError as SQLIntegrityError
 import pandas as pd
@@ -59,9 +56,10 @@ class BasicSubmission(BaseClass):
     reagents_id = Column(String, ForeignKey("_reagent.id", ondelete="SET NULL", name="fk_BS_reagents_id")) #: id of used reagents
     extraction_info = Column(JSON) #: unstructured output from the extraction table logger.
     run_cost = Column(FLOAT(2)) #: total cost of running the plate. Set from constant and mutable kit costs at time of creation.
-    uploaded_by = Column(String(32)) #: user name of person who submitted the submission to the database.
+    signed_by = Column(String(32)) #: user name of person who submitted the submission to the database.
     comment = Column(JSON) #: user notes
     submission_category = Column(String(64)) #: ["Research", "Diagnostic", "Surveillance", "Validation"], else defaults to submission_type_name
+    cost_centre = Column(String(64)) #: Permanent storage of used cost centre in case organization field changed in the future.
 
     submission_sample_associations = relationship(
         "SubmissionSampleAssociation",
@@ -103,12 +101,59 @@ class BasicSubmission(BaseClass):
         return f"{submission_type}Submission({self.rsl_plate_num})"
 
     @classmethod
-    def jsons(cls):
+    def jsons(cls) -> List[str]:
         output = [item.name for item in cls.__table__.columns if isinstance(item.type, JSON)]
         if issubclass(cls, BasicSubmission) and not cls.__name__ == "BasicSubmission":
             output += BasicSubmission.jsons()
         return output
 
+    @classmethod
+    def get_default_info(cls, *args):
+        # Create defaults for all submission_types
+        # print(args)
+        recover = ['filepath', 'samples', 'csv', 'comment', 'equipment']
+        dicto = dict(
+            details_ignore = ['excluded', 'reagents', 'samples',  
+                                'extraction_info', 'comment', 'barcode', 
+                                'platemap', 'export_map', 'equipment'],
+            form_recover = recover,
+            form_ignore = ['reagents', 'ctx', 'id', 'cost', 'extraction_info', 'signed_by'] + recover,
+            parser_ignore = ['samples', 'signed_by'] + cls.jsons(),
+            excel_ignore = []
+        )
+        # Grab subtype specific info.
+        st = cls.get_submission_type()
+        if st is None:
+            logger.error("No default info for BasicSubmission.")
+            return dicto
+        else:
+            dicto['submission_type'] = st.name
+            output = {}
+            for k,v in dicto.items():
+                if len(args) > 0 and k not in args:
+                    logger.debug(f"Don't want {k}")
+                    continue
+                else:
+                    output[k] = v
+            for k,v in st.defaults.items():
+                if len(args) > 0 and k not in args:
+                    logger.debug(f"Don't want {k}")
+                    continue
+                else:
+                    match v:
+                        case list():
+                            output[k] += v
+                        case _:
+                            output[k] = v
+        if len(args) == 1:
+            return output[args[0]]
+        return output
+
+    @classmethod
+    def get_submission_type(cls):
+        name = cls.__mapper_args__['polymorphic_identity']
+        return SubmissionType.query(name=name)
+
     def to_dict(self, full_data:bool=False, backup:bool=False, report:bool=False) -> dict:
         """
         Constructs dictionary used in submissions summary
@@ -168,6 +213,11 @@ class BasicSubmission(BaseClass):
             logger.debug(f"Attempting reagents.")
             try:
                 reagents = [item.to_sub_dict(extraction_kit=self.extraction_kit) for item in self.submission_reagent_associations]
+                for k in self.extraction_kit.construct_xl_map_for_use(self.submission_type):
+                        if k == 'info':
+                            continue
+                        if not any([item['type']==k for item in reagents]):
+                            reagents.append(dict(type=k, name="Not Applicable", lot="NA", expiry=date(year=1970, month=1, day=1), missing=True))
             except Exception as e:
                 logger.error(f"We got an error retrieving reagents: {e}")
                 reagents = None
@@ -181,10 +231,12 @@ class BasicSubmission(BaseClass):
             except Exception as e:
                 logger.error(f"Error setting equipment: {e}")
                 equipment = None
+            cost_centre = self.cost_centre
         else:
             reagents = None
             samples = None
             equipment = None
+            cost_centre = None
         # logger.debug("Getting comments")
         try:
             comments = self.comment
@@ -198,6 +250,8 @@ class BasicSubmission(BaseClass):
         output["extraction_info"] = ext_info
         output["comment"] = comments
         output["equipment"] = equipment
+        output["Cost Centre"] = cost_centre
+        output["Signed By"] = self.signed_by
         return output
     
     def calculate_column_count(self) -> int:
@@ -293,18 +347,18 @@ class BasicSubmission(BaseClass):
         """        
         return [item.role for item in self.submission_equipment_associations]
 
-    def make_plate_barcode(self, width:int=100, height:int=25) -> Drawing:
-        """
-        Creates a barcode image for this BasicSubmission.
+    # def make_plate_barcode(self, width:int=100, height:int=25) -> Drawing:
+    #     """
+    #     Creates a barcode image for this BasicSubmission.
 
-        Args:
-            width (int, optional): Width (pixels) of image. Defaults to 100.
-            height (int, optional): Height (pixels) of image. Defaults to 25.
+    #     Args:
+    #         width (int, optional): Width (pixels) of image. Defaults to 100.
+    #         height (int, optional): Height (pixels) of image. Defaults to 25.
 
-        Returns:
-            Drawing: image object
-        """    
-        return createBarcodeImageInMemory('Code128', value=self.rsl_plate_num, width=width*mm, height=height*mm, humanReadable=True, format="png")
+    #     Returns:
+    #         Drawing: image object
+    #     """    
+    #     return createBarcodeImageInMemory('Code128', value=self.rsl_plate_num, width=width*mm, height=height*mm, humanReadable=True, format="png")
 
     @classmethod
     def submissions_to_df(cls, submission_type:str|None=None, limit:int=0) -> pd.DataFrame:
@@ -384,11 +438,17 @@ class BasicSubmission(BaseClass):
             case item if item in self.jsons():
                 logger.debug(f"Setting JSON attribute.")
                 existing = self.__getattribute__(key)
+                if value == "" or value is None or value == 'null':
+                    logger.error(f"No value given, not setting.")
+                    return
                 if existing is None:
                     existing = []
                 if value in existing:
                     logger.warning("Value already exists. Preventing duplicate addition.")
                     return
+                else:
+                    if isinstance(value, list):
+                        existing += value
                     else:
                         existing.append(value)
                 self.__setattr__(key, existing)
@@ -634,7 +694,7 @@ class BasicSubmission(BaseClass):
         from backend.validators import RSLNamer
         logger.debug(f"instr coming into {cls}: {instr}")
         logger.debug(f"data coming into {cls}: {data}")
-        defaults = cls.get_default_info()
+        defaults = cls.get_default_info("abbreviation", "submission_type")
         data['abbreviation'] = defaults['abbreviation']
         if 'submission_type' not in data.keys() or data['submission_type'] in [None, ""]:
             data['submission_type'] = defaults['submission_type']
@@ -737,9 +797,7 @@ class BasicSubmission(BaseClass):
         Returns:
             Tuple(dict, Template): (Updated dictionary, Template to be rendered)
         """        
-        base_dict['excluded'] = ['excluded', 'reagents', 'samples', 'controls', 
-                                'extraction_info', 'pcr_info', 'comment', 
-                                'barcode', 'platemap', 'export_map', 'equipment']
+        base_dict['excluded'] = cls.get_default_info('details_ignore')
         env = jinja_template_loading()
         temp_name = f"{cls.__name__.lower()}_details.html"
         logger.debug(f"Returning template: {temp_name}")
@@ -1067,9 +1125,9 @@ class BacterialCulture(BasicSubmission):
             output['controls'] = [item.to_sub_dict() for item in self.controls]
         return output
     
-    @classmethod
-    def get_default_info(cls) -> dict:
-        return dict(abbreviation="BC", submission_type="Bacterial Culture")
+    # @classmethod
+    # def get_default_info(cls) -> dict:
+    #     return dict(abbreviation="BC", submission_type="Bacterial Culture")
 
     @classmethod
     def custom_platemap(cls, xl: pd.ExcelFile, plate_map: pd.DataFrame) -> pd.DataFrame:
@@ -1214,13 +1272,19 @@ class Wastewater(BasicSubmission):
             output['pcr_info'] = self.pcr_info
         except TypeError as e:
             pass
-        ext_tech = self.ext_technician or self.technician
-        pcr_tech = self.pcr_technician or self.technician
-        output['Technician'] = f"Enr: {self.technician}, Ext: {ext_tech}, PCR: {pcr_tech}"
+        if self.ext_technician is None or self.ext_technician == "None":
+            output['Ext Technician'] = self.technician
+        else:
+            output["Ext Technician"] = self.ext_technician
+        if self.pcr_technician is None or self.pcr_technician == "None":
+            output["PCR Technician"] = self.technician
+        else:
+            output['PCR Technician'] = self.pcr_technician
+        # output['Technician'] = self.technician}, Ext: {ext_tech}, PCR: {pcr_tech}"
         return output
     
-    @classmethod
-    def get_default_info(cls) -> dict:
+    # @classmethod
+    # def get_default_info(cls) -> dict:
         return dict(abbreviation="WW", submission_type="Wastewater")
 
     @classmethod
@@ -1334,36 +1398,6 @@ class Wastewater(BasicSubmission):
         from frontend.widgets import select_open_file
         fname = select_open_file(obj=obj, file_extension="xlsx")
         parser = PCRParser(filepath=fname)
-        # Check if PCR info already exists
-        # if hasattr(self, 'pcr_info') and self.pcr_info != None:
-        #     # existing = json.loads(sub.pcr_info)
-        #     existing = self.pcr_info
-        #     logger.debug(f"Found existing pcr info: {pformat(self.pcr_info)}")
-        # else:
-        #     existing = None
-        # if existing != None:
-        #     # update pcr_info
-        #     try:
-        #         logger.debug(f"Updating {type(existing)}:\n {pformat(existing)} with {type(parser.pcr)}:\n {pformat(parser.pcr)}")
-        #         # if json.dumps(parser.pcr) not in sub.pcr_info:
-        #         if parser.pcr not in self.pcr_info:
-        #             logger.debug(f"This is new pcr info, appending to existing")
-        #             existing.append(parser.pcr)
-        #         else:
-        #             logger.debug("This info already exists, skipping.")
-        #         # logger.debug(f"Setting {self.rsl_plate_num} PCR to:\n {pformat(existing)}")
-        #         # sub.pcr_info = json.dumps(existing)
-        #         self.pcr_info = existing
-        #     except TypeError:
-        #         logger.error(f"Error updating!")
-        #         # sub.pcr_info = json.dumps([parser.pcr])
-        #         self.pcr_info = [parser.pcr]
-        #     logger.debug(f"Final pcr info for {self.rsl_plate_num}:\n {pformat(self.pcr_info)}")
-        # else:
-        #     # sub.pcr_info = json.dumps([parser.pcr])
-        #     self.pcr_info = [parser.pcr]
-        # # logger.debug(f"Existing {type(self.pcr_info)}: {self.pcr_info}")
-        # # logger.debug(f"Inserting {type(parser.pcr)}: {parser.pcr}")
         self.set_attribute("pcr_info", parser.pcr)
         self.save(original=False)
         logger.debug(f"Got {len(parser.samples)} samples to update!")
@@ -1390,7 +1424,6 @@ class WastewaterArtic(BasicSubmission):
     gel_controls = Column(JSON) #: locations of controls on the gel
     source_plates = Column(JSON) #: wastewater plates that samples come from
 
-
     __mapper_args__ = dict(polymorphic_identity="Wastewater Artic", 
                            polymorphic_load="inline", 
                            inherit_condition=(id == BasicSubmission.id))
@@ -1411,9 +1444,9 @@ class WastewaterArtic(BasicSubmission):
         output['source_plates'] = self.source_plates
         return output
 
-    @classmethod
-    def get_default_info(cls) -> str:
-        return dict(abbreviation="AR", submission_type="Wastewater Artic")
+    # @classmethod
+    # def get_default_info(cls) -> str:
+    #     return dict(abbreviation="AR", submission_type="Wastewater Artic")
 
     @classmethod
     def parse_info(cls, input_dict:dict, xl:pd.ExcelFile|None=None) -> dict:
@@ -1429,12 +1462,16 @@ class WastewaterArtic(BasicSubmission):
         """        
         # from backend.validators import RSLNamer
         input_dict = super().parse_info(input_dict)
-        ws = load_workbook(xl.io, data_only=True)['Egel results']
-        data = [ws.cell(row=jj,column=ii) for ii in range(15,27) for jj in range(10,18)]
+        workbook = load_workbook(xl.io, data_only=True)
+        ws = workbook['Egel results']
+        data = [ws.cell(row=ii,column=jj) for jj in range(15,27) for ii in range(10,18)]
         data = [cell for cell in data if cell.value is not None and "NTC" in cell.value]
         input_dict['gel_controls'] = [dict(sample_id=cell.value, location=f"{row_map[cell.row-9]}{str(cell.column-14).zfill(2)}") for cell in data]
         # df = xl.parse("Egel results").iloc[7:16, 13:26]
         # df = df.set_index(df.columns[0])
+        ws = workbook['First Strand List']
+        data = [dict(plate=ws.cell(row=ii, column=3).value, starting_sample=ws.cell(row=ii, column=4).value) for ii in range(8,11)]
+        input_dict['source_plates'] = data
         return input_dict
 
     @classmethod
@@ -1500,34 +1537,38 @@ class WastewaterArtic(BasicSubmission):
         Returns:
             str: output name
         """       
+        logger.debug(f"input string raw: {input_str}")
         # Remove letters. 
-        processed = re.sub(r"[A-Z]", "", input_str)
+        processed = re.sub(r"[A-QS-Z]+\d*", "", input_str)
         # Remove trailing '-' if any
         processed = processed.strip("-")
+        logger.debug(f"Processed after stripping letters: {processed}")
         try:
             en_num = re.search(r"\-\d{1}$", processed).group()
             processed = rreplace(processed, en_num, "")
         except AttributeError:
             en_num = "1"
         en_num = en_num.strip("-")
-        # logger.debug(f"Processed after en-num: {processed}")
+        logger.debug(f"Processed after en-num: {processed}")
         try: 
-            plate_num = re.search(r"\-\d{1}$", processed).group()
+            plate_num = re.search(r"\-\d{1}R?\d?$", processed).group()
             processed = rreplace(processed, plate_num, "")
         except AttributeError:
             plate_num = "1"
         plate_num = plate_num.strip("-")
-        # logger.debug(f"Processed after plate-num: {processed}")
+        logger.debug(f"Processed after plate-num: {processed}")
         day = re.search(r"\d{2}$", processed).group()
         processed = rreplace(processed, day, "")
-        # logger.debug(f"Processed after day: {processed}")
+        logger.debug(f"Processed after day: {processed}")
         month = re.search(r"\d{2}$", processed).group()
         processed = rreplace(processed, month, "")
         processed = processed.replace("--", "")
-        # logger.debug(f"Processed after month: {processed}")
+        logger.debug(f"Processed after month: {processed}")
         year = re.search(r'^(?:\d{2})?\d{2}', processed).group()
         year = f"20{year}"
-        return f"EN{year}{month}{day}-{en_num}"
+        final_en_name = f"EN{year}{month}{day}-{en_num}"
+        logger.debug(f"Final EN name: {final_en_name}")
+        return final_en_name
 
     @classmethod
     def get_regex(cls) -> str:
@@ -1599,7 +1640,23 @@ class WastewaterArtic(BasicSubmission):
         #     for jjj, value in enumerate(plate, start=3):
         #         worksheet.cell(row=iii, column=jjj, value=value)
         logger.debug(f"Info:\n{pformat(info)}")
-        check = 'gel_info' in info.keys() and info['gel_info']['value'] != None
+        check = 'source_plates' in info.keys() and info['source_plates'] is not None
+        if check:
+            worksheet = input_excel['First Strand List']
+            start_row = 8
+            for iii, plate in enumerate(info['source_plates']['value']):
+                logger.debug(f"Plate: {plate}")
+                row = start_row + iii
+                try:
+                    worksheet.cell(row=row, column=3, value=plate['plate'])
+                except TypeError:
+                    pass
+                try:
+                    worksheet.cell(row=row, column=4, value=plate['starting_sample'])
+                except TypeError:
+                    pass
+            # sys.exit(f"Hardcoded stop: backend.models.submissions:1629")
+        check = 'gel_info' in info.keys() and info['gel_info']['value'] is not None
         if check:
             # logger.debug(f"Gel info check passed.")
             if info['gel_info'] != None:
@@ -1618,7 +1675,7 @@ class WastewaterArtic(BasicSubmission):
                         column = start_column + 2 + jjj
                         worksheet.cell(row=start_row, column=column, value=kj['name'])
                         worksheet.cell(row=row, column=column, value=kj['value'])
-        check = 'gel_image' in info.keys() and info['gel_image']['value'] != None
+        check = 'gel_image' in info.keys() and info['gel_image']['value'] is not None
         if check:
             if info['gel_image'] != None:
                 worksheet = input_excel['Egel results']

src/submissions/backend/excel/parser.py

@@ -62,9 +62,11 @@ class SheetParser(object):
         parser = InfoParser(xl=self.xl, submission_type=self.sub['submission_type']['value'])
         info = parser.parse_info() 
         self.info_map = parser.map
+        # exclude_from_info = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=self.sub['submission_type']).exclude_from_info_parser()
         for k,v in info.items():
             match k:
                 case "sample":
+                # case item if 
                     pass
                 case _:
                     self.sub[k] = v
@@ -97,9 +99,9 @@ class SheetParser(object):
         """
         Enforce that the parser has an extraction kit
         """    
-        from frontend.widgets.pop_ups import KitSelector
+        from frontend.widgets.pop_ups import ObjectSelector
         if not check_not_nan(self.sub['extraction_kit']['value']):
-            dlg = KitSelector(title="Kit Needed", message="At minimum a kit is needed. Please select one.")
+            dlg = ObjectSelector(title="Kit Needed", message="At minimum a kit is needed. Please select one.", obj_type=KitType)
             if dlg.exec():
                 self.sub['extraction_kit'] = dict(value=dlg.getValues(), missing=True)
             else:
@@ -133,7 +135,11 @@ class SheetParser(object):
         """       
         # logger.debug(f"Submission dictionary coming into 'to_pydantic':\n{pformat(self.sub)}")
         logger.debug(f"Equipment: {self.sub['equipment']}")
-        if len(self.sub['equipment']) == 0:
+        try:
+            check = len(self.sub['equipment']) == 0
+        except TypeError:
+            check = True
+        if check:
             self.sub['equipment'] = None
         psm = PydSubmission(filepath=self.filepath, **self.sub)
         return psm
@@ -142,11 +148,12 @@ class InfoParser(object):
 
     def __init__(self, xl:pd.ExcelFile, submission_type:str):
         logger.info(f"\n\Hello from InfoParser!\n\n")
-        self.map = self.fetch_submission_info_map(submission_type=submission_type)
+        self.submission_type = submission_type
+        self.map = self.fetch_submission_info_map()
         self.xl = xl
         logger.debug(f"Info map for InfoParser: {pformat(self.map)}")
         
-    def fetch_submission_info_map(self, submission_type:str|dict) -> dict:
+    def fetch_submission_info_map(self) -> dict:
         """
         Gets location of basic info from the submission_type object in the database.
 
@@ -156,10 +163,10 @@ class InfoParser(object):
         Returns:
             dict: Location map of all info for this submission type
         """        
-        if isinstance(submission_type, str):
-            submission_type = dict(value=submission_type, missing=True)
-        logger.debug(f"Looking up submission type: {submission_type['value']}")
-        submission_type = SubmissionType.query(name=submission_type['value'])
+        if isinstance(self.submission_type, str):
+            self.submission_type = dict(value=self.submission_type, missing=True)
+        logger.debug(f"Looking up submission type: {self.submission_type['value']}")
+        submission_type = SubmissionType.query(name=self.submission_type['value'])
         info_map = submission_type.info_map
         # Get the parse_info method from the submission type specified
         self.custom_parser = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=submission_type.name).parse_info
@@ -172,16 +179,25 @@ class InfoParser(object):
         Returns:
             dict: key:value of basic info
         """        
+        if isinstance(self.submission_type, str):
+            self.submission_type = dict(value=self.submission_type, missing=True)
         dicto = {}
+        exclude_from_generic = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=self.submission_type['value']).get_default_info("parser_ignore")
+        # This loop parses generic info
+        logger.debug(f"Map: {self.map}")
+        # time.sleep(5)
         for sheet in self.xl.sheet_names:
             df = self.xl.parse(sheet, header=None)
             relevant = {}
             for k, v in self.map.items():
+                # exclude from generic parsing
+                if k in exclude_from_generic:
+                    continue
+                # If the value is hardcoded put it in the dictionary directly.
                 if isinstance(v, str):
                     dicto[k] = dict(value=v, missing=False)
                     continue
-                if k in ["samples", "all_sheets"]:
-                    continue
+                logger.debug(f"Looking for {k} in self.map")
                 if sheet in self.map[k]['sheets']:
                     relevant[k] = v
             logger.debug(f"relevant map for {sheet}: {pformat(relevant)}")
@@ -252,6 +268,7 @@ class ReagentParser(object):
                     lot = df.iat[relevant[item]['lot']['row']-1, relevant[item]['lot']['column']-1]
                     expiry = df.iat[relevant[item]['expiry']['row']-1, relevant[item]['expiry']['column']-1]
                     if 'comment' in relevant[item].keys():
+                        logger.debug(f"looking for {relevant[item]} comment.")
                         comment = df.iat[relevant[item]['comment']['row']-1, relevant[item]['comment']['column']-1]
                     else:
                         comment = ""
@@ -294,7 +311,7 @@ class SampleParser(object):
         sample_info_map = self.fetch_sample_info_map(submission_type=submission_type, sample_map=sample_map)
         logger.debug(f"sample_info_map: {sample_info_map}")
         self.plate_map = self.construct_plate_map(plate_map_location=sample_info_map['plate_map'])
-        logger.debug(f"plate_map: {self.plate_map}")
+        # logger.debug(f"plate_map: {self.plate_map}")
         self.lookup_table = self.construct_lookup_table(lookup_table_location=sample_info_map['lookup_table'])
         if "plates" in sample_info_map:
             self.plates = sample_info_map['plates']
@@ -439,7 +456,7 @@ class SampleParser(object):
         """        
         result = None
         new_samples = []
-        logger.debug(f"Starting samples: {pformat(self.samples)}")
+        # logger.debug(f"Starting samples: {pformat(self.samples)}")
         for sample in self.samples:
             translated_dict = {}
             for k, v in sample.items():

src/submissions/backend/validators/__init__.py

@@ -74,8 +74,8 @@ class RSLNamer(object):
             check = True
         if check:
             # logger.debug("Final option, ask the user for submission type")
-            from frontend.widgets import SubmissionTypeSelector
-            dlg = SubmissionTypeSelector(title="Couldn't parse submission type.", message="Please select submission type from list below.")
+            from frontend.widgets import ObjectSelector
+            dlg = ObjectSelector(title="Couldn't parse submission type.", message="Please select submission type from list below.", obj_type=SubmissionType)
             if dlg.exec():
                 submission_type = dlg.parse_form()
         submission_type = submission_type.replace("_", " ")

src/submissions/backend/validators/pydant.py

@@ -8,13 +8,13 @@ from pydantic import BaseModel, field_validator, Field
 from datetime import date, datetime, timedelta
 from dateutil.parser import parse
 from dateutil.parser._parser import ParserError
-from typing import List, Tuple
+from typing import List, Tuple, Literal
 from . import RSLNamer
 from pathlib import Path
 from tools import check_not_nan, convert_nans_to_nones, Report, Result, row_map
 from backend.db.models import *
 from sqlalchemy.exc import StatementError, IntegrityError
-from PyQt6.QtWidgets import QComboBox, QWidget
+from PyQt6.QtWidgets import QWidget
 from openpyxl import load_workbook, Workbook
 from io import BytesIO
 
@@ -24,7 +24,7 @@ class PydReagent(BaseModel):
     
     lot: str|None
     type: str|None
-    expiry: date|None
+    expiry: date|Literal['NA']|None
     name: str|None
     missing: bool = Field(default=True)
     comment: str|None = Field(default="", validate_default=True)
@@ -77,6 +77,8 @@ class PydReagent(BaseModel):
             match value:
                 case int():
                     return datetime.fromordinal(datetime(1900, 1, 1).toordinal() + value - 2).date()
+                case 'NA':
+                    return value
                 case str():
                     return parse(value)
                 case date():
@@ -87,6 +89,13 @@ class PydReagent(BaseModel):
             value = date.today()
         return value
     
+    @field_validator("expiry")
+    @classmethod
+    def date_na(cls, value):
+        if isinstance(value, date) and value.year == 1970:
+            value = "NA"
+        return value
+    
     @field_validator("name", mode="before")
     @classmethod
     def enforce_name(cls, value, values):
@@ -125,6 +134,10 @@ class PydReagent(BaseModel):
                             reagent.type.append(reagent_type)
                     case "comment":
                         continue
+                    case "expiry":
+                        if isinstance(value, str):
+                            value = date(year=1970, month=1, day=1)
+                        reagent.expiry = value
                     case _:
                         try:
                             reagent.__setattr__(key, value)
@@ -271,6 +284,7 @@ class PydSubmission(BaseModel, extra='allow'):
     reagents: List[dict]|List[PydReagent] = []
     samples: List[PydSample]
     equipment: List[PydEquipment]|None =[]
+    cost_centre: dict|None = Field(default=dict(value=None, missing=True), validate_default=True)
 
     @field_validator('equipment', mode='before')
     @classmethod
@@ -332,10 +346,28 @@ class PydSubmission(BaseModel, extra='allow'):
     @field_validator("submitting_lab", mode="before")
     @classmethod
     def rescue_submitting_lab(cls, value):
-        if value == None:
+        if value is None:
             return dict(value=None, missing=True)
         return value
     
+    @field_validator("submitting_lab")
+    @classmethod
+    def lookup_submitting_lab(cls, value):
+        if isinstance(value['value'], str):
+            try:
+                value['value'] = Organization.query(name=value['value']).name
+            except AttributeError:
+                value['value'] = None
+        if value['value'] is None:
+            value['missing'] = True
+            from frontend.widgets.pop_ups import ObjectSelector
+            dlg = ObjectSelector(title="Missing Submitting Lab", message="We need a submitting lab. Please select from the list.", obj_type=Organization)
+            if dlg.exec():
+                value['value'] = dlg.getValues()
+            else:
+                value['value'] = None
+        return value
+        
     @field_validator("rsl_plate_num", mode='before')
     @classmethod
     def rescue_rsl_number(cls, value):
@@ -427,6 +459,30 @@ class PydSubmission(BaseModel, extra='allow'):
             output.append(sample)
         return output
 
+    @field_validator("cost_centre", mode="before")
+    @classmethod
+    def rescue_cost_centre(cls, value):
+        match value:
+            case dict():
+                return value
+            case _:
+                return dict(value=value, missing=True)
+
+    @field_validator("cost_centre")
+    @classmethod
+    def get_cost_centre(cls, value, values):
+        # logger.debug(f"Value coming in for cost_centre: {value}")
+        match value['value']:
+            case None:
+                from backend.db.models import Organization
+                org = Organization.query(name=values.data['submitting_lab']['value'])
+                try:
+                    return dict(value=org.cost_centre, missing=True)
+                except AttributeError:
+                    return dict(value="xxx", missing=True)
+            case _:
+                return value
+
     def set_attribute(self, key, value):
         self.__setattr__(name=key, value=value)
 
@@ -599,6 +655,7 @@ class PydSubmission(BaseModel, extra='allow'):
         else:
             info = {k:v for k,v in self.improved_dict().items() if isinstance(v, dict)}
             reagents = self.reagents
+            
         if len(reagents + list(info.keys())) == 0:
             # logger.warning("No info to fill in, returning")
             return None
@@ -616,14 +673,14 @@ class PydSubmission(BaseModel, extra='allow'):
             new_reagent = {}
             new_reagent['type'] = reagent.type
             new_reagent['lot'] = excel_map[new_reagent['type']]['lot']
-            new_reagent['lot']['value'] = reagent.lot
+            new_reagent['lot']['value'] = reagent.lot or "NA"
             new_reagent['expiry'] = excel_map[new_reagent['type']]['expiry']
-            new_reagent['expiry']['value'] = reagent.expiry
+            new_reagent['expiry']['value'] = reagent.expiry or "NA"
             new_reagent['sheet'] = excel_map[new_reagent['type']]['sheet']
             # name is only present for Bacterial Culture
             try:
                 new_reagent['name'] = excel_map[new_reagent['type']]['name']
-                new_reagent['name']['value'] = reagent.name
+                new_reagent['name']['value'] = reagent.name or "Not Applicable"
             except Exception as e:
                 logger.error(f"Couldn't get name due to {e}")
             new_reagents.append(new_reagent)
@@ -657,6 +714,8 @@ class PydSubmission(BaseModel, extra='allow'):
                 # logger.debug(f"Attempting to write lot {reagent['lot']['value']} in: row {reagent['lot']['row']}, column {reagent['lot']['column']}")
                 worksheet.cell(row=reagent['lot']['row'], column=reagent['lot']['column'], value=reagent['lot']['value'])
                 # logger.debug(f"Attempting to write expiry {reagent['expiry']['value']} in: row {reagent['expiry']['row']}, column {reagent['expiry']['column']}")
+                if reagent['expiry']['value'].year == 1970:
+                    reagent['expiry']['value'] = "NA"
                 worksheet.cell(row=reagent['expiry']['row'], column=reagent['expiry']['column'], value=reagent['expiry']['value'])
                 try:
                     # logger.debug(f"Attempting to write name {reagent['name']['value']} in: row {reagent['name']['row']}, column {reagent['name']['column']}")
@@ -790,14 +849,13 @@ class PydSubmission(BaseModel, extra='allow'):
         logger.debug(f"Extraction kit: {extraction_kit}. Is it a string? {isinstance(extraction_kit, str)}")
         if isinstance(extraction_kit, str):
             extraction_kit = dict(value=extraction_kit)
-        if extraction_kit is not None:
-            if extraction_kit != self.extraction_kit['value']:
+        if extraction_kit is not None and extraction_kit != self.extraction_kit['value']:
                 self.extraction_kit['value'] = extraction_kit['value']
-                reagenttypes = []
-            else:
-                reagenttypes = [item.type for item in self.reagents]
-        else:
-            reagenttypes = [item.type for item in self.reagents]
+        #         reagenttypes = []
+        #     else:
+        #         reagenttypes = [item.type for item in self.reagents]
+        # else:
+        #     reagenttypes = [item.type for item in self.reagents]
         logger.debug(f"Looking up {self.extraction_kit['value']}")
         ext_kit = KitType.query(name=self.extraction_kit['value'])
         ext_kit_rtypes = [item.to_pydantic() for item in ext_kit.get_reagents(required=True, submission_type=self.submission_type['value'])]
@@ -808,21 +866,26 @@ class PydSubmission(BaseModel, extra='allow'):
         # logger.debug(f"Checking if reagents match kit contents: {check}")
         # # what reagent types are in both lists?
         # missing = list(set(ext_kit_rtypes).difference(reagenttypes))
-        missing = []
-        output_reagents = self.reagents
-        # output_reagents = ext_kit_rtypes
-        logger.debug(f"Already have these reagent types: {reagenttypes}")
-        for rt in ext_kit_rtypes:
-            if rt.type not in reagenttypes:
-                missing.append(rt)
-                if rt.type not in [item.type for item in output_reagents]:
-                    output_reagents.append(rt)
-        logger.debug(f"Missing reagents types: {missing}")
+        # missing = []
+        # Exclude any reagenttype found in this pyd not expected in kit.
+        expected_check = [item.type for item in ext_kit_rtypes]
+        output_reagents = [rt for rt in self.reagents if rt.type in expected_check]
+        logger.debug(f"Already have these reagent types: {output_reagents}")
+        missing_check = [item.type for item in output_reagents]
+        missing_reagents = [rt for rt in ext_kit_rtypes if rt.type not in missing_check]
+        missing_reagents += [rt for rt in output_reagents if rt.missing]
+        # for rt in ext_kit_rtypes:
+        #     if rt.type not in [item.type for item in output_reagents]:
+        #         missing.append(rt)
+        #         if rt.type not in [item.type for item in output_reagents]:
+        #             output_reagents.append(rt)
+        output_reagents += [rt for rt in missing_reagents if rt not in output_reagents]
+        logger.debug(f"Missing reagents types: {missing_reagents}")
         # if lists are equal return no problem
-        if len(missing)==0:
+        if len(missing_reagents)==0:
             result = None
         else:
-            result = Result(msg=f"The submission you are importing is missing some reagents expected by the kit.\n\nIt looks like you are missing: {[item.type.upper() for item in missing]}\n\nAlternatively, you may have set the wrong extraction kit.\n\nThe program will populate lists using existing reagents.\n\nPlease make sure you check the lots carefully!", status="Warning")
+            result = Result(msg=f"The excel sheet you are importing is missing some reagents expected by the kit.\n\nIt looks like you are missing: {[item.type.upper() for item in missing_reagents]}\n\nAlternatively, you may have set the wrong extraction kit.\n\nThe program will populate lists using existing reagents.\n\nPlease make sure you check the lots carefully!", status="Warning")
         report.add_result(result)
         return output_reagents, report
 

src/submissions/frontend/widgets/misc.py

@@ -85,10 +85,10 @@ class AddReagentForm(QDialog):
         Returns:
             dict: Output info
         """        
-        return dict(name=self.name_input.currentText(), 
-                    lot=self.lot_input.text(), 
+        return dict(name=self.name_input.currentText().strip(), 
+                    lot=self.lot_input.text().strip(), 
                     expiry=self.exp_input.date().toPyDate(),
-                    type=self.type_input.currentText())
+                    type=self.type_input.currentText().strip())
 
     def update_names(self):
         """

src/submissions/frontend/widgets/pop_ups.py

@@ -7,7 +7,7 @@ from PyQt6.QtWidgets import (
 )
 from tools import jinja_template_loading
 import logging
-from backend.db.models import KitType, SubmissionType
+from backend.db import models
 from typing import Literal
 
 logger = logging.getLogger(f"submissions.{__name__}")
@@ -45,16 +45,18 @@ class AlertPop(QMessageBox):
         self.setInformativeText(message)
         self.setWindowTitle(f"{owner} - {status.title()}")
 
-class KitSelector(QDialog):
+class ObjectSelector(QDialog):
     """
-    dialog to input KitType manually
+    dialog to input BaseClass type manually
     """    
-    def __init__(self, title:str, message:str) -> QDialog:
+    def __init__(self, title:str, message:str, obj_type:str|models.BaseClass) -> QDialog:
         super().__init__()
         self.setWindowTitle(title)
         self.widget = QComboBox()
-        kits = [item.name for item in KitType.query()]
-        self.widget.addItems(kits)
+        if isinstance(obj_type, str):
+            obj_type: models.BaseClass = getattr(models, obj_type)
+        items = [item.name for item in obj_type.query()]
+        self.widget.addItems(items)
         self.widget.setEditable(False)
         # set yes/no buttons
         QBtn = QDialogButtonBox.StandardButton.Ok | QDialogButtonBox.StandardButton.Cancel
@@ -78,36 +80,69 @@ class KitSelector(QDialog):
         """        
         return self.widget.currentText()
 
-class SubmissionTypeSelector(QDialog):
-    """
-    dialog to input SubmissionType manually
-    """    
-    def __init__(self, title:str, message:str) -> QDialog:
-        super().__init__()
-        self.setWindowTitle(title)
-        self.widget = QComboBox()
-        # sub_type = [item.name for item in lookup_submission_type(ctx=ctx)]
-        sub_type = [item.name for item in SubmissionType.query()]
-        self.widget.addItems(sub_type)
-        self.widget.setEditable(False)
-        # set yes/no buttons
-        QBtn = QDialogButtonBox.StandardButton.Ok | QDialogButtonBox.StandardButton.Cancel
-        self.buttonBox = QDialogButtonBox(QBtn)
-        self.buttonBox.accepted.connect(self.accept)
-        self.buttonBox.rejected.connect(self.reject)
-        self.layout = QVBoxLayout()
-        # Text for the yes/no question
-        message = QLabel(message)
-        self.layout.addWidget(message)
-        self.layout.addWidget(self.widget)
-        self.layout.addWidget(self.buttonBox)
-        self.setLayout(self.layout)
+# class KitSelector(QDialog):
+#     """
+#     dialog to input KitType manually
+#     """    
+#     def __init__(self, title:str, message:str) -> QDialog:
+#         super().__init__()
+#         self.setWindowTitle(title)
+#         self.widget = QComboBox()
+#         kits = [item.name for item in KitType.query()]
+#         self.widget.addItems(kits)
+#         self.widget.setEditable(False)
+#         # set yes/no buttons
+#         QBtn = QDialogButtonBox.StandardButton.Ok | QDialogButtonBox.StandardButton.Cancel
+#         self.buttonBox = QDialogButtonBox(QBtn)
+#         self.buttonBox.accepted.connect(self.accept)
+#         self.buttonBox.rejected.connect(self.reject)
+#         self.layout = QVBoxLayout()
+#         # Text for the yes/no question
+#         message = QLabel(message)
+#         self.layout.addWidget(message)
+#         self.layout.addWidget(self.widget)
+#         self.layout.addWidget(self.buttonBox)
+#         self.setLayout(self.layout)
 
-    def parse_form(self) -> str:
-        """
-        Pulls SubmissionType(str) from widget
+#     def getValues(self) -> str:
+#         """
+#         Get KitType(str) from widget
 
-        Returns:
-            str: SubmissionType as str
-        """
-        return self.widget.currentText()
+#         Returns:
+#             str: KitType as str
+#         """        
+#         return self.widget.currentText()
+
+# class SubmissionTypeSelector(QDialog):
+#     """
+#     dialog to input SubmissionType manually
+#     """    
+#     def __init__(self, title:str, message:str) -> QDialog:
+#         super().__init__()
+#         self.setWindowTitle(title)
+#         self.widget = QComboBox()
+#         # sub_type = [item.name for item in lookup_submission_type(ctx=ctx)]
+#         sub_type = [item.name for item in SubmissionType.query()]
+#         self.widget.addItems(sub_type)
+#         self.widget.setEditable(False)
+#         # set yes/no buttons
+#         QBtn = QDialogButtonBox.StandardButton.Ok | QDialogButtonBox.StandardButton.Cancel
+#         self.buttonBox = QDialogButtonBox(QBtn)
+#         self.buttonBox.accepted.connect(self.accept)
+#         self.buttonBox.rejected.connect(self.reject)
+#         self.layout = QVBoxLayout()
+#         # Text for the yes/no question
+#         message = QLabel(message)
+#         self.layout.addWidget(message)
+#         self.layout.addWidget(self.widget)
+#         self.layout.addWidget(self.buttonBox)
+#         self.setLayout(self.layout)
+
+#     def parse_form(self) -> str:
+#         """
+#         Pulls SubmissionType(str) from widget
+
+#         Returns:
+#             str: SubmissionType as str
+#         """
+#         return self.widget.currentText()

src/submissions/frontend/widgets/submission_details.py

@@ -5,12 +5,12 @@ from PyQt6.QtWebChannel import QWebChannel
 from PyQt6.QtCore import Qt, pyqtSlot
 
 from backend.db.models import BasicSubmission, BasicSample
-from tools import check_if_app, check_authorization, is_power_user
+from tools import is_power_user, html_to_pdf
 from .functions import select_save_file
 from io import BytesIO
 from tempfile import TemporaryFile, TemporaryDirectory
 from pathlib import Path
-from xhtml2pdf import pisa
+# from xhtml2pdf import pisa
 import logging, base64
 from getpass import getuser
 from datetime import datetime
@@ -54,6 +54,7 @@ class SubmissionDetails(QDialog):
         self.channel = QWebChannel()
         self.channel.registerObject('backend', self)
         self.submission_details(submission=sub)
+        self.rsl_plate_num = sub.rsl_plate_num
         self.webview.page().setWebChannel(self.channel)
 
     @pyqtSlot(str)
@@ -86,9 +87,9 @@ class SubmissionDetails(QDialog):
         logger.debug(f"Submission details data:\n{pformat({k:v for k,v in self.base_dict.items() if k != 'samples'})}")
         # don't want id
         del self.base_dict['id']
-        logger.debug(f"Creating barcode.")
-        if not check_if_app():
-            self.base_dict['barcode'] = base64.b64encode(submission.make_plate_barcode(width=120, height=30)).decode('utf-8')
+        # logger.debug(f"Creating barcode.")
+        # if not check_if_app():
+        #     self.base_dict['barcode'] = base64.b64encode(submission.make_plate_barcode(width=120, height=30)).decode('utf-8')
         logger.debug(f"Making platemap...")
         self.base_dict['platemap'] = submission.make_plate_map()
         self.base_dict, self.template = submission.get_details_template(base_dict=self.base_dict)
@@ -103,8 +104,9 @@ class SubmissionDetails(QDialog):
         logger.debug(f"Signing off on {submission} - ({getuser()})")
         if isinstance(submission, str):
             submission = BasicSubmission.query(rsl_number=submission)
-        submission.uploaded_by = getuser()
+        submission.signed_by = getuser()
         submission.save()
+        self.submission_details(submission=self.rsl_plate_num)
 
     def export(self):
         """
@@ -113,7 +115,7 @@ class SubmissionDetails(QDialog):
         fname = select_save_file(obj=self, default_name=self.base_dict['Plate Number'], extension="pdf")
         image_io = BytesIO()
         temp_dir = Path(TemporaryDirectory().name)
-        hti = Html2Image(output_path=temp_dir, size=(1200, 750))
+        hti = Html2Image(output_path=temp_dir, size=(2400, 1500))
         temp_file = Path(TemporaryFile(dir=temp_dir, suffix=".png").name)
         screenshot = hti.screenshot(self.base_dict['platemap'], save_as=temp_file.name)
         export_map = Image.open(screenshot[0])
@@ -126,8 +128,9 @@ class SubmissionDetails(QDialog):
         del self.base_dict['platemap']
         self.html2 = self.template.render(sub=self.base_dict)
         try:
-            with open(fname, "w+b") as f:
-                pisa.CreatePDF(self.html2, dest=f)
+            # with open(fname, "w+b") as f:
+            #     pisa.CreatePDF(self.html2, dest=f)
+            html_to_pdf(html=self.html2, output_file=fname)
         except PermissionError as e:
             logger.error(f"Error saving pdf: {e}")
             msg = QMessageBox()

src/submissions/frontend/widgets/submission_table.py

@@ -8,8 +8,8 @@ from PyQt6.QtCore import Qt, QAbstractTableModel, QSortFilterProxyModel
 from PyQt6.QtGui import QAction, QCursor
 from backend.db.models import BasicSubmission
 from backend.excel import make_report_html, make_report_xlsx
-from tools import Report, Result, row_map, get_first_blank_df_row
-from xhtml2pdf import pisa
+from tools import Report, Result, row_map, get_first_blank_df_row, html_to_pdf
+# from xhtml2pdf import pisa
 from .functions import select_save_file, select_open_file
 from .misc import ReportDatePicker
 import pandas as pd
@@ -324,8 +324,9 @@ class SubmissionsSheet(QTableView):
             html = make_report_html(df=summary_df, start_date=info['start_date'], end_date=info['end_date'])
             # get save location of report
             fname = select_save_file(obj=self, default_name=f"Submissions_Report_{info['start_date']}-{info['end_date']}.pdf", extension="pdf")
-            with open(fname, "w+b") as f:
-                pisa.CreatePDF(html, dest=f)
+            # with open(fname, "w+b") as f:
+            #     pisa.CreatePDF(html, dest=f)
+            html_to_pdf(html=html, output_file=fname)
             writer = pd.ExcelWriter(fname.with_suffix(".xlsx"), engine='openpyxl')
             summary_df.to_excel(writer, sheet_name="Report")
             detailed_df.to_excel(writer, sheet_name="Details", index=False)

src/submissions/frontend/widgets/submission_widget.py

@@ -135,7 +135,7 @@ class SubmissionFormContainer(QWidget):
             info = dlg.parse_form()
             logger.debug(f"Reagent info: {info}")
             # create reagent object
-            reagent = PydReagent(ctx=self.app.ctx, **info)
+            reagent = PydReagent(ctx=self.app.ctx, **info, missing=False)
             # send reagent to db
             sqlobj, result = reagent.toSQL()
             sqlobj.save()
@@ -150,17 +150,18 @@ class SubmissionFormWidget(QWidget):
         # self.report = Report()
         self.app = parent.app
         self.pyd = submission
-        # self.input = [{k:v} for k,v in kwargs.items()]
-        # self.samples = []
         self.missing_info = []
-        self.ignore = ['filepath', 'samples', 'reagents', 'csv', 'ctx', 'comment', 
-                       'equipment', 'gel_controls', 'id', 'cost', 'extraction_info', 
-                       'controls', 'pcr_info', 'gel_info', 'gel_image']
-        self.recover = ['filepath', 'samples', 'csv', 'comment', 'equipment']
+        st = SubmissionType.query(name=self.pyd.submission_type['value']).get_submission_class()
+        defaults = st.get_default_info("form_recover", "form_ignore")
+        self.recover = defaults['form_recover']
+        self.ignore = defaults['form_ignore']
+        # self.ignore += self.recover
+        # logger.debug(f"Attempting to extend ignore list with {self.pyd.submission_type['value']}")
         self.layout = QVBoxLayout()
         # for k, v in kwargs.items():
         for k in list(self.pyd.model_fields.keys()) + list(self.pyd.model_extra.keys()):
-            if k not in self.ignore:
+            if k in self.ignore:
+                continue
             try:
                 value = self.pyd.__getattribute__(k)
             except AttributeError:
@@ -171,24 +172,7 @@ class SubmissionFormWidget(QWidget):
                 self.layout.addWidget(add_widget)
             if k == "extraction_kit":
                 add_widget.input.currentTextChanged.connect(self.scrape_reagents)
-            # else:
-            # self.__setattr__(k, v)
-        # self.scrape_reagents(self.extraction_kit['value'])
         self.scrape_reagents(self.pyd.extraction_kit)
-        # extraction kit must be added last so widget order makes sense.
-        # self.layout.addWidget(self.create_widget(key="extraction_kit", value=self.extraction_kit, submission_type=self.submission_type))
-        # if hasattr(self.pyd, "csv"):
-        #     export_csv_btn = QPushButton("Export CSV")
-        #     export_csv_btn.setObjectName("export_csv_btn")
-        #     self.layout.addWidget(export_csv_btn)
-        #     export_csv_btn.clicked.connect(self.export_csv_function)
-        # submit_btn = QPushButton("Submit")
-        # submit_btn.setObjectName("submit_btn")
-        # self.layout.addWidget(submit_btn)
-        # submit_btn.clicked.connect(self.submit_new_sample_function)
-        # self.setLayout(self.layout)
-        # self.app.report.add_result(self.report)
-        # self.app.result_reporter()
 
     def create_widget(self, key:str, value:dict|PydReagent, submission_type:str|None=None, extraction_kit:str|None=None) -> "self.InfoItem":
         """
@@ -633,6 +617,11 @@ class SubmissionFormWidget(QWidget):
             self.reagent = reagent
             self.extraction_kit = extraction_kit
             layout = QVBoxLayout()
+            # layout = QGridLayout()
+            # self.check_box = QCheckBox(self)
+            # self.check_box.setChecked(True)
+            # self.check_box.stateChanged.connect(self.check_uncheck)
+            # layout.addWidget(self.check_box, 0,0)
             self.label = self.ReagentParsedLabel(reagent=reagent)
             layout.addWidget(self.label)
             self.lot = self.ReagentLot(reagent=reagent, extraction_kit=extraction_kit)
@@ -645,6 +634,14 @@ class SubmissionFormWidget(QWidget):
             # If changed set self.missing to True and update self.label
             self.lot.currentTextChanged.connect(self.updated)
 
+        # def check_uncheck(self):
+        #     if self.check_box.isChecked():
+        #         self.lot.setCurrentIndex(0)
+        #         self.lot.setEnabled(True)
+        #     else:
+        #         self.lot.setCurrentText("Not Applicable")
+        #         self.lot.setEnabled(False)
+
         def parse_form(self) -> Tuple[PydReagent, dict]:
             """
             Pulls form info into PydReagent
@@ -652,6 +649,8 @@ class SubmissionFormWidget(QWidget):
             Returns:
                 Tuple[PydReagent, dict]: PydReagent and Report(?)
             """        
+            # if not self.check_box.isChecked():
+            #     return None, None
             lot = self.lot.currentText()
             logger.debug(f"Using this lot for the reagent {self.reagent}: {lot}")
             wanted_reagent = Reagent.query(lot_number=lot, reagent_type=self.reagent.type)
@@ -671,7 +670,7 @@ class SubmissionFormWidget(QWidget):
                 rt = ReagentType.query(name=self.reagent.type)
                 if rt == None:
                     rt = ReagentType.query(kit_type=self.extraction_kit, reagent=wanted_reagent)
-                return PydReagent(name=wanted_reagent.name, lot=wanted_reagent.lot, type=rt.name, expiry=wanted_reagent.expiry, parsed=not self.missing), None
+                return PydReagent(name=wanted_reagent.name, lot=wanted_reagent.lot, type=rt.name, expiry=wanted_reagent.expiry, missing=False), None
 
         def updated(self):
             """
@@ -736,8 +735,11 @@ class SubmissionFormWidget(QWidget):
                             looked_up_reg = None
                         # logger.debug(f"Because there was no reagent listed for {reagent.lot}, we will insert the last lot used: {looked_up_reg}")
                         if looked_up_reg != None:
+                            try:
                                 relevant_reagents.remove(str(looked_up_reg.lot))
                                 relevant_reagents.insert(0, str(looked_up_reg.lot))
+                            except ValueError as e:
+                                logger.error(f"Error reordering relevant reagents: {e}")
                 else:
                     if len(relevant_reagents) > 1:
                         # logger.debug(f"Found {reagent.lot} in relevant reagents: {relevant_reagents}. Moving to front of list.")

src/submissions/templates/basicsubmission_details.html

@@ -113,7 +113,7 @@
         {% endif %}
         {% if sub['export_map'] %}
         <h3><u>Plate map:</u></h3>
-        <img height="300px" width="650px" src="data:image/jpeg;base64,{{ sub['export_map'] | safe }}">
+        <img height="600px" width="1300px" src="data:image/jpeg;base64,{{ sub['export_map'] | safe }}">
         {% endif %}
         {% endblock %}
         {% if signing_permission %}

src/submissions/tools.py

@@ -14,6 +14,11 @@ from sqlalchemy import create_engine
 from pydantic import field_validator, BaseModel, Field
 from pydantic_settings import BaseSettings, SettingsConfigDict
 from typing import Any, Tuple, Literal, List
+from PyQt6.QtGui import QTextDocument, QPageSize
+from PyQt6.QtWebEngineWidgets import QWebEngineView
+
+# from PyQt6 import QtPrintSupport, QtCore, QtWebEngineWidgets
+from PyQt6.QtPrintSupport import QPrinter
 
 logger = logging.getLogger(f"submissions.{__name__}")
 
@@ -535,6 +540,18 @@ class Report(BaseModel):
 def rreplace(s, old, new):
     return (s[::-1].replace(old[::-1],new[::-1], 1))[::-1]
 
+def html_to_pdf(html, output_file:Path|str):
+    if isinstance(output_file, str):
+        output_file = Path(output_file)
+    # document = QTextDocument()
+    document = QWebEngineView()
+    document.setHtml(html) 
+    printer = QPrinter(QPrinter.PrinterMode.HighResolution)
+    printer.setOutputFormat(QPrinter.OutputFormat.PdfFormat)
+    printer.setOutputFileName(output_file.absolute().__str__())
+    printer.setPageSize(QPageSize(QPageSize.PageSizeId.A4))
+    document.print(printer)
+
 ctx = get_config(None)
 
 def is_power_user() -> bool: