Pending large code cleanup

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 202401.04
+
+- Large scale database refactor to increase modularity.
+
 ## 202401.01
 
 - Improved tooltips and form regeneration.

alembic.ini

@@ -55,8 +55,8 @@ version_path_separator = os  # Use os.pathsep. Default configuration used for ne
 # are written from script.py.mako
 # output_encoding = utf-8
 
-; sqlalchemy.url = sqlite:///L:\Robotics Laboratory Support\Submissions\submissions.db
+sqlalchemy.url = sqlite:///L:\Robotics Laboratory Support\Submissions\submissions.db
-sqlalchemy.url = sqlite:///C:\Users\lwark\Documents\Archives\Submissions_app_backups\DB_backups\submissions-new.db
+; sqlalchemy.url = sqlite:///C:\Users\lwark\Documents\Archives\Submissions_app_backups\DB_backups\submissions-new.db
 ; sqlalchemy.url = sqlite:///C:\Users\lwark\Documents\python\submissions\tests\test_assets\submissions-test.db
 
 

alembic/versions/70426df72f80_adding_gel_image_info_again.py

@@ -0,0 +1,34 @@
+"""adding gel image, info. Again
+
+Revision ID: 70426df72f80
+Revises: c4201b0ea9fe
+Create Date: 2024-01-30 08:47:22.809841
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '70426df72f80'
+down_revision = 'c4201b0ea9fe'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('_wastewaterartic', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('gel_image', sa.String(length=64), nullable=True))
+        batch_op.add_column(sa.Column('gel_info', sa.JSON(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('_wastewaterartic', schema=None) as batch_op:
+        batch_op.drop_column('gel_info')
+        batch_op.drop_column('gel_image')
+
+    # ### end Alembic commands ###

alembic/versions/70d5a751f579_tweaking_submission_sample_association.py

@@ -0,0 +1,38 @@
+"""tweaking submission sample association
+
+Revision ID: 70d5a751f579
+Revises: 97392dda5436
+Create Date: 2024-01-25 13:39:34.163501
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '70d5a751f579'
+down_revision = '97392dda5436'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('_submissionsampleassociation', schema=None) as batch_op:
+        batch_op.alter_column('id',
+               existing_type=sa.INTEGER(),
+               nullable=False)
+        batch_op.create_unique_constraint("ssa_id_unique", ['id'])
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('_submissionsampleassociation', schema=None) as batch_op:
+        batch_op.drop_constraint("ssa_id_unique", type_='unique')
+        batch_op.alter_column('id',
+               existing_type=sa.INTEGER(),
+               nullable=False)
+
+    # ### end Alembic commands ###

alembic/versions/97392dda5436_update_to_submissionsampleassociation.py

@@ -0,0 +1,50 @@
+"""Update to submissionsampleassociation
+
+Revision ID: 97392dda5436
+Revises: e3f6770ef515
+Create Date: 2024-01-25 09:10:04.384194
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '97392dda5436'
+down_revision = 'e3f6770ef515'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('_submissionsampleassociation', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=True))
+        batch_op.create_unique_constraint("submissionsampleassociation_id", ['id'])
+
+    with op.batch_alter_table('_wastewaterassociation', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('id', sa.INTEGER(), nullable=False))
+        # batch_op.drop_constraint("sample_id", type_='foreignkey')
+        # batch_op.drop_constraint("submission_id", type_='foreignkey')
+        batch_op.create_foreign_key("fk_subsampassoc_id", '_submissionsampleassociation', ['id'], ['id'])
+        # batch_op.drop_column('sample_id')
+        # batch_op.drop_column('submission_id')
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('_wastewaterassociation', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('submission_id', sa.INTEGER(), nullable=False))
+        batch_op.add_column(sa.Column('sample_id', sa.INTEGER(), nullable=False))
+        batch_op.drop_constraint(None, type_='foreignkey')
+        batch_op.create_foreign_key(None, '_submissionsampleassociation', ['submission_id'], ['submission_id'])
+        batch_op.create_foreign_key(None, '_submissionsampleassociation', ['sample_id'], ['sample_id'])
+        batch_op.drop_column('id')
+
+    with op.batch_alter_table('_submissionsampleassociation', schema=None) as batch_op:
+        batch_op.drop_constraint(None, type_='unique')
+        batch_op.drop_column('id')
+
+    # ### end Alembic commands ###

alembic/versions/c4201b0ea9fe_adding_gel_image_info.py

@@ -0,0 +1,42 @@
+"""adding gel image, info
+
+Revision ID: c4201b0ea9fe
+Revises: 70d5a751f579
+Create Date: 2024-01-30 08:42:03.928933
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'c4201b0ea9fe'
+down_revision = '70d5a751f579'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('_submissionsampleassociation', schema=None) as batch_op:
+        batch_op.create_unique_constraint("unique_ssa_id", ['id'])
+
+    with op.batch_alter_table('_wastewaterassociation', schema=None) as batch_op:
+        batch_op.alter_column('id',
+               existing_type=sa.INTEGER(),
+               nullable=False)
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('_wastewaterassociation', schema=None) as batch_op:
+        batch_op.alter_column('id',
+               existing_type=sa.INTEGER(),
+               nullable=True)
+
+    with op.batch_alter_table('_submissionsampleassociation', schema=None) as batch_op:
+        batch_op.drop_constraint("unique_ssa_id", type_='unique')
+
+    # ### end Alembic commands ###

alembic/versions/e3f6770ef515_first_commit.py

@@ -0,0 +1,340 @@
+"""First Commit
+
+Revision ID: e3f6770ef515
+Revises: 
+Create Date: 2024-01-22 14:01:02.958292
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'e3f6770ef515'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('_basicsample',
+    sa.Column('id', sa.INTEGER(), nullable=False),
+    sa.Column('submitter_id', sa.String(length=64), nullable=False),
+    sa.Column('sample_type', sa.String(length=32), nullable=True),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('submitter_id')
+    )
+    op.create_table('_contact',
+    sa.Column('id', sa.INTEGER(), nullable=False),
+    sa.Column('name', sa.String(length=64), nullable=True),
+    sa.Column('email', sa.String(length=64), nullable=True),
+    sa.Column('phone', sa.String(length=32), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('_controltype',
+    sa.Column('id', sa.INTEGER(), nullable=False),
+    sa.Column('name', sa.String(length=255), nullable=True),
+    sa.Column('targets', sa.JSON(), nullable=True),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('name')
+    )
+    op.create_table('_equipment',
+    sa.Column('id', sa.INTEGER(), nullable=False),
+    sa.Column('name', sa.String(length=64), nullable=True),
+    sa.Column('nickname', sa.String(length=64), nullable=True),
+    sa.Column('asset_number', sa.String(length=16), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('_equipmentrole',
+    sa.Column('id', sa.INTEGER(), nullable=False),
+    sa.Column('name', sa.String(length=32), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('_kittype',
+    sa.Column('id', sa.INTEGER(), nullable=False),
+    sa.Column('name', sa.String(length=64), nullable=True),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('name')
+    )
+    op.create_table('_organization',
+    sa.Column('id', sa.INTEGER(), nullable=False),
+    sa.Column('name', sa.String(length=64), nullable=True),
+    sa.Column('cost_centre', sa.String(), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('_process',
+    sa.Column('id', sa.INTEGER(), nullable=False),
+    sa.Column('name', sa.String(length=64), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('_reagenttype',
+    sa.Column('id', sa.INTEGER(), nullable=False),
+    sa.Column('name', sa.String(length=64), nullable=True),
+    sa.Column('eol_ext', sa.Interval(), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('_submissiontype',
+    sa.Column('id', sa.INTEGER(), nullable=False),
+    sa.Column('name', sa.String(length=128), nullable=True),
+    sa.Column('info_map', sa.JSON(), nullable=True),
+    sa.Column('template_file', sa.BLOB(), nullable=True),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('name')
+    )
+    op.create_table('_bacterialculturesample',
+    sa.Column('id', sa.INTEGER(), nullable=False),
+    sa.Column('organism', sa.String(length=64), nullable=True),
+    sa.Column('concentration', sa.String(length=16), nullable=True),
+    sa.ForeignKeyConstraint(['id'], ['_basicsample.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('_discount',
+    sa.Column('id', sa.INTEGER(), nullable=False),
+    sa.Column('kit_id', sa.INTEGER(), nullable=True),
+    sa.Column('client_id', sa.INTEGER(), nullable=True),
+    sa.Column('name', sa.String(length=128), nullable=True),
+    sa.Column('amount', sa.FLOAT(precision=2), nullable=True),
+    sa.ForeignKeyConstraint(['client_id'], ['_organization.id'], name='fk_org_id', ondelete='SET NULL'),
+    sa.ForeignKeyConstraint(['kit_id'], ['_kittype.id'], name='fk_kit_type_id', ondelete='SET NULL'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('_equipment_processes',
+    sa.Column('process_id', sa.INTEGER(), nullable=True),
+    sa.Column('equipment_id', sa.INTEGER(), nullable=True),
+    sa.ForeignKeyConstraint(['equipment_id'], ['_equipment.id'], ),
+    sa.ForeignKeyConstraint(['process_id'], ['_process.id'], )
+    )
+    op.create_table('_equipmentroles_equipment',
+    sa.Column('equipment_id', sa.INTEGER(), nullable=True),
+    sa.Column('equipmentroles_id', sa.INTEGER(), nullable=True),
+    sa.ForeignKeyConstraint(['equipment_id'], ['_equipment.id'], ),
+    sa.ForeignKeyConstraint(['equipmentroles_id'], ['_equipmentrole.id'], )
+    )
+    op.create_table('_equipmentroles_processes',
+    sa.Column('process_id', sa.INTEGER(), nullable=True),
+    sa.Column('equipmentrole_id', sa.INTEGER(), nullable=True),
+    sa.ForeignKeyConstraint(['equipmentrole_id'], ['_equipmentrole.id'], ),
+    sa.ForeignKeyConstraint(['process_id'], ['_process.id'], )
+    )
+    op.create_table('_kittypereagenttypeassociation',
+    sa.Column('reagent_types_id', sa.INTEGER(), nullable=False),
+    sa.Column('kits_id', sa.INTEGER(), nullable=False),
+    sa.Column('submission_type_id', sa.INTEGER(), nullable=False),
+    sa.Column('uses', sa.JSON(), nullable=True),
+    sa.Column('required', sa.INTEGER(), nullable=True),
+    sa.Column('last_used', sa.String(length=32), nullable=True),
+    sa.ForeignKeyConstraint(['kits_id'], ['_kittype.id'], ),
+    sa.ForeignKeyConstraint(['reagent_types_id'], ['_reagenttype.id'], ),
+    sa.ForeignKeyConstraint(['submission_type_id'], ['_submissiontype.id'], ),
+    sa.PrimaryKeyConstraint('reagent_types_id', 'kits_id', 'submission_type_id')
+    )
+    op.create_table('_kittypes_processes',
+    sa.Column('process_id', sa.INTEGER(), nullable=True),
+    sa.Column('kit_id', sa.INTEGER(), nullable=True),
+    sa.ForeignKeyConstraint(['kit_id'], ['_kittype.id'], ),
+    sa.ForeignKeyConstraint(['process_id'], ['_process.id'], )
+    )
+    op.create_table('_orgs_contacts',
+    sa.Column('org_id', sa.INTEGER(), nullable=True),
+    sa.Column('contact_id', sa.INTEGER(), nullable=True),
+    sa.ForeignKeyConstraint(['contact_id'], ['_contact.id'], ),
+    sa.ForeignKeyConstraint(['org_id'], ['_organization.id'], )
+    )
+    op.create_table('_reagent',
+    sa.Column('id', sa.INTEGER(), nullable=False),
+    sa.Column('type_id', sa.INTEGER(), nullable=True),
+    sa.Column('name', sa.String(length=64), nullable=True),
+    sa.Column('lot', sa.String(length=64), nullable=True),
+    sa.Column('expiry', sa.TIMESTAMP(), nullable=True),
+    sa.ForeignKeyConstraint(['type_id'], ['_reagenttype.id'], name='fk_reagent_type_id', ondelete='SET NULL'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('_submissiontypeequipmentroleassociation',
+    sa.Column('equipmentrole_id', sa.INTEGER(), nullable=False),
+    sa.Column('submissiontype_id', sa.INTEGER(), nullable=False),
+    sa.Column('uses', sa.JSON(), nullable=True),
+    sa.Column('static', sa.INTEGER(), nullable=True),
+    sa.ForeignKeyConstraint(['equipmentrole_id'], ['_equipmentrole.id'], ),
+    sa.ForeignKeyConstraint(['submissiontype_id'], ['_submissiontype.id'], ),
+    sa.PrimaryKeyConstraint('equipmentrole_id', 'submissiontype_id')
+    )
+    op.create_table('_submissiontypekittypeassociation',
+    sa.Column('submission_types_id', sa.INTEGER(), nullable=False),
+    sa.Column('kits_id', sa.INTEGER(), nullable=False),
+    sa.Column('mutable_cost_column', sa.FLOAT(precision=2), nullable=True),
+    sa.Column('mutable_cost_sample', sa.FLOAT(precision=2), nullable=True),
+    sa.Column('constant_cost', sa.FLOAT(precision=2), nullable=True),
+    sa.ForeignKeyConstraint(['kits_id'], ['_kittype.id'], ),
+    sa.ForeignKeyConstraint(['submission_types_id'], ['_submissiontype.id'], ),
+    sa.PrimaryKeyConstraint('submission_types_id', 'kits_id')
+    )
+    op.create_table('_submissiontypes_processes',
+    sa.Column('process_id', sa.INTEGER(), nullable=True),
+    sa.Column('equipmentroles_id', sa.INTEGER(), nullable=True),
+    sa.ForeignKeyConstraint(['equipmentroles_id'], ['_submissiontype.id'], ),
+    sa.ForeignKeyConstraint(['process_id'], ['_process.id'], )
+    )
+    op.create_table('_wastewatersample',
+    sa.Column('id', sa.INTEGER(), nullable=False),
+    sa.Column('ww_processing_num', sa.String(length=64), nullable=True),
+    sa.Column('ww_full_sample_id', sa.String(length=64), nullable=True),
+    sa.Column('rsl_number', sa.String(length=64), nullable=True),
+    sa.Column('collection_date', sa.TIMESTAMP(), nullable=True),
+    sa.Column('received_date', sa.TIMESTAMP(), nullable=True),
+    sa.Column('notes', sa.String(length=2000), nullable=True),
+    sa.Column('sample_location', sa.String(length=8), nullable=True),
+    sa.ForeignKeyConstraint(['id'], ['_basicsample.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('_basicsubmission',
+    sa.Column('id', sa.INTEGER(), nullable=False),
+    sa.Column('rsl_plate_num', sa.String(length=32), nullable=False),
+    sa.Column('submitter_plate_num', sa.String(length=127), nullable=True),
+    sa.Column('submitted_date', sa.TIMESTAMP(), nullable=True),
+    sa.Column('submitting_lab_id', sa.INTEGER(), nullable=True),
+    sa.Column('sample_count', sa.INTEGER(), nullable=True),
+    sa.Column('extraction_kit_id', sa.INTEGER(), nullable=True),
+    sa.Column('submission_type_name', sa.String(), nullable=True),
+    sa.Column('technician', sa.String(length=64), nullable=True),
+    sa.Column('reagents_id', sa.String(), nullable=True),
+    sa.Column('extraction_info', sa.JSON(), nullable=True),
+    sa.Column('pcr_info', sa.JSON(), nullable=True),
+    sa.Column('run_cost', sa.FLOAT(precision=2), nullable=True),
+    sa.Column('uploaded_by', sa.String(length=32), nullable=True),
+    sa.Column('comment', sa.JSON(), nullable=True),
+    sa.Column('submission_category', sa.String(length=64), nullable=True),
+    sa.ForeignKeyConstraint(['extraction_kit_id'], ['_kittype.id'], name='fk_BS_extkit_id', ondelete='SET NULL'),
+    sa.ForeignKeyConstraint(['reagents_id'], ['_reagent.id'], name='fk_BS_reagents_id', ondelete='SET NULL'),
+    sa.ForeignKeyConstraint(['submission_type_name'], ['_submissiontype.name'], name='fk_BS_subtype_name', ondelete='SET NULL'),
+    sa.ForeignKeyConstraint(['submitting_lab_id'], ['_organization.id'], name='fk_BS_sublab_id', ondelete='SET NULL'),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('rsl_plate_num'),
+    sa.UniqueConstraint('submitter_plate_num')
+    )
+    op.create_table('_reagenttypes_reagents',
+    sa.Column('reagent_id', sa.INTEGER(), nullable=True),
+    sa.Column('reagenttype_id', sa.INTEGER(), nullable=True),
+    sa.ForeignKeyConstraint(['reagent_id'], ['_reagent.id'], ),
+    sa.ForeignKeyConstraint(['reagenttype_id'], ['_reagenttype.id'], )
+    )
+    op.create_table('_bacterialculture',
+    sa.Column('id', sa.INTEGER(), nullable=False),
+    sa.ForeignKeyConstraint(['id'], ['_basicsubmission.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('_control',
+    sa.Column('id', sa.INTEGER(), nullable=False),
+    sa.Column('parent_id', sa.String(), nullable=True),
+    sa.Column('name', sa.String(length=255), nullable=True),
+    sa.Column('submitted_date', sa.TIMESTAMP(), nullable=True),
+    sa.Column('contains', sa.JSON(), nullable=True),
+    sa.Column('matches', sa.JSON(), nullable=True),
+    sa.Column('kraken', sa.JSON(), nullable=True),
+    sa.Column('submission_id', sa.INTEGER(), nullable=True),
+    sa.Column('refseq_version', sa.String(length=16), nullable=True),
+    sa.Column('kraken2_version', sa.String(length=16), nullable=True),
+    sa.Column('kraken2_db_version', sa.String(length=32), nullable=True),
+    sa.Column('sample_id', sa.INTEGER(), nullable=True),
+    sa.ForeignKeyConstraint(['parent_id'], ['_controltype.id'], name='fk_control_parent_id'),
+    sa.ForeignKeyConstraint(['sample_id'], ['_basicsample.id'], name='cont_BCS_id', ondelete='SET NULL'),
+    sa.ForeignKeyConstraint(['submission_id'], ['_basicsubmission.id'], ),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('name')
+    )
+    op.create_table('_submissionequipmentassociation',
+    sa.Column('equipment_id', sa.INTEGER(), nullable=False),
+    sa.Column('submission_id', sa.INTEGER(), nullable=False),
+    sa.Column('role', sa.String(length=64), nullable=False),
+    sa.Column('process_id', sa.INTEGER(), nullable=True),
+    sa.Column('start_time', sa.TIMESTAMP(), nullable=True),
+    sa.Column('end_time', sa.TIMESTAMP(), nullable=True),
+    sa.Column('comments', sa.String(length=1024), nullable=True),
+    sa.ForeignKeyConstraint(['equipment_id'], ['_equipment.id'], ),
+    sa.ForeignKeyConstraint(['process_id'], ['_process.id'], name='SEA_Process_id', ondelete='SET NULL'),
+    sa.ForeignKeyConstraint(['submission_id'], ['_basicsubmission.id'], ),
+    sa.PrimaryKeyConstraint('equipment_id', 'submission_id', 'role')
+    )
+    op.create_table('_submissionreagentassociation',
+    sa.Column('reagent_id', sa.INTEGER(), nullable=False),
+    sa.Column('submission_id', sa.INTEGER(), nullable=False),
+    sa.Column('comments', sa.String(length=1024), nullable=True),
+    sa.ForeignKeyConstraint(['reagent_id'], ['_reagent.id'], ),
+    sa.ForeignKeyConstraint(['submission_id'], ['_basicsubmission.id'], ),
+    sa.PrimaryKeyConstraint('reagent_id', 'submission_id')
+    )
+    op.create_table('_submissionsampleassociation',
+    sa.Column('sample_id', sa.INTEGER(), nullable=False),
+    sa.Column('submission_id', sa.INTEGER(), nullable=False),
+    sa.Column('row', sa.INTEGER(), nullable=False),
+    sa.Column('column', sa.INTEGER(), nullable=False),
+    sa.Column('base_sub_type', sa.String(), nullable=True),
+    sa.ForeignKeyConstraint(['sample_id'], ['_basicsample.id'], ),
+    sa.ForeignKeyConstraint(['submission_id'], ['_basicsubmission.id'], ),
+    sa.PrimaryKeyConstraint('submission_id', 'row', 'column')
+    )
+    op.create_table('_wastewater',
+    sa.Column('id', sa.INTEGER(), nullable=False),
+    sa.Column('ext_technician', sa.String(length=64), nullable=True),
+    sa.Column('pcr_technician', sa.String(length=64), nullable=True),
+    sa.ForeignKeyConstraint(['id'], ['_basicsubmission.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('_wastewaterartic',
+    sa.Column('id', sa.INTEGER(), nullable=False),
+    sa.Column('artic_technician', sa.String(length=64), nullable=True),
+    sa.Column('dna_core_submission_number', sa.String(length=64), nullable=True),
+    sa.ForeignKeyConstraint(['id'], ['_basicsubmission.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('_wastewaterassociation',
+    sa.Column('sample_id', sa.INTEGER(), nullable=False),
+    sa.Column('submission_id', sa.INTEGER(), nullable=False),
+    sa.Column('ct_n1', sa.FLOAT(precision=2), nullable=True),
+    sa.Column('ct_n2', sa.FLOAT(precision=2), nullable=True),
+    sa.Column('n1_status', sa.String(length=32), nullable=True),
+    sa.Column('n2_status', sa.String(length=32), nullable=True),
+    sa.Column('pcr_results', sa.JSON(), nullable=True),
+    sa.ForeignKeyConstraint(['sample_id'], ['_submissionsampleassociation.sample_id'], ),
+    sa.ForeignKeyConstraint(['submission_id'], ['_submissionsampleassociation.submission_id'], ),
+    sa.PrimaryKeyConstraint('sample_id', 'submission_id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('_wastewaterassociation')
+    op.drop_table('_wastewaterartic')
+    op.drop_table('_wastewater')
+    op.drop_table('_submissionsampleassociation')
+    op.drop_table('_submissionreagentassociation')
+    op.drop_table('_submissionequipmentassociation')
+    op.drop_table('_control')
+    op.drop_table('_bacterialculture')
+    op.drop_table('_reagenttypes_reagents')
+    op.drop_table('_basicsubmission')
+    op.drop_table('_wastewatersample')
+    op.drop_table('_submissiontypes_processes')
+    op.drop_table('_submissiontypekittypeassociation')
+    op.drop_table('_submissiontypeequipmentroleassociation')
+    op.drop_table('_reagent')
+    op.drop_table('_orgs_contacts')
+    op.drop_table('_kittypes_processes')
+    op.drop_table('_kittypereagenttypeassociation')
+    op.drop_table('_equipmentroles_processes')
+    op.drop_table('_equipmentroles_equipment')
+    op.drop_table('_equipment_processes')
+    op.drop_table('_discount')
+    op.drop_table('_bacterialculturesample')
+    op.drop_table('_submissiontype')
+    op.drop_table('_reagenttype')
+    op.drop_table('_process')
+    op.drop_table('_organization')
+    op.drop_table('_kittype')
+    op.drop_table('_equipmentrole')
+    op.drop_table('_equipment')
+    op.drop_table('_controltype')
+    op.drop_table('_contact')
+    op.drop_table('_basicsample')
+    # ### end Alembic commands ###

src/submissions/__init__.py

@@ -4,7 +4,7 @@ from pathlib import Path
 
 # Version of the realpython-reader package
 __project__ = "submissions"
-__version__ = "202401.2b"
+__version__ = "202401.4b"
 __author__ = {"name":"Landon Wark", "email":"Landon.Wark@phac-aspc.gc.ca"}
 __copyright__ = "2022-2024, Government of Canada"
 

src/submissions/backend/db/models/__init__.py

@@ -50,7 +50,7 @@ class BaseClass(Base):
         return ctx.backup_path
     
     def save(self):
-        logger.debug(f"Saving {self}")
+        # logger.debug(f"Saving {self}")
         try:
             self.__database_session__.add(self)
             self.__database_session__.commit()

src/submissions/backend/db/models/controls.py

@@ -78,20 +78,20 @@ class Control(BaseClass):
     # __tablename__ = '_control_samples'
     
     id = Column(INTEGER, primary_key=True) #: primary key
-    parent_id = Column(String, ForeignKey("_control_types.id", name="fk_control_parent_id")) #: primary key of control type
+    parent_id = Column(String, ForeignKey("_controltype.id", name="fk_control_parent_id")) #: primary key of control type
     controltype = relationship("ControlType", back_populates="instances", foreign_keys=[parent_id]) #: reference to parent control type
     name = Column(String(255), unique=True) #: Sample ID
     submitted_date = Column(TIMESTAMP) #: Date submitted to Robotics
     contains = Column(JSON) #: unstructured hashes in contains.tsv for each organism
     matches = Column(JSON) #: unstructured hashes in matches.tsv for each organism
     kraken = Column(JSON) #: unstructured output from kraken_report
-    submission_id = Column(INTEGER, ForeignKey("_submissions.id")) #: parent submission id
+    submission_id = Column(INTEGER, ForeignKey("_basicsubmission.id")) #: parent submission id
     submission = relationship("BacterialCulture", back_populates="controls", foreign_keys=[submission_id]) #: parent submission
     refseq_version = Column(String(16)) #: version of refseq used in fastq parsing
     kraken2_version = Column(String(16)) #: version of kraken2 used in fastq parsing
     kraken2_db_version = Column(String(32)) #: folder name of kraken2 db
     sample = relationship("BacterialCultureSample", back_populates="control")
-    sample_id = Column(INTEGER, ForeignKey("_samples.id", ondelete="SET NULL", name="cont_BCS_id"))
+    sample_id = Column(INTEGER, ForeignKey("_basicsample.id", ondelete="SET NULL", name="cont_BCS_id"))
 
     def __repr__(self) -> str:
         return f"<Control({self.name})>"

src/submissions/backend/db/models/kits.py

@@ -18,8 +18,8 @@ logger = logging.getLogger(f'submissions.{__name__}')
 reagenttypes_reagents = Table(
                                 "_reagenttypes_reagents", 
                                 Base.metadata, 
-                                Column("reagent_id", INTEGER, ForeignKey("_reagents.id")), 
+                                Column("reagent_id", INTEGER, ForeignKey("_reagent.id")), 
-                                Column("reagenttype_id", INTEGER, ForeignKey("_reagent_types.id")),
+                                Column("reagenttype_id", INTEGER, ForeignKey("_reagenttype.id")),
                                 extend_existing = True
                                 )
 
@@ -27,7 +27,7 @@ equipmentroles_equipment = Table(
     "_equipmentroles_equipment",
     Base.metadata,
     Column("equipment_id", INTEGER, ForeignKey("_equipment.id")),
-    Column("equipmentroles_id", INTEGER, ForeignKey("_equipment_roles.id")),
+    Column("equipmentroles_id", INTEGER, ForeignKey("_equipmentrole.id")),
     extend_existing=True
 )
 
@@ -43,7 +43,7 @@ equipmentroles_processes = Table(
     "_equipmentroles_processes",
     Base.metadata,
     Column("process_id", INTEGER, ForeignKey("_process.id")),
-    Column("equipmentrole_id", INTEGER, ForeignKey("_equipment_roles.id")),
+    Column("equipmentrole_id", INTEGER, ForeignKey("_equipmentrole.id")),
     extend_existing=True
 )
 
@@ -51,7 +51,7 @@ submissiontypes_processes = Table(
     "_submissiontypes_processes",
     Base.metadata,
     Column("process_id", INTEGER, ForeignKey("_process.id")),
-    Column("equipmentroles_id", INTEGER, ForeignKey("_submission_types.id")),
+    Column("equipmentroles_id", INTEGER, ForeignKey("_submissiontype.id")),
     extend_existing=True
 )
 
@@ -59,7 +59,7 @@ kittypes_processes = Table(
     "_kittypes_processes",
     Base.metadata,
     Column("process_id", INTEGER, ForeignKey("_process.id")),
-    Column("kit_id", INTEGER, ForeignKey("_kits.id")),
+    Column("kit_id", INTEGER, ForeignKey("_kittype.id")),
     extend_existing=True
 )
 
@@ -304,7 +304,7 @@ class Reagent(BaseClass):
 
     id = Column(INTEGER, primary_key=True) #: primary key
     type = relationship("ReagentType", back_populates="instances", secondary=reagenttypes_reagents) #: joined parent reagent type
-    type_id = Column(INTEGER, ForeignKey("_reagent_types.id", ondelete='SET NULL', name="fk_reagent_type_id")) #: id of parent reagent type
+    type_id = Column(INTEGER, ForeignKey("_reagenttype.id", ondelete='SET NULL', name="fk_reagent_type_id")) #: id of parent reagent type
     name = Column(String(64)) #: reagent name
     lot = Column(String(64)) #: lot number of reagent
     expiry = Column(TIMESTAMP) #: expiry date - extended by eol_ext of parent programmatically
@@ -442,9 +442,9 @@ class Discount(BaseClass):
 
     id = Column(INTEGER, primary_key=True) #: primary key
     kit = relationship("KitType") #: joined parent reagent type
-    kit_id = Column(INTEGER, ForeignKey("_kits.id", ondelete='SET NULL', name="fk_kit_type_id")) #: id of joined kit
+    kit_id = Column(INTEGER, ForeignKey("_kittype.id", ondelete='SET NULL', name="fk_kit_type_id")) #: id of joined kit
     client = relationship("Organization") #: joined client lab
-    client_id = Column(INTEGER, ForeignKey("_organizations.id", ondelete='SET NULL', name="fk_org_id")) #: id of joined client
+    client_id = Column(INTEGER, ForeignKey("_organization.id", ondelete='SET NULL', name="fk_org_id")) #: id of joined client
     name = Column(String(128)) #: Short description 
     amount = Column(FLOAT(2)) #: Dollar amount of discount
 
@@ -625,8 +625,9 @@ class SubmissionType(BaseClass):
         """
         Adds this instances to the database and commits.
         """        
-        self.__database_session__.add(self)
+        # self.__database_session__.add(self)
-        self.__database_session__.commit()
+        # self.__database_session__.commit()
+        super().save()
   
 class SubmissionTypeKitTypeAssociation(BaseClass):
     """
@@ -634,8 +635,8 @@ class SubmissionTypeKitTypeAssociation(BaseClass):
     """    
     # __tablename__ = "_submissiontypes_kittypes"
 
-    submission_types_id = Column(INTEGER, ForeignKey("_submission_types.id"), primary_key=True) #: id of joined submission type
+    submission_types_id = Column(INTEGER, ForeignKey("_submissiontype.id"), primary_key=True) #: id of joined submission type
-    kits_id = Column(INTEGER, ForeignKey("_kits.id"), primary_key=True) #: id of joined kit
+    kits_id = Column(INTEGER, ForeignKey("_kittype.id"), primary_key=True) #: id of joined kit
     mutable_cost_column = Column(FLOAT(2)) #: dollar amount per 96 well plate that can change with number of columns (reagents, tips, etc)
     mutable_cost_sample = Column(FLOAT(2)) #: dollar amount that can change with number of samples (reagents, tips, etc)
     constant_cost = Column(FLOAT(2)) #: dollar amount per plate that will remain constant (plates, man hours, etc)
@@ -707,9 +708,9 @@ class KitTypeReagentTypeAssociation(BaseClass):
     """    
     # __tablename__ = "_reagenttypes_kittypes"
 
-    reagent_types_id = Column(INTEGER, ForeignKey("_reagent_types.id"), primary_key=True) #: id of associated reagent type
+    reagent_types_id = Column(INTEGER, ForeignKey("_reagenttype.id"), primary_key=True) #: id of associated reagent type
-    kits_id = Column(INTEGER, ForeignKey("_kits.id"), primary_key=True) #: id of associated reagent type
+    kits_id = Column(INTEGER, ForeignKey("_kittype.id"), primary_key=True) #: id of associated reagent type
-    submission_type_id = Column(INTEGER, ForeignKey("_submission_types.id"), primary_key=True)
+    submission_type_id = Column(INTEGER, ForeignKey("_submissiontype.id"), primary_key=True)
     uses = Column(JSON) #: map to location on excel sheets of different submission types
     required = Column(INTEGER) #: whether the reagent type is required for the kit (Boolean 1 or 0)
     last_used = Column(String(32)) #: last used lot number of this type of reagent
@@ -810,8 +811,8 @@ class SubmissionReagentAssociation(BaseClass):
 
     # __tablename__ = "_reagents_submissions"
 
-    reagent_id = Column(INTEGER, ForeignKey("_reagents.id"), primary_key=True) #: id of associated sample
+    reagent_id = Column(INTEGER, ForeignKey("_reagent.id"), primary_key=True) #: id of associated sample
-    submission_id = Column(INTEGER, ForeignKey("_submissions.id"), primary_key=True)
+    submission_id = Column(INTEGER, ForeignKey("_basicsubmission.id"), primary_key=True)
     comments = Column(String(1024))
 
     submission = relationship("BasicSubmission", back_populates="submission_reagent_associations") #: associated submission
@@ -1060,7 +1061,7 @@ class SubmissionEquipmentAssociation(BaseClass):
     # __tablename__ = "_equipment_submissions"
 
     equipment_id = Column(INTEGER, ForeignKey("_equipment.id"), primary_key=True) #: id of associated equipment
-    submission_id = Column(INTEGER, ForeignKey("_submissions.id"), primary_key=True) #: id of associated submission
+    submission_id = Column(INTEGER, ForeignKey("_basicsubmission.id"), primary_key=True) #: id of associated submission
     role = Column(String(64), primary_key=True) #: name of the role the equipment fills
     # process = Column(String(64)) #: name of the process run on this equipment
     process_id = Column(INTEGER, ForeignKey("_process.id",ondelete="SET NULL", name="SEA_Process_id"))
@@ -1090,8 +1091,8 @@ class SubmissionTypeEquipmentRoleAssociation(BaseClass):
 
     # __tablename__ = "_submissiontype_equipmentrole"
 
-    equipmentrole_id = Column(INTEGER, ForeignKey("_equipment_roles.id"), primary_key=True) #: id of associated equipment
+    equipmentrole_id = Column(INTEGER, ForeignKey("_equipmentrole.id"), primary_key=True) #: id of associated equipment
-    submissiontype_id = Column(INTEGER, ForeignKey("_submission_types.id"), primary_key=True) #: id of associated submission
+    submissiontype_id = Column(INTEGER, ForeignKey("_submissiontype.id"), primary_key=True) #: id of associated submission
     uses = Column(JSON) #: locations of equipment on the submission type excel sheet.
     static = Column(INTEGER, default=1) #: if 1 this piece of equipment will always be used, otherwise it will need to be selected from list?
 
@@ -1156,7 +1157,7 @@ class Process(BaseClass):
     
     @classmethod
     @setup_lookup
-    def query(cls, name:str|None, limit:int=0):
+    def query(cls, name:str|None=None, limit:int=0):
         query = cls.__database_session__.query(cls)
         match name:
             case str():

src/submissions/backend/db/models/organizations.py

@@ -15,8 +15,8 @@ logger = logging.getLogger(f"submissions.{__name__}")
 orgs_contacts = Table(
                         "_orgs_contacts", 
                         Base.metadata, 
-                        Column("org_id", INTEGER, ForeignKey("_organizations.id")), 
+                        Column("org_id", INTEGER, ForeignKey("_organization.id")), 
-                        Column("contact_id", INTEGER, ForeignKey("_contacts.id")), 
+                        Column("contact_id", INTEGER, ForeignKey("_contact.id")), 
                         # __table_args__ = {'extend_existing': True} 
                         extend_existing = True
                     )

src/submissions/backend/db/models/submissions.py

@@ -3,7 +3,8 @@ Models for the main submission types.
 '''
 from __future__ import annotations
 from getpass import getuser
-import math, json, logging, uuid, tempfile, re, yaml
+import math, json, logging, uuid, tempfile, re, yaml, zipfile
+import sys
 from operator import attrgetter
 from pprint import pformat
 from . import Reagent, SubmissionType, KitType, Organization
@@ -13,9 +14,10 @@ from json.decoder import JSONDecodeError
 from sqlalchemy.ext.associationproxy import association_proxy
 import pandas as pd
 from openpyxl import Workbook
-from . import BaseClass, Equipment
+from openpyxl.worksheet.worksheet import Worksheet
+from . import BaseClass
 from tools import check_not_nan, row_map, query_return, setup_lookup, jinja_template_loading
-from datetime import datetime, date, time
+from datetime import datetime, date
 from typing import List, Any
 from dateutil.parser import parse
 from dateutil.parser._parser import ParserError
@@ -37,17 +39,16 @@ class BasicSubmission(BaseClass):
     submitter_plate_num = Column(String(127), unique=True) #: The number given to the submission by the submitting lab
     submitted_date = Column(TIMESTAMP) #: Date submission received
     submitting_lab = relationship("Organization", back_populates="submissions") #: client org
-    submitting_lab_id = Column(INTEGER, ForeignKey("_organizations.id", ondelete="SET NULL", name="fk_BS_sublab_id")) #: client lab id from _organizations
+    submitting_lab_id = Column(INTEGER, ForeignKey("_organization.id", ondelete="SET NULL", name="fk_BS_sublab_id")) #: client lab id from _organizations
     sample_count = Column(INTEGER) #: Number of samples in the submission
     extraction_kit = relationship("KitType", back_populates="submissions") #: The extraction kit used
-    extraction_kit_id = Column(INTEGER, ForeignKey("_kits.id", ondelete="SET NULL", name="fk_BS_extkit_id")) #: id of joined extraction kit
+    extraction_kit_id = Column(INTEGER, ForeignKey("_kittype.id", ondelete="SET NULL", name="fk_BS_extkit_id")) #: id of joined extraction kit
-    submission_type_name = Column(String, ForeignKey("_submission_types.name", ondelete="SET NULL", name="fk_BS_subtype_name")) #: name of joined submission type
+    submission_type_name = Column(String, ForeignKey("_submissiontype.name", ondelete="SET NULL", name="fk_BS_subtype_name")) #: name of joined submission type
     technician = Column(String(64)) #: initials of processing tech(s)
     # Move this into custom types?
     # reagents = relationship("Reagent", back_populates="submissions", secondary=reagents_submissions) #: relationship to reagents
-    reagents_id = Column(String, ForeignKey("_reagents.id", ondelete="SET NULL", name="fk_BS_reagents_id")) #: id of used reagents
+    reagents_id = Column(String, ForeignKey("_reagent.id", ondelete="SET NULL", name="fk_BS_reagents_id")) #: id of used reagents
     extraction_info = Column(JSON) #: unstructured output from the extraction table logger.
-    pcr_info = Column(JSON) #: unstructured output from pcr table logger or user(Artic)
     run_cost = Column(FLOAT(2)) #: total cost of running the plate. Set from constant and mutable kit costs at time of creation.
     uploaded_by = Column(String(32)) #: user name of person who submitted the submission to the database.
     comment = Column(JSON) #: user notes
@@ -132,19 +133,21 @@ class BasicSubmission(BaseClass):
             logger.error(f"Json error in {self.rsl_plate_num}: {e}")
         # Updated 2023-09 to use the extraction kit to pull reagents.
         if full_data:
+            logger.debug(f"Attempting reagents.")
             try:
                 reagents = [item.to_sub_dict(extraction_kit=self.extraction_kit) for item in self.submission_reagent_associations]
             except Exception as e:
                 logger.error(f"We got an error retrieving reagents: {e}")
                 reagents = None
             # samples = [item.sample.to_sub_dict(submission_rsl=self.rsl_plate_num) for item in self.submission_sample_associations]
+            logger.debug(f"Running samples.")
             samples = self.adjust_to_dict_samples(backup=backup)
             try:
                 equipment = [item.to_sub_dict() for item in self.submission_equipment_associations]
                 if len(equipment) == 0:
                     equipment = None
             except Exception as e:
-                logger.error(f"Error setting equipment: {self.equipment}")
+                logger.error(f"Error setting equipment: {e}")
                 equipment = None
         else:
             reagents = None
@@ -155,7 +158,6 @@ class BasicSubmission(BaseClass):
         except Exception as e:
             logger.error(f"Error setting comment: {self.comment}")
             comments = None
-        
         output = {
             "id": self.id,
             "Plate Number": self.rsl_plate_num,
@@ -440,6 +442,7 @@ class BasicSubmission(BaseClass):
     def filename_template(cls) -> str:
         """
         Constructs template for filename of this class.
+        Note: This is meant to be used with the dictionary constructed in self.to_dict(). Keys need to have spaces removed
 
         Returns:
             str: filename template in jinja friendly format.
@@ -462,6 +465,20 @@ class BasicSubmission(BaseClass):
                 logger.warning(f"Couldn't drop '{item}' column from submissionsheet df.")
         return df
 
+    @classmethod
+    def custom_sample_autofill_row(cls, sample, worksheet:Worksheet) -> int:
+        """
+        _summary_
+
+        Args:
+            sample (_type_): _description_
+            worksheet (Workbook): _description_
+
+        Returns:
+            int: _description_
+        """      
+        return None  
+
     def set_attribute(self, key:str, value):
         """
         Performs custom attribute setting based on values.
@@ -547,7 +564,7 @@ class BasicSubmission(BaseClass):
         """        
         from backend.validators import PydSubmission, PydSample, PydReagent, PydEquipment
         dicto = self.to_dict(full_data=True, backup=backup)
-        logger.debug(f"Backup dictionary: {pformat(dicto)}")
+        # logger.debug(f"Backup dictionary: {pformat(dicto)}")
         # dicto['filepath'] = Path(tempfile.TemporaryFile().name)
         new_dict = {}
         for key, value in dicto.items():
@@ -572,7 +589,7 @@ class BasicSubmission(BaseClass):
                     # new_dict[key.lower().replace(" ", "_")]['value'] = value
                     # new_dict[key.lower().replace(" ", "_")]['missing'] = True
         new_dict['filepath'] = Path(tempfile.TemporaryFile().name)
-        logger.debug(f"Dictionary coming into PydSubmission: {pformat(new_dict)}")
+        # logger.debug(f"Dictionary coming into PydSubmission: {pformat(new_dict)}")
         # sys.exit()
         return PydSubmission(**new_dict)
 
@@ -797,22 +814,15 @@ class BasicSubmission(BaseClass):
             # logger.debug(f"Save result: {result}")
 
     def add_equipment(self, obj):
-        # submission_type = submission.submission_type_name
         from frontend.widgets.equipment_usage import EquipmentUsage
-        dlg = EquipmentUsage(parent=obj, submission_type=self.submission_type_name, submission=self)
+        dlg = EquipmentUsage(parent=obj, submission=self)
         if dlg.exec():
             equipment = dlg.parse_form()
             logger.debug(f"We've got equipment: {equipment}")
             for equip in equipment:
-                # e = Equipment.query(name=equip.name)
+                logger.debug(f"Processing: {equip}")
-                # assoc = SubmissionEquipmentAssociation(submission=submission, equipment=e)
-                # process = Process.query(name=equip.processes)
-                # assoc.process = process
-                # assoc.role = equip.role
                 _, assoc = equip.toSQL(submission=self)
-                # submission.submission_equipment_associations.append(assoc)
                 logger.debug(f"Appending SubmissionEquipmentAssociation: {assoc}")
-                # submission.save()
                 assoc.save()
         else:
             pass
@@ -825,12 +835,14 @@ class BasicSubmission(BaseClass):
             fname (Path): Filename of xlsx file.
         """        
         logger.debug("Hello from backup.")
+        pyd = self.to_pydantic(backup=True)
         if fname == None:
             from frontend.widgets.functions import select_save_file
-            from backend.validators import RSLNamer
+            fname = select_save_file(default_name=pyd.construct_filename(), extension="xlsx", obj=obj)
-            abbreviation = self.get_abbreviation()
+        logger.debug(fname.name)
-            file_data = dict(rsl_plate_num=self.rsl_plate_num, submission_type=self.submission_type_name, submitted_date=self.submitted_date, abbreviation=abbreviation)
+        if fname.name == "":
-            fname = select_save_file(default_name=RSLNamer.construct_new_plate_name(data=file_data), extension="xlsx", obj=obj)
+            logger.debug(f"export cancelled.")
+            return
         if full_backup:
             backup = self.to_dict(full_data=True)
             try:
@@ -838,7 +850,6 @@ class BasicSubmission(BaseClass):
                     yaml.dump(backup, f)
             except KeyError as e:
                 logger.error(f"Problem saving yml backup file: {e}")
-        pyd = self.to_pydantic(backup=True)
         wb = pyd.autofill_excel()
         wb = pyd.autofill_samples(wb)
         wb = pyd.autofill_equipment(wb)
@@ -856,14 +867,14 @@ class BacterialCulture(BasicSubmission):
                            polymorphic_load="inline", 
                            inherit_condition=(id == BasicSubmission.id))
 
-    def to_dict(self, full_data:bool=False) -> dict:
+    def to_dict(self, full_data:bool=False, backup:bool=False) -> dict:
         """
         Extends parent class method to add controls to dict
 
         Returns:
             dict: dictionary used in submissions summary
         """        
-        output = super().to_dict(full_data=full_data)
+        output = super().to_dict(full_data=full_data, backup=backup)
         if full_data:
             output['controls'] = [item.to_sub_dict() for item in self.controls]
         return output
@@ -996,6 +1007,22 @@ class BacterialCulture(BasicSubmission):
         input_dict['submitted_date']['missing'] = True
         return input_dict
 
+    @classmethod
+    def custom_sample_autofill_row(cls, sample, worksheet: Worksheet) -> int:
+        logger.debug(f"Checking {sample.well}")
+        logger.debug(f"here's the worksheet: {worksheet}")
+        row = super().custom_sample_autofill_row(sample, worksheet)
+        df = pd.DataFrame(list(worksheet.values))
+        # logger.debug(f"Here's the dataframe: {df}")
+        idx = df[df[0]==sample.well]
+        if idx.empty:
+            new = f"{sample.well[0]}{sample.well[1:].zfill(2)}"
+            logger.debug(f"Checking: {new}")
+            idx = df[df[0]==new]
+        logger.debug(f"Here is the row: {idx}")
+        row = idx.index.to_list()[0]
+        return row + 1
+
 class Wastewater(BasicSubmission):
     """
     derivative submission type from BasicSubmission
@@ -1003,11 +1030,13 @@ class Wastewater(BasicSubmission):
     id = Column(INTEGER, ForeignKey('_basicsubmission.id'), primary_key=True)
     ext_technician = Column(String(64))
     pcr_technician = Column(String(64))
+    pcr_info = Column(JSON) #: unstructured output from pcr table logger or user(Artic)
+
     __mapper_args__ = __mapper_args__ = dict(polymorphic_identity="Wastewater", 
                            polymorphic_load="inline", 
                            inherit_condition=(id == BasicSubmission.id))
 
-    def to_dict(self, full_data:bool=False) -> dict:
+    def to_dict(self, full_data:bool=False, backup:bool=False) -> dict:
         """
         Extends parent class method to add controls to dict
 
@@ -1020,6 +1049,7 @@ class Wastewater(BasicSubmission):
         except TypeError as e:
             pass
         output['Technician'] = f"Enr: {self.technician}, Ext: {self.ext_technician}, PCR: {self.pcr_technician}"
+        
         return output
     
     @classmethod
@@ -1144,6 +1174,18 @@ class Wastewater(BasicSubmission):
     def adjust_autofill_samples(cls, samples: List[Any]) -> List[Any]:
         samples = super().adjust_autofill_samples(samples)
         return [item for item in samples if not item.submitter_id.startswith("EN")]
+    
+    @classmethod
+    def custom_sample_autofill_row(cls, sample, worksheet: Worksheet) -> int:
+        logger.debug(f"Checking {sample.well}")
+        logger.debug(f"here's the worksheet: {worksheet}")
+        row = super().custom_sample_autofill_row(sample, worksheet)
+        df = pd.DataFrame(list(worksheet.values))
+        logger.debug(f"Here's the dataframe: {df}")
+        idx = df[df[1]==sample.sample_location]
+        logger.debug(f"Here is the row: {idx}")
+        row = idx.index.to_list()[0]
+        return row + 1
 
 class WastewaterArtic(BasicSubmission):
     """
@@ -1155,6 +1197,9 @@ class WastewaterArtic(BasicSubmission):
                            inherit_condition=(id == BasicSubmission.id))
     artic_technician = Column(String(64))
     dna_core_submission_number = Column(String(64))
+    pcr_info = Column(JSON) #: unstructured output from pcr table logger or user(Artic)
+    gel_image = Column(String(64))
+    gel_info = Column(JSON)
 
     def calculate_base_cost(self):
         """
@@ -1381,10 +1426,19 @@ class WastewaterArtic(BasicSubmission):
     
     def gel_box(self, obj):
         from frontend.widgets.gel_checker import GelBox
-        dlg = GelBox(parent=obj)
+        from frontend.widgets import select_open_file
+        fname = select_open_file(obj=obj, file_extension="jpg")
+        dlg = GelBox(parent=obj, img_path=fname)
         if dlg.exec():
-            output = dlg.parse_form()
+            img_path, output = dlg.parse_form()
-        print(output)
+            self.gel_image = img_path.name
+            self.gel_info = output
+            with zipfile.ZipFile(self.__directory_path__.joinpath("submission_imgs.zip"), 'a') as zipf:
+                # Add a file located at the source_path to the destination within the zip
+                # file. It will overwrite existing files if the names collide, but it
+                # will give a warning
+                zipf.write(img_path, self.gel_image)
+            self.save()
 
 # Sample Classes
 
@@ -1439,7 +1493,10 @@ class BasicSample(BaseClass):
             return value
         
     def __repr__(self) -> str:
-        return f"<{self.sample_type.replace('_', ' ').title().replace(' ', '')}({self.submitter_id})>"
+        try:
+            return f"<{self.sample_type.replace('_', ' ').title().replace(' ', '')}({self.submitter_id})>"
+        except AttributeError:
+            return f"<Sample({self.submitter_id})"
     
     def to_sub_dict(self, submission_rsl:str) -> dict:
         """
@@ -1448,6 +1505,7 @@ class BasicSample(BaseClass):
         Returns:
             dict: well location and name (sample id, organism) NOTE: keys must sync with WWSample to_sub_dict above
         """
+        # logger.debug(f"Converting {self} to dict.")
         sample = {}
         sample['submitter_id'] = self.submitter_id
         sample['sample_type'] = self.sample_type
@@ -1642,6 +1700,9 @@ class WastewaterSample(BasicSample):
         """
         sample = super().to_sub_dict(submission_rsl=submission_rsl)
         sample['ww_processing_num'] = self.ww_processing_num
+        sample['sample_location'] = self.sample_location
+        sample['received_date'] = self.received_date
+        sample['collection_date'] = self.collection_date
         return sample
         
     @classmethod
@@ -1721,9 +1782,10 @@ class SubmissionSampleAssociation(BaseClass):
     """    
 
     # __tablename__ = "_submission_sample"
-
+    
-    sample_id = Column(INTEGER, ForeignKey("_samples.id"), nullable=False) #: id of associated sample
+    id = Column(INTEGER, unique=True, nullable=False)
-    submission_id = Column(INTEGER, ForeignKey("_submissions.id"), primary_key=True) #: id of associated submission
+    sample_id = Column(INTEGER, ForeignKey("_basicsample.id"), nullable=False) #: id of associated sample
+    submission_id = Column(INTEGER, ForeignKey("_basicsubmission.id"), primary_key=True) #: id of associated submission
     row = Column(INTEGER, primary_key=True) #: row on the 96 well plate
     column = Column(INTEGER, primary_key=True) #: column on the 96 well plate
 
@@ -1743,14 +1805,23 @@ class SubmissionSampleAssociation(BaseClass):
         "with_polymorphic": "*",
     }
 
-    def __init__(self, submission:BasicSubmission=None, sample:BasicSample=None, row:int=1, column:int=1):
+    def __init__(self, submission:BasicSubmission=None, sample:BasicSample=None, row:int=1, column:int=1, id:int|None=None):
         self.submission = submission
         self.sample = sample
         self.row = row
         self.column = column
+        if id != None:
+            self.id = id
+        else:
+            self.id = self.__class__.autoincrement_id()
+        logger.debug(f"Using id: {self.id}")
 
     def __repr__(self) -> str:
-        return f"<SubmissionSampleAssociation({self.submission.rsl_plate_num} & {self.sample.submitter_id})"
+        try:
+            return f"<{self.__class__.__name__}({self.submission.rsl_plate_num} & {self.sample.submitter_id})"
+        except AttributeError as e:
+            logger.error(f"Unable to construct __repr__ due to: {e}")
+            return super().__repr__()
     
     def to_sub_dict(self) -> dict:
         """
@@ -1760,6 +1831,7 @@ class SubmissionSampleAssociation(BaseClass):
             dict: Updated dictionary with row, column and well updated
         """        
         # Get sample info
+        # logger.debug(f"Running {self.__repr__()}")
         sample = self.sample.to_sub_dict(submission_rsl=self.submission)
         # sample = {}
         sample['name'] = self.sample.submitter_id
@@ -1787,6 +1859,7 @@ class SubmissionSampleAssociation(BaseClass):
         # Since there is no PCR, negliable result is necessary.
         # assoc = [item for item in self.sample_submission_associations if item.submission.rsl_plate_num==submission_rsl][0]
         sample = self.to_sub_dict()
+        logger.debug(f"Sample dict to hitpick: {sample}")
         env = jinja_template_loading()
         template = env.get_template("tooltip.html")
         tooltip_text = template.render(fields=sample)
@@ -1801,6 +1874,14 @@ class SubmissionSampleAssociation(BaseClass):
         sample.update(dict(name=self.sample.submitter_id[:10], tooltip=tooltip_text))
         return sample
 
+    @classmethod
+    def autoincrement_id(cls):
+        try:
+            return max([item.id for item in cls.query()]) + 1
+        except ValueError as e:
+            logger.error(f"Problem incrementing id: {e}")
+            return 1
+
     @classmethod
     def find_polymorphic_subclass(cls, polymorphic_identity:str|None=None) -> SubmissionSampleAssociation:   
         """
@@ -1890,6 +1971,7 @@ class SubmissionSampleAssociation(BaseClass):
                 association_type:str="Basic Association", 
                 submission:BasicSubmission|str|None=None,
                 sample:BasicSample|str|None=None,
+                id:int|None=None,
                 **kwargs) -> SubmissionSampleAssociation:
         """
         Queries for an association, if none exists creates a new one.
@@ -1931,7 +2013,7 @@ class SubmissionSampleAssociation(BaseClass):
             instance = None
         if instance == None:
             used_cls = cls.find_polymorphic_subclass(polymorphic_identity=association_type)
-            instance = used_cls(submission=submission, sample=sample, **kwargs)
+            instance = used_cls(submission=submission, sample=sample, id=id, **kwargs)
         return instance
 
     def delete(self):
@@ -1941,8 +2023,8 @@ class WastewaterAssociation(SubmissionSampleAssociation):
     """
     Derivative custom Wastewater/Submission Association... fancy.
     """    
-    sample_id = Column(INTEGER, ForeignKey('_submissionsampleassociation.sample_id'), primary_key=True)
+    # sample_id = Column(INTEGER, ForeignKey('_submissionsampleassociation.sample_id'), primary_key=True)
-    submission_id = Column(INTEGER, ForeignKey('_submissionsampleassociation.submission_id'), primary_key=True)
+    id = Column(INTEGER, ForeignKey("_submissionsampleassociation.id"), primary_key=True)
     ct_n1 = Column(FLOAT(2)) #: AKA ct for N1
     ct_n2 = Column(FLOAT(2)) #: AKA ct for N2
     n1_status = Column(String(32)) #: positive or negative for N1
@@ -1952,7 +2034,11 @@ class WastewaterAssociation(SubmissionSampleAssociation):
     # __mapper_args__ = {"polymorphic_identity": "Wastewater Association", "polymorphic_load": "inline"}
     __mapper_args__ = dict(polymorphic_identity="Wastewater Association", 
                            polymorphic_load="inline", 
-                           inherit_condition=(sample_id == SubmissionSampleAssociation.sample_id))
+                        #    inherit_condition=(submission_id==SubmissionSampleAssociation.submission_id and 
+                        #                       row==SubmissionSampleAssociation.row and 
+                        #                       column==SubmissionSampleAssociation.column))
+                        inherit_condition=(id==SubmissionSampleAssociation.id))
+                        # inherit_foreign_keys=(sample_id == SubmissionSampleAssociation.sample_id, submission_id == SubmissionSampleAssociation.submission_id))
 
     def to_sub_dict(self) -> dict:
         sample = super().to_sub_dict()
@@ -1970,3 +2056,12 @@ class WastewaterAssociation(SubmissionSampleAssociation):
         except (TypeError, AttributeError) as e:
             logger.error(f"Couldn't set tooltip for {self.sample.rsl_number}. Looks like there isn't PCR data.")
         return sample
+
+    @classmethod
+    def autoincrement_id(cls):
+        try:
+            parent = [base for base in cls.__bases__ if base.__name__=="SubmissionSampleAssociation"][0]
+            return max([item.id for item in parent.query()]) + 1
+        except ValueError as e:
+            logger.error(f"Problem incrementing id: {e}")
+            return 1       

src/submissions/backend/validators/__init__.py

@@ -3,6 +3,7 @@ from pathlib import Path
 from openpyxl import load_workbook
 from backend.db.models import BasicSubmission, SubmissionType
 from datetime import date
+from tools import jinja_template_loading
 
 logger = logging.getLogger(f"submissions.{__name__}")
 
@@ -126,9 +127,20 @@ class RSLNamer(object):
                 today = parse(today.group())
             except AttributeError:
                 today = datetime.now()
-        previous = BasicSubmission.query(start_date=today, end_date=today, submission_type=data['submission_type'])
+        if "rsl_plate_num" in data.keys():
-        plate_number = len(previous) + 1
+            plate_number = data['rsl_plate_num'].split("-")[-1][0]
+        else:
+            previous = BasicSubmission.query(start_date=today, end_date=today, submission_type=data['submission_type'])
+            plate_number = len(previous) + 1
         return f"RSL-{data['abbreviation']}-{today.year}{str(today.month).zfill(2)}{str(today.day).zfill(2)}-{plate_number}"
+    
+    @classmethod
+    def construct_export_name(cls, template, **kwargs):
+        logger.debug(f"Kwargs: {kwargs}")
+        logger.debug(f"Template: {template}")
+        environment = jinja_template_loading()
+        template = environment.from_string(template)
+        return template.render(**kwargs)
 
         
 from .pydant import *

src/submissions/backend/validators/pydant.py

@@ -8,7 +8,7 @@ from pydantic import BaseModel, field_validator, Field
 from datetime import date, datetime, timedelta
 from dateutil.parser import parse
 from dateutil.parser._parser import ParserError
-from typing import List, Any, Tuple
+from typing import List, Tuple
 from . import RSLNamer
 from pathlib import Path
 from tools import check_not_nan, convert_nans_to_nones, jinja_template_loading, Report, Result, row_map
@@ -156,8 +156,9 @@ class PydSample(BaseModel, extra='allow'):
     sample_type: str
     row: int|List[int]|None
     column: int|List[int]|None
+    assoc_id: int|List[int]|None = Field(default=None)
 
-    @field_validator("row", "column")
+    @field_validator("row", "column", "assoc_id")
     @classmethod
     def row_int_to_list(cls, value):
         if isinstance(value, int):
@@ -193,14 +194,14 @@ class PydSample(BaseModel, extra='allow'):
         out_associations = []
         if submission != None:
             assoc_type = self.sample_type.replace("Sample", "").strip()
-            for row, column in zip(self.row, self.column):
+            for row, column, id in zip(self.row, self.column, self.assoc_id):
-                # logger.debug(f"Looking up association with identity: ({submission.submission_type_name} Association)")
+                logger.debug(f"Looking up association with identity: ({submission.submission_type_name} Association)")
                 logger.debug(f"Looking up association with identity: ({assoc_type} Association)")
                 association = SubmissionSampleAssociation.query_or_create(association_type=f"{assoc_type} Association", 
-                                                                          submission=submission, 
+                                                                            submission=submission, 
-                                                                          sample=instance, 
+                                                                            sample=instance, 
-                                                                          row=row, column=column)
+                                                                            row=row, column=column, id=id)
-                logger.debug(f"Using submission_sample_association: {association}")
+                # logger.debug(f"Using submission_sample_association: {association}")
                 try:
                     instance.sample_submission_associations.append(association)
                     out_associations.append(association)
@@ -254,7 +255,7 @@ class PydEquipment(BaseModel, extra='ignore'):
             assoc.process = process
             assoc.role = self.role
             # equipment.equipment_submission_associations.append(assoc)
-            equipment.equipment_submission_associations.append(assoc)
+            # equipment.equipment_submission_associations.append(assoc)
         else:
             assoc = None
         return equipment, assoc
@@ -275,7 +276,7 @@ class PydSubmission(BaseModel, extra='allow'):
     comment: dict|None = Field(default=dict(value="", missing=True), validate_default=True)
     reagents: List[dict]|List[PydReagent] = []
     samples: List[PydSample]
-    equipment: List[PydEquipment]|None
+    equipment: List[PydEquipment]|None =[]
 
     @field_validator('equipment', mode='before')
     @classmethod
@@ -421,6 +422,16 @@ class PydSubmission(BaseModel, extra='allow'):
             value['value'] = values.data['submission_type']['value']
         return value
 
+    @field_validator("samples")
+    def assign_ids(cls, value, values):
+        starting_id = SubmissionSampleAssociation.autoincrement_id()
+        output = []
+        for iii, sample in enumerate(value, start=starting_id):
+            sample.assoc_id = [iii]
+            output.append(sample)
+        return output
+
+
     def handle_duplicate_samples(self):
         """
         Collapses multiple samples with same submitter id into one with lists for rows, columns.
@@ -428,14 +439,19 @@ class PydSubmission(BaseModel, extra='allow'):
         """        
         submitter_ids = list(set([sample.submitter_id for sample in self.samples]))
         output = []
-        for id in submitter_ids:
+        for iii, id in enumerate(submitter_ids, start=1):
             relevants = [item for item in self.samples if item.submitter_id==id]
             if len(relevants) <= 1:
                 output += relevants
             else:
                 rows = [item.row[0] for item in relevants]
                 columns = [item.column[0] for item in relevants]
+                ids = [item.assoc_id[0] for item in relevants]
+                # for jjj, rel in enumerate(relevants, start=1):
+                #     starting_id += jjj  
+                #     ids.append(starting_id)                  
                 dummy = relevants[0]
+                dummy.assoc_id = ids
                 dummy.row = rows
                 dummy.column = columns
                 output.append(dummy)
@@ -663,14 +679,17 @@ class PydSubmission(BaseModel, extra='allow'):
         logger.debug(f"Workbook sheets: {workbook.sheetnames}")
         worksheet = workbook[sample_info["lookup_table"]['sheet']]
         samples = sorted(self.samples, key=attrgetter('column', 'row'))
-        custom_sampler = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=self.submission_type).adjust_autofill_samples
+        submission_obj = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=self.submission_type)
-        samples = custom_sampler(samples=samples)
+        samples = submission_obj.adjust_autofill_samples(samples=samples)
         logger.debug(f"Samples: {pformat(samples)}")
         # Fail safe against multiple instances of the same sample
         for iii, sample in enumerate(samples, start=1):
-            row = sample_info['lookup_table']['start_row'] + iii
+            logger.debug(f"Sample: {sample}")
+            row = submission_obj.custom_sample_autofill_row(sample, worksheet=worksheet)
+            logger.debug(f"Writing to {row}")
+            if row == None:
+                row = sample_info['lookup_table']['start_row'] + iii
             fields = [field for field in list(sample.model_fields.keys()) + list(sample.model_extra.keys()) if field in sample_info['sample_columns'].keys()]
-            
             for field in fields:
                 column = sample_info['sample_columns'][field]
                 value = getattr(sample, field)

src/submissions/frontend/widgets/gel_checker.py

@@ -1,7 +1,7 @@
 # import required modules
-from PyQt6.QtCore import Qt
+# from PyQt6.QtCore import Qt
 from PyQt6.QtWidgets import *
-import sys
+# import sys
 from PyQt6.QtWidgets import QWidget
 import numpy as np
 import pyqtgraph as pg
@@ -13,12 +13,13 @@ import numpy as np
 # Main window class
 class GelBox(QDialog):
 
-    def __init__(self, parent):
+    def __init__(self, parent, img_path):
         super().__init__(parent)
         # setting title
         self.setWindowTitle("PyQtGraph")
+        self.img_path = img_path
         # setting geometry
-        self.setGeometry(100, 100, 600, 500)
+        self.setGeometry(50, 50, 1200, 900)
         # icon
         icon = QIcon("skin.png")
         # setting icon to the window
@@ -35,7 +36,7 @@ class GelBox(QDialog):
         pg.setConfigOptions(antialias=True)
         # creating image view object
         self.imv = pg.ImageView()
-        img = np.array(Image.open("C:\\Users\\lwark\\Desktop\\PLATE1_17012024_103607AM_1_4x26.jpg").rotate(-90).transpose(Image.FLIP_LEFT_RIGHT))
+        img = np.array(Image.open(self.img_path).rotate(-90).transpose(Image.FLIP_LEFT_RIGHT))
         self.imv.setImage(img)#, xvals=np.linspace(1., 3., data.shape[0]))
         layout = QGridLayout()
         # setting this layout to the widget
@@ -54,7 +55,7 @@ class GelBox(QDialog):
         self.setLayout(layout)
 
     def parse_form(self):
-        return self.form.parse_form()
+        return self.img_path, self.form.parse_form()
         
 
 class ControlsForm(QWidget):
@@ -79,6 +80,7 @@ class ControlsForm(QWidget):
         for iii in range(3):
             for jjj in range(3):
                 widge = QLineEdit()
+                widge.setText("Neg")
                 widge.setObjectName(f"{rows[iii]} : {columns[jjj]}")
                 self.layout.addWidget(widge, iii+1, jjj+2, 1, 1)
         self.setLayout(self.layout)

src/submissions/frontend/widgets/submission_widget.py

@@ -312,11 +312,12 @@ class SubmissionFormContainer(QWidget):
             # update_last_used(reagent=reagent, kit=base_submission.extraction_kit)
             reagent.update_last_used(kit=base_submission.extraction_kit)
         # sys.exit()
-        logger.debug(f"Here is the final submission: {pformat(base_submission.__dict__)}")
+        # logger.debug(f"Here is the final submission: {pformat(base_submission.__dict__)}")
-        logger.debug(f"Parsed reagents: {pformat(base_submission.reagents)}")    
+        # logger.debug(f"Parsed reagents: {pformat(base_submission.reagents)}")    
-        logger.debug(f"Sending submission: {base_submission.rsl_plate_num} to database.")
+        # logger.debug(f"Sending submission: {base_submission.rsl_plate_num} to database.")
-        logger.debug(f"Samples from pyd: {pformat(self.pyd.samples)}")
+        # logger.debug(f"Samples from pyd: {pformat(self.pyd.samples)}")
-        logger.debug(f"Samples SQL: {pformat([item.__dict__ for item in base_submission.samples])}")
+        # logger.debug(f"Samples SQL: {pformat([item.__dict__ for item in base_submission.samples])}")
+        # logger.debug(f"")
         base_submission.save()
         # update summary sheet
         self.app.table_widget.sub_wid.setData()

src/submissions/templates/tooltip.html

@@ -1,4 +1,4 @@
 Sample name: {{ fields['submitter_id'] }}<br>
 {% if fields['organism'] %}Organism: {{ fields['organism'] }}<br>{% endif %}
 {% if fields['concentration'] %}Concentration: {{ fields['concentration'] }}<br>{% endif %}
-Well: {{ fields['row'] }}{{ fields['column'] }}
+Well: {{ fields['well'] }}<!--{{ fields['column'] }}-->

src/submissions/tools.py

@@ -357,7 +357,7 @@ def copy_settings(settings_path:Path, settings:dict) -> dict:
             yaml.dump(settings, f)
     return settings
 
-def jinja_template_loading():
+def jinja_template_loading() -> Environment:
     """
     Returns jinja2 template environment.