From eda62fba5a4de31049832208df18e61d1b7c648f Mon Sep 17 00:00:00 2001 From: Landon Wark Date: Wed, 31 Jan 2024 15:21:07 -0600 Subject: [PATCH] Pending large code cleanup --- CHANGELOG.md | 4 + alembic.ini | 4 +- ...0426df72f80_adding_gel_image_info_again.py | 34 ++ ..._tweaking_submission_sample_association.py | 38 ++ ...6_update_to_submissionsampleassociation.py | 50 +++ .../c4201b0ea9fe_adding_gel_image_info.py | 42 +++ alembic/versions/e3f6770ef515_first_commit.py | 340 ++++++++++++++++++ src/submissions/__init__.py | 2 +- src/submissions/backend/db/models/__init__.py | 2 +- src/submissions/backend/db/models/controls.py | 6 +- src/submissions/backend/db/models/kits.py | 45 +-- .../backend/db/models/organizations.py | 4 +- .../backend/db/models/submissions.py | 179 ++++++--- .../backend/validators/__init__.py | 16 +- src/submissions/backend/validators/pydant.py | 49 ++- .../frontend/widgets/gel_checker.py | 14 +- .../frontend/widgets/submission_widget.py | 11 +- src/submissions/templates/tooltip.html | 2 +- src/submissions/tools.py | 2 +- 19 files changed, 741 insertions(+), 103 deletions(-) create mode 100644 alembic/versions/70426df72f80_adding_gel_image_info_again.py create mode 100644 alembic/versions/70d5a751f579_tweaking_submission_sample_association.py create mode 100644 alembic/versions/97392dda5436_update_to_submissionsampleassociation.py create mode 100644 alembic/versions/c4201b0ea9fe_adding_gel_image_info.py create mode 100644 alembic/versions/e3f6770ef515_first_commit.py diff --git a/CHANGELOG.md b/CHANGELOG.md index dfcb8e6..964badb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 202401.04 + +- Large scale database refactor to increase modularity. + ## 202401.01 - Improved tooltips and form regeneration. diff --git a/alembic.ini b/alembic.ini index 309811a..f157eb0 100644 --- a/alembic.ini +++ b/alembic.ini @@ -55,8 +55,8 @@ version_path_separator = os # Use os.pathsep. Default configuration used for ne # are written from script.py.mako # output_encoding = utf-8 -; sqlalchemy.url = sqlite:///L:\Robotics Laboratory Support\Submissions\submissions.db -sqlalchemy.url = sqlite:///C:\Users\lwark\Documents\Archives\Submissions_app_backups\DB_backups\submissions-new.db +sqlalchemy.url = sqlite:///L:\Robotics Laboratory Support\Submissions\submissions.db +; sqlalchemy.url = sqlite:///C:\Users\lwark\Documents\Archives\Submissions_app_backups\DB_backups\submissions-new.db ; sqlalchemy.url = sqlite:///C:\Users\lwark\Documents\python\submissions\tests\test_assets\submissions-test.db diff --git a/alembic/versions/70426df72f80_adding_gel_image_info_again.py b/alembic/versions/70426df72f80_adding_gel_image_info_again.py new file mode 100644 index 0000000..44144b7 --- /dev/null +++ b/alembic/versions/70426df72f80_adding_gel_image_info_again.py @@ -0,0 +1,34 @@ +"""adding gel image, info. Again + +Revision ID: 70426df72f80 +Revises: c4201b0ea9fe +Create Date: 2024-01-30 08:47:22.809841 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '70426df72f80' +down_revision = 'c4201b0ea9fe' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('_wastewaterartic', schema=None) as batch_op: + batch_op.add_column(sa.Column('gel_image', sa.String(length=64), nullable=True)) + batch_op.add_column(sa.Column('gel_info', sa.JSON(), nullable=True)) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('_wastewaterartic', schema=None) as batch_op: + batch_op.drop_column('gel_info') + batch_op.drop_column('gel_image') + + # ### end Alembic commands ### diff --git a/alembic/versions/70d5a751f579_tweaking_submission_sample_association.py b/alembic/versions/70d5a751f579_tweaking_submission_sample_association.py new file mode 100644 index 0000000..970f97a --- /dev/null +++ b/alembic/versions/70d5a751f579_tweaking_submission_sample_association.py @@ -0,0 +1,38 @@ +"""tweaking submission sample association + +Revision ID: 70d5a751f579 +Revises: 97392dda5436 +Create Date: 2024-01-25 13:39:34.163501 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '70d5a751f579' +down_revision = '97392dda5436' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('_submissionsampleassociation', schema=None) as batch_op: + batch_op.alter_column('id', + existing_type=sa.INTEGER(), + nullable=False) + batch_op.create_unique_constraint("ssa_id_unique", ['id']) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('_submissionsampleassociation', schema=None) as batch_op: + batch_op.drop_constraint("ssa_id_unique", type_='unique') + batch_op.alter_column('id', + existing_type=sa.INTEGER(), + nullable=False) + + # ### end Alembic commands ### diff --git a/alembic/versions/97392dda5436_update_to_submissionsampleassociation.py b/alembic/versions/97392dda5436_update_to_submissionsampleassociation.py new file mode 100644 index 0000000..a8844f6 --- /dev/null +++ b/alembic/versions/97392dda5436_update_to_submissionsampleassociation.py @@ -0,0 +1,50 @@ +"""Update to submissionsampleassociation + +Revision ID: 97392dda5436 +Revises: e3f6770ef515 +Create Date: 2024-01-25 09:10:04.384194 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '97392dda5436' +down_revision = 'e3f6770ef515' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('_submissionsampleassociation', schema=None) as batch_op: + batch_op.add_column(sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=True)) + batch_op.create_unique_constraint("submissionsampleassociation_id", ['id']) + + with op.batch_alter_table('_wastewaterassociation', schema=None) as batch_op: + batch_op.add_column(sa.Column('id', sa.INTEGER(), nullable=False)) + # batch_op.drop_constraint("sample_id", type_='foreignkey') + # batch_op.drop_constraint("submission_id", type_='foreignkey') + batch_op.create_foreign_key("fk_subsampassoc_id", '_submissionsampleassociation', ['id'], ['id']) + # batch_op.drop_column('sample_id') + # batch_op.drop_column('submission_id') + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('_wastewaterassociation', schema=None) as batch_op: + batch_op.add_column(sa.Column('submission_id', sa.INTEGER(), nullable=False)) + batch_op.add_column(sa.Column('sample_id', sa.INTEGER(), nullable=False)) + batch_op.drop_constraint(None, type_='foreignkey') + batch_op.create_foreign_key(None, '_submissionsampleassociation', ['submission_id'], ['submission_id']) + batch_op.create_foreign_key(None, '_submissionsampleassociation', ['sample_id'], ['sample_id']) + batch_op.drop_column('id') + + with op.batch_alter_table('_submissionsampleassociation', schema=None) as batch_op: + batch_op.drop_constraint(None, type_='unique') + batch_op.drop_column('id') + + # ### end Alembic commands ### diff --git a/alembic/versions/c4201b0ea9fe_adding_gel_image_info.py b/alembic/versions/c4201b0ea9fe_adding_gel_image_info.py new file mode 100644 index 0000000..7c3cd53 --- /dev/null +++ b/alembic/versions/c4201b0ea9fe_adding_gel_image_info.py @@ -0,0 +1,42 @@ +"""adding gel image, info + +Revision ID: c4201b0ea9fe +Revises: 70d5a751f579 +Create Date: 2024-01-30 08:42:03.928933 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'c4201b0ea9fe' +down_revision = '70d5a751f579' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('_submissionsampleassociation', schema=None) as batch_op: + batch_op.create_unique_constraint("unique_ssa_id", ['id']) + + with op.batch_alter_table('_wastewaterassociation', schema=None) as batch_op: + batch_op.alter_column('id', + existing_type=sa.INTEGER(), + nullable=False) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('_wastewaterassociation', schema=None) as batch_op: + batch_op.alter_column('id', + existing_type=sa.INTEGER(), + nullable=True) + + with op.batch_alter_table('_submissionsampleassociation', schema=None) as batch_op: + batch_op.drop_constraint("unique_ssa_id", type_='unique') + + # ### end Alembic commands ### diff --git a/alembic/versions/e3f6770ef515_first_commit.py b/alembic/versions/e3f6770ef515_first_commit.py new file mode 100644 index 0000000..b6c6d95 --- /dev/null +++ b/alembic/versions/e3f6770ef515_first_commit.py @@ -0,0 +1,340 @@ +"""First Commit + +Revision ID: e3f6770ef515 +Revises: +Create Date: 2024-01-22 14:01:02.958292 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'e3f6770ef515' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('_basicsample', + sa.Column('id', sa.INTEGER(), nullable=False), + sa.Column('submitter_id', sa.String(length=64), nullable=False), + sa.Column('sample_type', sa.String(length=32), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('submitter_id') + ) + op.create_table('_contact', + sa.Column('id', sa.INTEGER(), nullable=False), + sa.Column('name', sa.String(length=64), nullable=True), + sa.Column('email', sa.String(length=64), nullable=True), + sa.Column('phone', sa.String(length=32), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('_controltype', + sa.Column('id', sa.INTEGER(), nullable=False), + sa.Column('name', sa.String(length=255), nullable=True), + sa.Column('targets', sa.JSON(), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name') + ) + op.create_table('_equipment', + sa.Column('id', sa.INTEGER(), nullable=False), + sa.Column('name', sa.String(length=64), nullable=True), + sa.Column('nickname', sa.String(length=64), nullable=True), + sa.Column('asset_number', sa.String(length=16), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('_equipmentrole', + sa.Column('id', sa.INTEGER(), nullable=False), + sa.Column('name', sa.String(length=32), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('_kittype', + sa.Column('id', sa.INTEGER(), nullable=False), + sa.Column('name', sa.String(length=64), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name') + ) + op.create_table('_organization', + sa.Column('id', sa.INTEGER(), nullable=False), + sa.Column('name', sa.String(length=64), nullable=True), + sa.Column('cost_centre', sa.String(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('_process', + sa.Column('id', sa.INTEGER(), nullable=False), + sa.Column('name', sa.String(length=64), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('_reagenttype', + sa.Column('id', sa.INTEGER(), nullable=False), + sa.Column('name', sa.String(length=64), nullable=True), + sa.Column('eol_ext', sa.Interval(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('_submissiontype', + sa.Column('id', sa.INTEGER(), nullable=False), + sa.Column('name', sa.String(length=128), nullable=True), + sa.Column('info_map', sa.JSON(), nullable=True), + sa.Column('template_file', sa.BLOB(), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name') + ) + op.create_table('_bacterialculturesample', + sa.Column('id', sa.INTEGER(), nullable=False), + sa.Column('organism', sa.String(length=64), nullable=True), + sa.Column('concentration', sa.String(length=16), nullable=True), + sa.ForeignKeyConstraint(['id'], ['_basicsample.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('_discount', + sa.Column('id', sa.INTEGER(), nullable=False), + sa.Column('kit_id', sa.INTEGER(), nullable=True), + sa.Column('client_id', sa.INTEGER(), nullable=True), + sa.Column('name', sa.String(length=128), nullable=True), + sa.Column('amount', sa.FLOAT(precision=2), nullable=True), + sa.ForeignKeyConstraint(['client_id'], ['_organization.id'], name='fk_org_id', ondelete='SET NULL'), + sa.ForeignKeyConstraint(['kit_id'], ['_kittype.id'], name='fk_kit_type_id', ondelete='SET NULL'), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('_equipment_processes', + sa.Column('process_id', sa.INTEGER(), nullable=True), + sa.Column('equipment_id', sa.INTEGER(), nullable=True), + sa.ForeignKeyConstraint(['equipment_id'], ['_equipment.id'], ), + sa.ForeignKeyConstraint(['process_id'], ['_process.id'], ) + ) + op.create_table('_equipmentroles_equipment', + sa.Column('equipment_id', sa.INTEGER(), nullable=True), + sa.Column('equipmentroles_id', sa.INTEGER(), nullable=True), + sa.ForeignKeyConstraint(['equipment_id'], ['_equipment.id'], ), + sa.ForeignKeyConstraint(['equipmentroles_id'], ['_equipmentrole.id'], ) + ) + op.create_table('_equipmentroles_processes', + sa.Column('process_id', sa.INTEGER(), nullable=True), + sa.Column('equipmentrole_id', sa.INTEGER(), nullable=True), + sa.ForeignKeyConstraint(['equipmentrole_id'], ['_equipmentrole.id'], ), + sa.ForeignKeyConstraint(['process_id'], ['_process.id'], ) + ) + op.create_table('_kittypereagenttypeassociation', + sa.Column('reagent_types_id', sa.INTEGER(), nullable=False), + sa.Column('kits_id', sa.INTEGER(), nullable=False), + sa.Column('submission_type_id', sa.INTEGER(), nullable=False), + sa.Column('uses', sa.JSON(), nullable=True), + sa.Column('required', sa.INTEGER(), nullable=True), + sa.Column('last_used', sa.String(length=32), nullable=True), + sa.ForeignKeyConstraint(['kits_id'], ['_kittype.id'], ), + sa.ForeignKeyConstraint(['reagent_types_id'], ['_reagenttype.id'], ), + sa.ForeignKeyConstraint(['submission_type_id'], ['_submissiontype.id'], ), + sa.PrimaryKeyConstraint('reagent_types_id', 'kits_id', 'submission_type_id') + ) + op.create_table('_kittypes_processes', + sa.Column('process_id', sa.INTEGER(), nullable=True), + sa.Column('kit_id', sa.INTEGER(), nullable=True), + sa.ForeignKeyConstraint(['kit_id'], ['_kittype.id'], ), + sa.ForeignKeyConstraint(['process_id'], ['_process.id'], ) + ) + op.create_table('_orgs_contacts', + sa.Column('org_id', sa.INTEGER(), nullable=True), + sa.Column('contact_id', sa.INTEGER(), nullable=True), + sa.ForeignKeyConstraint(['contact_id'], ['_contact.id'], ), + sa.ForeignKeyConstraint(['org_id'], ['_organization.id'], ) + ) + op.create_table('_reagent', + sa.Column('id', sa.INTEGER(), nullable=False), + sa.Column('type_id', sa.INTEGER(), nullable=True), + sa.Column('name', sa.String(length=64), nullable=True), + sa.Column('lot', sa.String(length=64), nullable=True), + sa.Column('expiry', sa.TIMESTAMP(), nullable=True), + sa.ForeignKeyConstraint(['type_id'], ['_reagenttype.id'], name='fk_reagent_type_id', ondelete='SET NULL'), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('_submissiontypeequipmentroleassociation', + sa.Column('equipmentrole_id', sa.INTEGER(), nullable=False), + sa.Column('submissiontype_id', sa.INTEGER(), nullable=False), + sa.Column('uses', sa.JSON(), nullable=True), + sa.Column('static', sa.INTEGER(), nullable=True), + sa.ForeignKeyConstraint(['equipmentrole_id'], ['_equipmentrole.id'], ), + sa.ForeignKeyConstraint(['submissiontype_id'], ['_submissiontype.id'], ), + sa.PrimaryKeyConstraint('equipmentrole_id', 'submissiontype_id') + ) + op.create_table('_submissiontypekittypeassociation', + sa.Column('submission_types_id', sa.INTEGER(), nullable=False), + sa.Column('kits_id', sa.INTEGER(), nullable=False), + sa.Column('mutable_cost_column', sa.FLOAT(precision=2), nullable=True), + sa.Column('mutable_cost_sample', sa.FLOAT(precision=2), nullable=True), + sa.Column('constant_cost', sa.FLOAT(precision=2), nullable=True), + sa.ForeignKeyConstraint(['kits_id'], ['_kittype.id'], ), + sa.ForeignKeyConstraint(['submission_types_id'], ['_submissiontype.id'], ), + sa.PrimaryKeyConstraint('submission_types_id', 'kits_id') + ) + op.create_table('_submissiontypes_processes', + sa.Column('process_id', sa.INTEGER(), nullable=True), + sa.Column('equipmentroles_id', sa.INTEGER(), nullable=True), + sa.ForeignKeyConstraint(['equipmentroles_id'], ['_submissiontype.id'], ), + sa.ForeignKeyConstraint(['process_id'], ['_process.id'], ) + ) + op.create_table('_wastewatersample', + sa.Column('id', sa.INTEGER(), nullable=False), + sa.Column('ww_processing_num', sa.String(length=64), nullable=True), + sa.Column('ww_full_sample_id', sa.String(length=64), nullable=True), + sa.Column('rsl_number', sa.String(length=64), nullable=True), + sa.Column('collection_date', sa.TIMESTAMP(), nullable=True), + sa.Column('received_date', sa.TIMESTAMP(), nullable=True), + sa.Column('notes', sa.String(length=2000), nullable=True), + sa.Column('sample_location', sa.String(length=8), nullable=True), + sa.ForeignKeyConstraint(['id'], ['_basicsample.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('_basicsubmission', + sa.Column('id', sa.INTEGER(), nullable=False), + sa.Column('rsl_plate_num', sa.String(length=32), nullable=False), + sa.Column('submitter_plate_num', sa.String(length=127), nullable=True), + sa.Column('submitted_date', sa.TIMESTAMP(), nullable=True), + sa.Column('submitting_lab_id', sa.INTEGER(), nullable=True), + sa.Column('sample_count', sa.INTEGER(), nullable=True), + sa.Column('extraction_kit_id', sa.INTEGER(), nullable=True), + sa.Column('submission_type_name', sa.String(), nullable=True), + sa.Column('technician', sa.String(length=64), nullable=True), + sa.Column('reagents_id', sa.String(), nullable=True), + sa.Column('extraction_info', sa.JSON(), nullable=True), + sa.Column('pcr_info', sa.JSON(), nullable=True), + sa.Column('run_cost', sa.FLOAT(precision=2), nullable=True), + sa.Column('uploaded_by', sa.String(length=32), nullable=True), + sa.Column('comment', sa.JSON(), nullable=True), + sa.Column('submission_category', sa.String(length=64), nullable=True), + sa.ForeignKeyConstraint(['extraction_kit_id'], ['_kittype.id'], name='fk_BS_extkit_id', ondelete='SET NULL'), + sa.ForeignKeyConstraint(['reagents_id'], ['_reagent.id'], name='fk_BS_reagents_id', ondelete='SET NULL'), + sa.ForeignKeyConstraint(['submission_type_name'], ['_submissiontype.name'], name='fk_BS_subtype_name', ondelete='SET NULL'), + sa.ForeignKeyConstraint(['submitting_lab_id'], ['_organization.id'], name='fk_BS_sublab_id', ondelete='SET NULL'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('rsl_plate_num'), + sa.UniqueConstraint('submitter_plate_num') + ) + op.create_table('_reagenttypes_reagents', + sa.Column('reagent_id', sa.INTEGER(), nullable=True), + sa.Column('reagenttype_id', sa.INTEGER(), nullable=True), + sa.ForeignKeyConstraint(['reagent_id'], ['_reagent.id'], ), + sa.ForeignKeyConstraint(['reagenttype_id'], ['_reagenttype.id'], ) + ) + op.create_table('_bacterialculture', + sa.Column('id', sa.INTEGER(), nullable=False), + sa.ForeignKeyConstraint(['id'], ['_basicsubmission.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('_control', + sa.Column('id', sa.INTEGER(), nullable=False), + sa.Column('parent_id', sa.String(), nullable=True), + sa.Column('name', sa.String(length=255), nullable=True), + sa.Column('submitted_date', sa.TIMESTAMP(), nullable=True), + sa.Column('contains', sa.JSON(), nullable=True), + sa.Column('matches', sa.JSON(), nullable=True), + sa.Column('kraken', sa.JSON(), nullable=True), + sa.Column('submission_id', sa.INTEGER(), nullable=True), + sa.Column('refseq_version', sa.String(length=16), nullable=True), + sa.Column('kraken2_version', sa.String(length=16), nullable=True), + sa.Column('kraken2_db_version', sa.String(length=32), nullable=True), + sa.Column('sample_id', sa.INTEGER(), nullable=True), + sa.ForeignKeyConstraint(['parent_id'], ['_controltype.id'], name='fk_control_parent_id'), + sa.ForeignKeyConstraint(['sample_id'], ['_basicsample.id'], name='cont_BCS_id', ondelete='SET NULL'), + sa.ForeignKeyConstraint(['submission_id'], ['_basicsubmission.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name') + ) + op.create_table('_submissionequipmentassociation', + sa.Column('equipment_id', sa.INTEGER(), nullable=False), + sa.Column('submission_id', sa.INTEGER(), nullable=False), + sa.Column('role', sa.String(length=64), nullable=False), + sa.Column('process_id', sa.INTEGER(), nullable=True), + sa.Column('start_time', sa.TIMESTAMP(), nullable=True), + sa.Column('end_time', sa.TIMESTAMP(), nullable=True), + sa.Column('comments', sa.String(length=1024), nullable=True), + sa.ForeignKeyConstraint(['equipment_id'], ['_equipment.id'], ), + sa.ForeignKeyConstraint(['process_id'], ['_process.id'], name='SEA_Process_id', ondelete='SET NULL'), + sa.ForeignKeyConstraint(['submission_id'], ['_basicsubmission.id'], ), + sa.PrimaryKeyConstraint('equipment_id', 'submission_id', 'role') + ) + op.create_table('_submissionreagentassociation', + sa.Column('reagent_id', sa.INTEGER(), nullable=False), + sa.Column('submission_id', sa.INTEGER(), nullable=False), + sa.Column('comments', sa.String(length=1024), nullable=True), + sa.ForeignKeyConstraint(['reagent_id'], ['_reagent.id'], ), + sa.ForeignKeyConstraint(['submission_id'], ['_basicsubmission.id'], ), + sa.PrimaryKeyConstraint('reagent_id', 'submission_id') + ) + op.create_table('_submissionsampleassociation', + sa.Column('sample_id', sa.INTEGER(), nullable=False), + sa.Column('submission_id', sa.INTEGER(), nullable=False), + sa.Column('row', sa.INTEGER(), nullable=False), + sa.Column('column', sa.INTEGER(), nullable=False), + sa.Column('base_sub_type', sa.String(), nullable=True), + sa.ForeignKeyConstraint(['sample_id'], ['_basicsample.id'], ), + sa.ForeignKeyConstraint(['submission_id'], ['_basicsubmission.id'], ), + sa.PrimaryKeyConstraint('submission_id', 'row', 'column') + ) + op.create_table('_wastewater', + sa.Column('id', sa.INTEGER(), nullable=False), + sa.Column('ext_technician', sa.String(length=64), nullable=True), + sa.Column('pcr_technician', sa.String(length=64), nullable=True), + sa.ForeignKeyConstraint(['id'], ['_basicsubmission.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('_wastewaterartic', + sa.Column('id', sa.INTEGER(), nullable=False), + sa.Column('artic_technician', sa.String(length=64), nullable=True), + sa.Column('dna_core_submission_number', sa.String(length=64), nullable=True), + sa.ForeignKeyConstraint(['id'], ['_basicsubmission.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('_wastewaterassociation', + sa.Column('sample_id', sa.INTEGER(), nullable=False), + sa.Column('submission_id', sa.INTEGER(), nullable=False), + sa.Column('ct_n1', sa.FLOAT(precision=2), nullable=True), + sa.Column('ct_n2', sa.FLOAT(precision=2), nullable=True), + sa.Column('n1_status', sa.String(length=32), nullable=True), + sa.Column('n2_status', sa.String(length=32), nullable=True), + sa.Column('pcr_results', sa.JSON(), nullable=True), + sa.ForeignKeyConstraint(['sample_id'], ['_submissionsampleassociation.sample_id'], ), + sa.ForeignKeyConstraint(['submission_id'], ['_submissionsampleassociation.submission_id'], ), + sa.PrimaryKeyConstraint('sample_id', 'submission_id') + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('_wastewaterassociation') + op.drop_table('_wastewaterartic') + op.drop_table('_wastewater') + op.drop_table('_submissionsampleassociation') + op.drop_table('_submissionreagentassociation') + op.drop_table('_submissionequipmentassociation') + op.drop_table('_control') + op.drop_table('_bacterialculture') + op.drop_table('_reagenttypes_reagents') + op.drop_table('_basicsubmission') + op.drop_table('_wastewatersample') + op.drop_table('_submissiontypes_processes') + op.drop_table('_submissiontypekittypeassociation') + op.drop_table('_submissiontypeequipmentroleassociation') + op.drop_table('_reagent') + op.drop_table('_orgs_contacts') + op.drop_table('_kittypes_processes') + op.drop_table('_kittypereagenttypeassociation') + op.drop_table('_equipmentroles_processes') + op.drop_table('_equipmentroles_equipment') + op.drop_table('_equipment_processes') + op.drop_table('_discount') + op.drop_table('_bacterialculturesample') + op.drop_table('_submissiontype') + op.drop_table('_reagenttype') + op.drop_table('_process') + op.drop_table('_organization') + op.drop_table('_kittype') + op.drop_table('_equipmentrole') + op.drop_table('_equipment') + op.drop_table('_controltype') + op.drop_table('_contact') + op.drop_table('_basicsample') + # ### end Alembic commands ### diff --git a/src/submissions/__init__.py b/src/submissions/__init__.py index 048459e..ec5a034 100644 --- a/src/submissions/__init__.py +++ b/src/submissions/__init__.py @@ -4,7 +4,7 @@ from pathlib import Path # Version of the realpython-reader package __project__ = "submissions" -__version__ = "202401.2b" +__version__ = "202401.4b" __author__ = {"name":"Landon Wark", "email":"Landon.Wark@phac-aspc.gc.ca"} __copyright__ = "2022-2024, Government of Canada" diff --git a/src/submissions/backend/db/models/__init__.py b/src/submissions/backend/db/models/__init__.py index 6422211..9be4d47 100644 --- a/src/submissions/backend/db/models/__init__.py +++ b/src/submissions/backend/db/models/__init__.py @@ -50,7 +50,7 @@ class BaseClass(Base): return ctx.backup_path def save(self): - logger.debug(f"Saving {self}") + # logger.debug(f"Saving {self}") try: self.__database_session__.add(self) self.__database_session__.commit() diff --git a/src/submissions/backend/db/models/controls.py b/src/submissions/backend/db/models/controls.py index 0546380..0a3472b 100644 --- a/src/submissions/backend/db/models/controls.py +++ b/src/submissions/backend/db/models/controls.py @@ -78,20 +78,20 @@ class Control(BaseClass): # __tablename__ = '_control_samples' id = Column(INTEGER, primary_key=True) #: primary key - parent_id = Column(String, ForeignKey("_control_types.id", name="fk_control_parent_id")) #: primary key of control type + parent_id = Column(String, ForeignKey("_controltype.id", name="fk_control_parent_id")) #: primary key of control type controltype = relationship("ControlType", back_populates="instances", foreign_keys=[parent_id]) #: reference to parent control type name = Column(String(255), unique=True) #: Sample ID submitted_date = Column(TIMESTAMP) #: Date submitted to Robotics contains = Column(JSON) #: unstructured hashes in contains.tsv for each organism matches = Column(JSON) #: unstructured hashes in matches.tsv for each organism kraken = Column(JSON) #: unstructured output from kraken_report - submission_id = Column(INTEGER, ForeignKey("_submissions.id")) #: parent submission id + submission_id = Column(INTEGER, ForeignKey("_basicsubmission.id")) #: parent submission id submission = relationship("BacterialCulture", back_populates="controls", foreign_keys=[submission_id]) #: parent submission refseq_version = Column(String(16)) #: version of refseq used in fastq parsing kraken2_version = Column(String(16)) #: version of kraken2 used in fastq parsing kraken2_db_version = Column(String(32)) #: folder name of kraken2 db sample = relationship("BacterialCultureSample", back_populates="control") - sample_id = Column(INTEGER, ForeignKey("_samples.id", ondelete="SET NULL", name="cont_BCS_id")) + sample_id = Column(INTEGER, ForeignKey("_basicsample.id", ondelete="SET NULL", name="cont_BCS_id")) def __repr__(self) -> str: return f"" diff --git a/src/submissions/backend/db/models/kits.py b/src/submissions/backend/db/models/kits.py index 7639931..7d18332 100644 --- a/src/submissions/backend/db/models/kits.py +++ b/src/submissions/backend/db/models/kits.py @@ -18,8 +18,8 @@ logger = logging.getLogger(f'submissions.{__name__}') reagenttypes_reagents = Table( "_reagenttypes_reagents", Base.metadata, - Column("reagent_id", INTEGER, ForeignKey("_reagents.id")), - Column("reagenttype_id", INTEGER, ForeignKey("_reagent_types.id")), + Column("reagent_id", INTEGER, ForeignKey("_reagent.id")), + Column("reagenttype_id", INTEGER, ForeignKey("_reagenttype.id")), extend_existing = True ) @@ -27,7 +27,7 @@ equipmentroles_equipment = Table( "_equipmentroles_equipment", Base.metadata, Column("equipment_id", INTEGER, ForeignKey("_equipment.id")), - Column("equipmentroles_id", INTEGER, ForeignKey("_equipment_roles.id")), + Column("equipmentroles_id", INTEGER, ForeignKey("_equipmentrole.id")), extend_existing=True ) @@ -43,7 +43,7 @@ equipmentroles_processes = Table( "_equipmentroles_processes", Base.metadata, Column("process_id", INTEGER, ForeignKey("_process.id")), - Column("equipmentrole_id", INTEGER, ForeignKey("_equipment_roles.id")), + Column("equipmentrole_id", INTEGER, ForeignKey("_equipmentrole.id")), extend_existing=True ) @@ -51,7 +51,7 @@ submissiontypes_processes = Table( "_submissiontypes_processes", Base.metadata, Column("process_id", INTEGER, ForeignKey("_process.id")), - Column("equipmentroles_id", INTEGER, ForeignKey("_submission_types.id")), + Column("equipmentroles_id", INTEGER, ForeignKey("_submissiontype.id")), extend_existing=True ) @@ -59,7 +59,7 @@ kittypes_processes = Table( "_kittypes_processes", Base.metadata, Column("process_id", INTEGER, ForeignKey("_process.id")), - Column("kit_id", INTEGER, ForeignKey("_kits.id")), + Column("kit_id", INTEGER, ForeignKey("_kittype.id")), extend_existing=True ) @@ -304,7 +304,7 @@ class Reagent(BaseClass): id = Column(INTEGER, primary_key=True) #: primary key type = relationship("ReagentType", back_populates="instances", secondary=reagenttypes_reagents) #: joined parent reagent type - type_id = Column(INTEGER, ForeignKey("_reagent_types.id", ondelete='SET NULL', name="fk_reagent_type_id")) #: id of parent reagent type + type_id = Column(INTEGER, ForeignKey("_reagenttype.id", ondelete='SET NULL', name="fk_reagent_type_id")) #: id of parent reagent type name = Column(String(64)) #: reagent name lot = Column(String(64)) #: lot number of reagent expiry = Column(TIMESTAMP) #: expiry date - extended by eol_ext of parent programmatically @@ -442,9 +442,9 @@ class Discount(BaseClass): id = Column(INTEGER, primary_key=True) #: primary key kit = relationship("KitType") #: joined parent reagent type - kit_id = Column(INTEGER, ForeignKey("_kits.id", ondelete='SET NULL', name="fk_kit_type_id")) #: id of joined kit + kit_id = Column(INTEGER, ForeignKey("_kittype.id", ondelete='SET NULL', name="fk_kit_type_id")) #: id of joined kit client = relationship("Organization") #: joined client lab - client_id = Column(INTEGER, ForeignKey("_organizations.id", ondelete='SET NULL', name="fk_org_id")) #: id of joined client + client_id = Column(INTEGER, ForeignKey("_organization.id", ondelete='SET NULL', name="fk_org_id")) #: id of joined client name = Column(String(128)) #: Short description amount = Column(FLOAT(2)) #: Dollar amount of discount @@ -625,8 +625,9 @@ class SubmissionType(BaseClass): """ Adds this instances to the database and commits. """ - self.__database_session__.add(self) - self.__database_session__.commit() + # self.__database_session__.add(self) + # self.__database_session__.commit() + super().save() class SubmissionTypeKitTypeAssociation(BaseClass): """ @@ -634,8 +635,8 @@ class SubmissionTypeKitTypeAssociation(BaseClass): """ # __tablename__ = "_submissiontypes_kittypes" - submission_types_id = Column(INTEGER, ForeignKey("_submission_types.id"), primary_key=True) #: id of joined submission type - kits_id = Column(INTEGER, ForeignKey("_kits.id"), primary_key=True) #: id of joined kit + submission_types_id = Column(INTEGER, ForeignKey("_submissiontype.id"), primary_key=True) #: id of joined submission type + kits_id = Column(INTEGER, ForeignKey("_kittype.id"), primary_key=True) #: id of joined kit mutable_cost_column = Column(FLOAT(2)) #: dollar amount per 96 well plate that can change with number of columns (reagents, tips, etc) mutable_cost_sample = Column(FLOAT(2)) #: dollar amount that can change with number of samples (reagents, tips, etc) constant_cost = Column(FLOAT(2)) #: dollar amount per plate that will remain constant (plates, man hours, etc) @@ -707,9 +708,9 @@ class KitTypeReagentTypeAssociation(BaseClass): """ # __tablename__ = "_reagenttypes_kittypes" - reagent_types_id = Column(INTEGER, ForeignKey("_reagent_types.id"), primary_key=True) #: id of associated reagent type - kits_id = Column(INTEGER, ForeignKey("_kits.id"), primary_key=True) #: id of associated reagent type - submission_type_id = Column(INTEGER, ForeignKey("_submission_types.id"), primary_key=True) + reagent_types_id = Column(INTEGER, ForeignKey("_reagenttype.id"), primary_key=True) #: id of associated reagent type + kits_id = Column(INTEGER, ForeignKey("_kittype.id"), primary_key=True) #: id of associated reagent type + submission_type_id = Column(INTEGER, ForeignKey("_submissiontype.id"), primary_key=True) uses = Column(JSON) #: map to location on excel sheets of different submission types required = Column(INTEGER) #: whether the reagent type is required for the kit (Boolean 1 or 0) last_used = Column(String(32)) #: last used lot number of this type of reagent @@ -810,8 +811,8 @@ class SubmissionReagentAssociation(BaseClass): # __tablename__ = "_reagents_submissions" - reagent_id = Column(INTEGER, ForeignKey("_reagents.id"), primary_key=True) #: id of associated sample - submission_id = Column(INTEGER, ForeignKey("_submissions.id"), primary_key=True) + reagent_id = Column(INTEGER, ForeignKey("_reagent.id"), primary_key=True) #: id of associated sample + submission_id = Column(INTEGER, ForeignKey("_basicsubmission.id"), primary_key=True) comments = Column(String(1024)) submission = relationship("BasicSubmission", back_populates="submission_reagent_associations") #: associated submission @@ -1060,7 +1061,7 @@ class SubmissionEquipmentAssociation(BaseClass): # __tablename__ = "_equipment_submissions" equipment_id = Column(INTEGER, ForeignKey("_equipment.id"), primary_key=True) #: id of associated equipment - submission_id = Column(INTEGER, ForeignKey("_submissions.id"), primary_key=True) #: id of associated submission + submission_id = Column(INTEGER, ForeignKey("_basicsubmission.id"), primary_key=True) #: id of associated submission role = Column(String(64), primary_key=True) #: name of the role the equipment fills # process = Column(String(64)) #: name of the process run on this equipment process_id = Column(INTEGER, ForeignKey("_process.id",ondelete="SET NULL", name="SEA_Process_id")) @@ -1090,8 +1091,8 @@ class SubmissionTypeEquipmentRoleAssociation(BaseClass): # __tablename__ = "_submissiontype_equipmentrole" - equipmentrole_id = Column(INTEGER, ForeignKey("_equipment_roles.id"), primary_key=True) #: id of associated equipment - submissiontype_id = Column(INTEGER, ForeignKey("_submission_types.id"), primary_key=True) #: id of associated submission + equipmentrole_id = Column(INTEGER, ForeignKey("_equipmentrole.id"), primary_key=True) #: id of associated equipment + submissiontype_id = Column(INTEGER, ForeignKey("_submissiontype.id"), primary_key=True) #: id of associated submission uses = Column(JSON) #: locations of equipment on the submission type excel sheet. static = Column(INTEGER, default=1) #: if 1 this piece of equipment will always be used, otherwise it will need to be selected from list? @@ -1156,7 +1157,7 @@ class Process(BaseClass): @classmethod @setup_lookup - def query(cls, name:str|None, limit:int=0): + def query(cls, name:str|None=None, limit:int=0): query = cls.__database_session__.query(cls) match name: case str(): diff --git a/src/submissions/backend/db/models/organizations.py b/src/submissions/backend/db/models/organizations.py index 4af9f7a..c49d03b 100644 --- a/src/submissions/backend/db/models/organizations.py +++ b/src/submissions/backend/db/models/organizations.py @@ -15,8 +15,8 @@ logger = logging.getLogger(f"submissions.{__name__}") orgs_contacts = Table( "_orgs_contacts", Base.metadata, - Column("org_id", INTEGER, ForeignKey("_organizations.id")), - Column("contact_id", INTEGER, ForeignKey("_contacts.id")), + Column("org_id", INTEGER, ForeignKey("_organization.id")), + Column("contact_id", INTEGER, ForeignKey("_contact.id")), # __table_args__ = {'extend_existing': True} extend_existing = True ) diff --git a/src/submissions/backend/db/models/submissions.py b/src/submissions/backend/db/models/submissions.py index c115f89..e4f2637 100644 --- a/src/submissions/backend/db/models/submissions.py +++ b/src/submissions/backend/db/models/submissions.py @@ -3,7 +3,8 @@ Models for the main submission types. ''' from __future__ import annotations from getpass import getuser -import math, json, logging, uuid, tempfile, re, yaml +import math, json, logging, uuid, tempfile, re, yaml, zipfile +import sys from operator import attrgetter from pprint import pformat from . import Reagent, SubmissionType, KitType, Organization @@ -13,9 +14,10 @@ from json.decoder import JSONDecodeError from sqlalchemy.ext.associationproxy import association_proxy import pandas as pd from openpyxl import Workbook -from . import BaseClass, Equipment +from openpyxl.worksheet.worksheet import Worksheet +from . import BaseClass from tools import check_not_nan, row_map, query_return, setup_lookup, jinja_template_loading -from datetime import datetime, date, time +from datetime import datetime, date from typing import List, Any from dateutil.parser import parse from dateutil.parser._parser import ParserError @@ -37,17 +39,16 @@ class BasicSubmission(BaseClass): submitter_plate_num = Column(String(127), unique=True) #: The number given to the submission by the submitting lab submitted_date = Column(TIMESTAMP) #: Date submission received submitting_lab = relationship("Organization", back_populates="submissions") #: client org - submitting_lab_id = Column(INTEGER, ForeignKey("_organizations.id", ondelete="SET NULL", name="fk_BS_sublab_id")) #: client lab id from _organizations + submitting_lab_id = Column(INTEGER, ForeignKey("_organization.id", ondelete="SET NULL", name="fk_BS_sublab_id")) #: client lab id from _organizations sample_count = Column(INTEGER) #: Number of samples in the submission extraction_kit = relationship("KitType", back_populates="submissions") #: The extraction kit used - extraction_kit_id = Column(INTEGER, ForeignKey("_kits.id", ondelete="SET NULL", name="fk_BS_extkit_id")) #: id of joined extraction kit - submission_type_name = Column(String, ForeignKey("_submission_types.name", ondelete="SET NULL", name="fk_BS_subtype_name")) #: name of joined submission type + extraction_kit_id = Column(INTEGER, ForeignKey("_kittype.id", ondelete="SET NULL", name="fk_BS_extkit_id")) #: id of joined extraction kit + submission_type_name = Column(String, ForeignKey("_submissiontype.name", ondelete="SET NULL", name="fk_BS_subtype_name")) #: name of joined submission type technician = Column(String(64)) #: initials of processing tech(s) # Move this into custom types? # reagents = relationship("Reagent", back_populates="submissions", secondary=reagents_submissions) #: relationship to reagents - reagents_id = Column(String, ForeignKey("_reagents.id", ondelete="SET NULL", name="fk_BS_reagents_id")) #: id of used reagents + reagents_id = Column(String, ForeignKey("_reagent.id", ondelete="SET NULL", name="fk_BS_reagents_id")) #: id of used reagents extraction_info = Column(JSON) #: unstructured output from the extraction table logger. - pcr_info = Column(JSON) #: unstructured output from pcr table logger or user(Artic) run_cost = Column(FLOAT(2)) #: total cost of running the plate. Set from constant and mutable kit costs at time of creation. uploaded_by = Column(String(32)) #: user name of person who submitted the submission to the database. comment = Column(JSON) #: user notes @@ -132,19 +133,21 @@ class BasicSubmission(BaseClass): logger.error(f"Json error in {self.rsl_plate_num}: {e}") # Updated 2023-09 to use the extraction kit to pull reagents. if full_data: + logger.debug(f"Attempting reagents.") try: reagents = [item.to_sub_dict(extraction_kit=self.extraction_kit) for item in self.submission_reagent_associations] except Exception as e: logger.error(f"We got an error retrieving reagents: {e}") reagents = None # samples = [item.sample.to_sub_dict(submission_rsl=self.rsl_plate_num) for item in self.submission_sample_associations] + logger.debug(f"Running samples.") samples = self.adjust_to_dict_samples(backup=backup) try: equipment = [item.to_sub_dict() for item in self.submission_equipment_associations] if len(equipment) == 0: equipment = None except Exception as e: - logger.error(f"Error setting equipment: {self.equipment}") + logger.error(f"Error setting equipment: {e}") equipment = None else: reagents = None @@ -155,7 +158,6 @@ class BasicSubmission(BaseClass): except Exception as e: logger.error(f"Error setting comment: {self.comment}") comments = None - output = { "id": self.id, "Plate Number": self.rsl_plate_num, @@ -440,6 +442,7 @@ class BasicSubmission(BaseClass): def filename_template(cls) -> str: """ Constructs template for filename of this class. + Note: This is meant to be used with the dictionary constructed in self.to_dict(). Keys need to have spaces removed Returns: str: filename template in jinja friendly format. @@ -462,6 +465,20 @@ class BasicSubmission(BaseClass): logger.warning(f"Couldn't drop '{item}' column from submissionsheet df.") return df + @classmethod + def custom_sample_autofill_row(cls, sample, worksheet:Worksheet) -> int: + """ + _summary_ + + Args: + sample (_type_): _description_ + worksheet (Workbook): _description_ + + Returns: + int: _description_ + """ + return None + def set_attribute(self, key:str, value): """ Performs custom attribute setting based on values. @@ -547,7 +564,7 @@ class BasicSubmission(BaseClass): """ from backend.validators import PydSubmission, PydSample, PydReagent, PydEquipment dicto = self.to_dict(full_data=True, backup=backup) - logger.debug(f"Backup dictionary: {pformat(dicto)}") + # logger.debug(f"Backup dictionary: {pformat(dicto)}") # dicto['filepath'] = Path(tempfile.TemporaryFile().name) new_dict = {} for key, value in dicto.items(): @@ -572,7 +589,7 @@ class BasicSubmission(BaseClass): # new_dict[key.lower().replace(" ", "_")]['value'] = value # new_dict[key.lower().replace(" ", "_")]['missing'] = True new_dict['filepath'] = Path(tempfile.TemporaryFile().name) - logger.debug(f"Dictionary coming into PydSubmission: {pformat(new_dict)}") + # logger.debug(f"Dictionary coming into PydSubmission: {pformat(new_dict)}") # sys.exit() return PydSubmission(**new_dict) @@ -797,22 +814,15 @@ class BasicSubmission(BaseClass): # logger.debug(f"Save result: {result}") def add_equipment(self, obj): - # submission_type = submission.submission_type_name from frontend.widgets.equipment_usage import EquipmentUsage - dlg = EquipmentUsage(parent=obj, submission_type=self.submission_type_name, submission=self) + dlg = EquipmentUsage(parent=obj, submission=self) if dlg.exec(): equipment = dlg.parse_form() logger.debug(f"We've got equipment: {equipment}") for equip in equipment: - # e = Equipment.query(name=equip.name) - # assoc = SubmissionEquipmentAssociation(submission=submission, equipment=e) - # process = Process.query(name=equip.processes) - # assoc.process = process - # assoc.role = equip.role + logger.debug(f"Processing: {equip}") _, assoc = equip.toSQL(submission=self) - # submission.submission_equipment_associations.append(assoc) logger.debug(f"Appending SubmissionEquipmentAssociation: {assoc}") - # submission.save() assoc.save() else: pass @@ -825,12 +835,14 @@ class BasicSubmission(BaseClass): fname (Path): Filename of xlsx file. """ logger.debug("Hello from backup.") + pyd = self.to_pydantic(backup=True) if fname == None: from frontend.widgets.functions import select_save_file - from backend.validators import RSLNamer - abbreviation = self.get_abbreviation() - file_data = dict(rsl_plate_num=self.rsl_plate_num, submission_type=self.submission_type_name, submitted_date=self.submitted_date, abbreviation=abbreviation) - fname = select_save_file(default_name=RSLNamer.construct_new_plate_name(data=file_data), extension="xlsx", obj=obj) + fname = select_save_file(default_name=pyd.construct_filename(), extension="xlsx", obj=obj) + logger.debug(fname.name) + if fname.name == "": + logger.debug(f"export cancelled.") + return if full_backup: backup = self.to_dict(full_data=True) try: @@ -838,7 +850,6 @@ class BasicSubmission(BaseClass): yaml.dump(backup, f) except KeyError as e: logger.error(f"Problem saving yml backup file: {e}") - pyd = self.to_pydantic(backup=True) wb = pyd.autofill_excel() wb = pyd.autofill_samples(wb) wb = pyd.autofill_equipment(wb) @@ -856,14 +867,14 @@ class BacterialCulture(BasicSubmission): polymorphic_load="inline", inherit_condition=(id == BasicSubmission.id)) - def to_dict(self, full_data:bool=False) -> dict: + def to_dict(self, full_data:bool=False, backup:bool=False) -> dict: """ Extends parent class method to add controls to dict Returns: dict: dictionary used in submissions summary """ - output = super().to_dict(full_data=full_data) + output = super().to_dict(full_data=full_data, backup=backup) if full_data: output['controls'] = [item.to_sub_dict() for item in self.controls] return output @@ -996,6 +1007,22 @@ class BacterialCulture(BasicSubmission): input_dict['submitted_date']['missing'] = True return input_dict + @classmethod + def custom_sample_autofill_row(cls, sample, worksheet: Worksheet) -> int: + logger.debug(f"Checking {sample.well}") + logger.debug(f"here's the worksheet: {worksheet}") + row = super().custom_sample_autofill_row(sample, worksheet) + df = pd.DataFrame(list(worksheet.values)) + # logger.debug(f"Here's the dataframe: {df}") + idx = df[df[0]==sample.well] + if idx.empty: + new = f"{sample.well[0]}{sample.well[1:].zfill(2)}" + logger.debug(f"Checking: {new}") + idx = df[df[0]==new] + logger.debug(f"Here is the row: {idx}") + row = idx.index.to_list()[0] + return row + 1 + class Wastewater(BasicSubmission): """ derivative submission type from BasicSubmission @@ -1003,11 +1030,13 @@ class Wastewater(BasicSubmission): id = Column(INTEGER, ForeignKey('_basicsubmission.id'), primary_key=True) ext_technician = Column(String(64)) pcr_technician = Column(String(64)) + pcr_info = Column(JSON) #: unstructured output from pcr table logger or user(Artic) + __mapper_args__ = __mapper_args__ = dict(polymorphic_identity="Wastewater", polymorphic_load="inline", inherit_condition=(id == BasicSubmission.id)) - def to_dict(self, full_data:bool=False) -> dict: + def to_dict(self, full_data:bool=False, backup:bool=False) -> dict: """ Extends parent class method to add controls to dict @@ -1020,6 +1049,7 @@ class Wastewater(BasicSubmission): except TypeError as e: pass output['Technician'] = f"Enr: {self.technician}, Ext: {self.ext_technician}, PCR: {self.pcr_technician}" + return output @classmethod @@ -1144,6 +1174,18 @@ class Wastewater(BasicSubmission): def adjust_autofill_samples(cls, samples: List[Any]) -> List[Any]: samples = super().adjust_autofill_samples(samples) return [item for item in samples if not item.submitter_id.startswith("EN")] + + @classmethod + def custom_sample_autofill_row(cls, sample, worksheet: Worksheet) -> int: + logger.debug(f"Checking {sample.well}") + logger.debug(f"here's the worksheet: {worksheet}") + row = super().custom_sample_autofill_row(sample, worksheet) + df = pd.DataFrame(list(worksheet.values)) + logger.debug(f"Here's the dataframe: {df}") + idx = df[df[1]==sample.sample_location] + logger.debug(f"Here is the row: {idx}") + row = idx.index.to_list()[0] + return row + 1 class WastewaterArtic(BasicSubmission): """ @@ -1155,6 +1197,9 @@ class WastewaterArtic(BasicSubmission): inherit_condition=(id == BasicSubmission.id)) artic_technician = Column(String(64)) dna_core_submission_number = Column(String(64)) + pcr_info = Column(JSON) #: unstructured output from pcr table logger or user(Artic) + gel_image = Column(String(64)) + gel_info = Column(JSON) def calculate_base_cost(self): """ @@ -1381,10 +1426,19 @@ class WastewaterArtic(BasicSubmission): def gel_box(self, obj): from frontend.widgets.gel_checker import GelBox - dlg = GelBox(parent=obj) + from frontend.widgets import select_open_file + fname = select_open_file(obj=obj, file_extension="jpg") + dlg = GelBox(parent=obj, img_path=fname) if dlg.exec(): - output = dlg.parse_form() - print(output) + img_path, output = dlg.parse_form() + self.gel_image = img_path.name + self.gel_info = output + with zipfile.ZipFile(self.__directory_path__.joinpath("submission_imgs.zip"), 'a') as zipf: + # Add a file located at the source_path to the destination within the zip + # file. It will overwrite existing files if the names collide, but it + # will give a warning + zipf.write(img_path, self.gel_image) + self.save() # Sample Classes @@ -1439,7 +1493,10 @@ class BasicSample(BaseClass): return value def __repr__(self) -> str: - return f"<{self.sample_type.replace('_', ' ').title().replace(' ', '')}({self.submitter_id})>" + try: + return f"<{self.sample_type.replace('_', ' ').title().replace(' ', '')}({self.submitter_id})>" + except AttributeError: + return f" dict: """ @@ -1448,6 +1505,7 @@ class BasicSample(BaseClass): Returns: dict: well location and name (sample id, organism) NOTE: keys must sync with WWSample to_sub_dict above """ + # logger.debug(f"Converting {self} to dict.") sample = {} sample['submitter_id'] = self.submitter_id sample['sample_type'] = self.sample_type @@ -1642,6 +1700,9 @@ class WastewaterSample(BasicSample): """ sample = super().to_sub_dict(submission_rsl=submission_rsl) sample['ww_processing_num'] = self.ww_processing_num + sample['sample_location'] = self.sample_location + sample['received_date'] = self.received_date + sample['collection_date'] = self.collection_date return sample @classmethod @@ -1721,9 +1782,10 @@ class SubmissionSampleAssociation(BaseClass): """ # __tablename__ = "_submission_sample" - - sample_id = Column(INTEGER, ForeignKey("_samples.id"), nullable=False) #: id of associated sample - submission_id = Column(INTEGER, ForeignKey("_submissions.id"), primary_key=True) #: id of associated submission + + id = Column(INTEGER, unique=True, nullable=False) + sample_id = Column(INTEGER, ForeignKey("_basicsample.id"), nullable=False) #: id of associated sample + submission_id = Column(INTEGER, ForeignKey("_basicsubmission.id"), primary_key=True) #: id of associated submission row = Column(INTEGER, primary_key=True) #: row on the 96 well plate column = Column(INTEGER, primary_key=True) #: column on the 96 well plate @@ -1743,14 +1805,23 @@ class SubmissionSampleAssociation(BaseClass): "with_polymorphic": "*", } - def __init__(self, submission:BasicSubmission=None, sample:BasicSample=None, row:int=1, column:int=1): + def __init__(self, submission:BasicSubmission=None, sample:BasicSample=None, row:int=1, column:int=1, id:int|None=None): self.submission = submission self.sample = sample self.row = row self.column = column + if id != None: + self.id = id + else: + self.id = self.__class__.autoincrement_id() + logger.debug(f"Using id: {self.id}") def __repr__(self) -> str: - return f" dict: """ @@ -1760,6 +1831,7 @@ class SubmissionSampleAssociation(BaseClass): dict: Updated dictionary with row, column and well updated """ # Get sample info + # logger.debug(f"Running {self.__repr__()}") sample = self.sample.to_sub_dict(submission_rsl=self.submission) # sample = {} sample['name'] = self.sample.submitter_id @@ -1787,6 +1859,7 @@ class SubmissionSampleAssociation(BaseClass): # Since there is no PCR, negliable result is necessary. # assoc = [item for item in self.sample_submission_associations if item.submission.rsl_plate_num==submission_rsl][0] sample = self.to_sub_dict() + logger.debug(f"Sample dict to hitpick: {sample}") env = jinja_template_loading() template = env.get_template("tooltip.html") tooltip_text = template.render(fields=sample) @@ -1801,6 +1874,14 @@ class SubmissionSampleAssociation(BaseClass): sample.update(dict(name=self.sample.submitter_id[:10], tooltip=tooltip_text)) return sample + @classmethod + def autoincrement_id(cls): + try: + return max([item.id for item in cls.query()]) + 1 + except ValueError as e: + logger.error(f"Problem incrementing id: {e}") + return 1 + @classmethod def find_polymorphic_subclass(cls, polymorphic_identity:str|None=None) -> SubmissionSampleAssociation: """ @@ -1890,6 +1971,7 @@ class SubmissionSampleAssociation(BaseClass): association_type:str="Basic Association", submission:BasicSubmission|str|None=None, sample:BasicSample|str|None=None, + id:int|None=None, **kwargs) -> SubmissionSampleAssociation: """ Queries for an association, if none exists creates a new one. @@ -1931,7 +2013,7 @@ class SubmissionSampleAssociation(BaseClass): instance = None if instance == None: used_cls = cls.find_polymorphic_subclass(polymorphic_identity=association_type) - instance = used_cls(submission=submission, sample=sample, **kwargs) + instance = used_cls(submission=submission, sample=sample, id=id, **kwargs) return instance def delete(self): @@ -1941,8 +2023,8 @@ class WastewaterAssociation(SubmissionSampleAssociation): """ Derivative custom Wastewater/Submission Association... fancy. """ - sample_id = Column(INTEGER, ForeignKey('_submissionsampleassociation.sample_id'), primary_key=True) - submission_id = Column(INTEGER, ForeignKey('_submissionsampleassociation.submission_id'), primary_key=True) + # sample_id = Column(INTEGER, ForeignKey('_submissionsampleassociation.sample_id'), primary_key=True) + id = Column(INTEGER, ForeignKey("_submissionsampleassociation.id"), primary_key=True) ct_n1 = Column(FLOAT(2)) #: AKA ct for N1 ct_n2 = Column(FLOAT(2)) #: AKA ct for N2 n1_status = Column(String(32)) #: positive or negative for N1 @@ -1952,7 +2034,11 @@ class WastewaterAssociation(SubmissionSampleAssociation): # __mapper_args__ = {"polymorphic_identity": "Wastewater Association", "polymorphic_load": "inline"} __mapper_args__ = dict(polymorphic_identity="Wastewater Association", polymorphic_load="inline", - inherit_condition=(sample_id == SubmissionSampleAssociation.sample_id)) + # inherit_condition=(submission_id==SubmissionSampleAssociation.submission_id and + # row==SubmissionSampleAssociation.row and + # column==SubmissionSampleAssociation.column)) + inherit_condition=(id==SubmissionSampleAssociation.id)) + # inherit_foreign_keys=(sample_id == SubmissionSampleAssociation.sample_id, submission_id == SubmissionSampleAssociation.submission_id)) def to_sub_dict(self) -> dict: sample = super().to_sub_dict() @@ -1970,3 +2056,12 @@ class WastewaterAssociation(SubmissionSampleAssociation): except (TypeError, AttributeError) as e: logger.error(f"Couldn't set tooltip for {self.sample.rsl_number}. Looks like there isn't PCR data.") return sample + + @classmethod + def autoincrement_id(cls): + try: + parent = [base for base in cls.__bases__ if base.__name__=="SubmissionSampleAssociation"][0] + return max([item.id for item in parent.query()]) + 1 + except ValueError as e: + logger.error(f"Problem incrementing id: {e}") + return 1 \ No newline at end of file diff --git a/src/submissions/backend/validators/__init__.py b/src/submissions/backend/validators/__init__.py index e7f47f2..02f425e 100644 --- a/src/submissions/backend/validators/__init__.py +++ b/src/submissions/backend/validators/__init__.py @@ -3,6 +3,7 @@ from pathlib import Path from openpyxl import load_workbook from backend.db.models import BasicSubmission, SubmissionType from datetime import date +from tools import jinja_template_loading logger = logging.getLogger(f"submissions.{__name__}") @@ -126,9 +127,20 @@ class RSLNamer(object): today = parse(today.group()) except AttributeError: today = datetime.now() - previous = BasicSubmission.query(start_date=today, end_date=today, submission_type=data['submission_type']) - plate_number = len(previous) + 1 + if "rsl_plate_num" in data.keys(): + plate_number = data['rsl_plate_num'].split("-")[-1][0] + else: + previous = BasicSubmission.query(start_date=today, end_date=today, submission_type=data['submission_type']) + plate_number = len(previous) + 1 return f"RSL-{data['abbreviation']}-{today.year}{str(today.month).zfill(2)}{str(today.day).zfill(2)}-{plate_number}" + + @classmethod + def construct_export_name(cls, template, **kwargs): + logger.debug(f"Kwargs: {kwargs}") + logger.debug(f"Template: {template}") + environment = jinja_template_loading() + template = environment.from_string(template) + return template.render(**kwargs) from .pydant import * \ No newline at end of file diff --git a/src/submissions/backend/validators/pydant.py b/src/submissions/backend/validators/pydant.py index 8bc9992..3ba6597 100644 --- a/src/submissions/backend/validators/pydant.py +++ b/src/submissions/backend/validators/pydant.py @@ -8,7 +8,7 @@ from pydantic import BaseModel, field_validator, Field from datetime import date, datetime, timedelta from dateutil.parser import parse from dateutil.parser._parser import ParserError -from typing import List, Any, Tuple +from typing import List, Tuple from . import RSLNamer from pathlib import Path from tools import check_not_nan, convert_nans_to_nones, jinja_template_loading, Report, Result, row_map @@ -156,8 +156,9 @@ class PydSample(BaseModel, extra='allow'): sample_type: str row: int|List[int]|None column: int|List[int]|None + assoc_id: int|List[int]|None = Field(default=None) - @field_validator("row", "column") + @field_validator("row", "column", "assoc_id") @classmethod def row_int_to_list(cls, value): if isinstance(value, int): @@ -193,14 +194,14 @@ class PydSample(BaseModel, extra='allow'): out_associations = [] if submission != None: assoc_type = self.sample_type.replace("Sample", "").strip() - for row, column in zip(self.row, self.column): - # logger.debug(f"Looking up association with identity: ({submission.submission_type_name} Association)") + for row, column, id in zip(self.row, self.column, self.assoc_id): + logger.debug(f"Looking up association with identity: ({submission.submission_type_name} Association)") logger.debug(f"Looking up association with identity: ({assoc_type} Association)") association = SubmissionSampleAssociation.query_or_create(association_type=f"{assoc_type} Association", - submission=submission, - sample=instance, - row=row, column=column) - logger.debug(f"Using submission_sample_association: {association}") + submission=submission, + sample=instance, + row=row, column=column, id=id) + # logger.debug(f"Using submission_sample_association: {association}") try: instance.sample_submission_associations.append(association) out_associations.append(association) @@ -254,7 +255,7 @@ class PydEquipment(BaseModel, extra='ignore'): assoc.process = process assoc.role = self.role # equipment.equipment_submission_associations.append(assoc) - equipment.equipment_submission_associations.append(assoc) + # equipment.equipment_submission_associations.append(assoc) else: assoc = None return equipment, assoc @@ -275,7 +276,7 @@ class PydSubmission(BaseModel, extra='allow'): comment: dict|None = Field(default=dict(value="", missing=True), validate_default=True) reagents: List[dict]|List[PydReagent] = [] samples: List[PydSample] - equipment: List[PydEquipment]|None + equipment: List[PydEquipment]|None =[] @field_validator('equipment', mode='before') @classmethod @@ -421,6 +422,16 @@ class PydSubmission(BaseModel, extra='allow'): value['value'] = values.data['submission_type']['value'] return value + @field_validator("samples") + def assign_ids(cls, value, values): + starting_id = SubmissionSampleAssociation.autoincrement_id() + output = [] + for iii, sample in enumerate(value, start=starting_id): + sample.assoc_id = [iii] + output.append(sample) + return output + + def handle_duplicate_samples(self): """ Collapses multiple samples with same submitter id into one with lists for rows, columns. @@ -428,14 +439,19 @@ class PydSubmission(BaseModel, extra='allow'): """ submitter_ids = list(set([sample.submitter_id for sample in self.samples])) output = [] - for id in submitter_ids: + for iii, id in enumerate(submitter_ids, start=1): relevants = [item for item in self.samples if item.submitter_id==id] if len(relevants) <= 1: output += relevants else: rows = [item.row[0] for item in relevants] columns = [item.column[0] for item in relevants] + ids = [item.assoc_id[0] for item in relevants] + # for jjj, rel in enumerate(relevants, start=1): + # starting_id += jjj + # ids.append(starting_id) dummy = relevants[0] + dummy.assoc_id = ids dummy.row = rows dummy.column = columns output.append(dummy) @@ -663,14 +679,17 @@ class PydSubmission(BaseModel, extra='allow'): logger.debug(f"Workbook sheets: {workbook.sheetnames}") worksheet = workbook[sample_info["lookup_table"]['sheet']] samples = sorted(self.samples, key=attrgetter('column', 'row')) - custom_sampler = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=self.submission_type).adjust_autofill_samples - samples = custom_sampler(samples=samples) + submission_obj = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=self.submission_type) + samples = submission_obj.adjust_autofill_samples(samples=samples) logger.debug(f"Samples: {pformat(samples)}") # Fail safe against multiple instances of the same sample for iii, sample in enumerate(samples, start=1): - row = sample_info['lookup_table']['start_row'] + iii + logger.debug(f"Sample: {sample}") + row = submission_obj.custom_sample_autofill_row(sample, worksheet=worksheet) + logger.debug(f"Writing to {row}") + if row == None: + row = sample_info['lookup_table']['start_row'] + iii fields = [field for field in list(sample.model_fields.keys()) + list(sample.model_extra.keys()) if field in sample_info['sample_columns'].keys()] - for field in fields: column = sample_info['sample_columns'][field] value = getattr(sample, field) diff --git a/src/submissions/frontend/widgets/gel_checker.py b/src/submissions/frontend/widgets/gel_checker.py index aed5086..0e47580 100644 --- a/src/submissions/frontend/widgets/gel_checker.py +++ b/src/submissions/frontend/widgets/gel_checker.py @@ -1,7 +1,7 @@ # import required modules -from PyQt6.QtCore import Qt +# from PyQt6.QtCore import Qt from PyQt6.QtWidgets import * -import sys +# import sys from PyQt6.QtWidgets import QWidget import numpy as np import pyqtgraph as pg @@ -13,12 +13,13 @@ import numpy as np # Main window class class GelBox(QDialog): - def __init__(self, parent): + def __init__(self, parent, img_path): super().__init__(parent) # setting title self.setWindowTitle("PyQtGraph") + self.img_path = img_path # setting geometry - self.setGeometry(100, 100, 600, 500) + self.setGeometry(50, 50, 1200, 900) # icon icon = QIcon("skin.png") # setting icon to the window @@ -35,7 +36,7 @@ class GelBox(QDialog): pg.setConfigOptions(antialias=True) # creating image view object self.imv = pg.ImageView() - img = np.array(Image.open("C:\\Users\\lwark\\Desktop\\PLATE1_17012024_103607AM_1_4x26.jpg").rotate(-90).transpose(Image.FLIP_LEFT_RIGHT)) + img = np.array(Image.open(self.img_path).rotate(-90).transpose(Image.FLIP_LEFT_RIGHT)) self.imv.setImage(img)#, xvals=np.linspace(1., 3., data.shape[0])) layout = QGridLayout() # setting this layout to the widget @@ -54,7 +55,7 @@ class GelBox(QDialog): self.setLayout(layout) def parse_form(self): - return self.form.parse_form() + return self.img_path, self.form.parse_form() class ControlsForm(QWidget): @@ -79,6 +80,7 @@ class ControlsForm(QWidget): for iii in range(3): for jjj in range(3): widge = QLineEdit() + widge.setText("Neg") widge.setObjectName(f"{rows[iii]} : {columns[jjj]}") self.layout.addWidget(widge, iii+1, jjj+2, 1, 1) self.setLayout(self.layout) diff --git a/src/submissions/frontend/widgets/submission_widget.py b/src/submissions/frontend/widgets/submission_widget.py index 10e6f0c..5d2de2f 100644 --- a/src/submissions/frontend/widgets/submission_widget.py +++ b/src/submissions/frontend/widgets/submission_widget.py @@ -312,11 +312,12 @@ class SubmissionFormContainer(QWidget): # update_last_used(reagent=reagent, kit=base_submission.extraction_kit) reagent.update_last_used(kit=base_submission.extraction_kit) # sys.exit() - logger.debug(f"Here is the final submission: {pformat(base_submission.__dict__)}") - logger.debug(f"Parsed reagents: {pformat(base_submission.reagents)}") - logger.debug(f"Sending submission: {base_submission.rsl_plate_num} to database.") - logger.debug(f"Samples from pyd: {pformat(self.pyd.samples)}") - logger.debug(f"Samples SQL: {pformat([item.__dict__ for item in base_submission.samples])}") + # logger.debug(f"Here is the final submission: {pformat(base_submission.__dict__)}") + # logger.debug(f"Parsed reagents: {pformat(base_submission.reagents)}") + # logger.debug(f"Sending submission: {base_submission.rsl_plate_num} to database.") + # logger.debug(f"Samples from pyd: {pformat(self.pyd.samples)}") + # logger.debug(f"Samples SQL: {pformat([item.__dict__ for item in base_submission.samples])}") + # logger.debug(f"") base_submission.save() # update summary sheet self.app.table_widget.sub_wid.setData() diff --git a/src/submissions/templates/tooltip.html b/src/submissions/templates/tooltip.html index 0b473f2..285d2c4 100644 --- a/src/submissions/templates/tooltip.html +++ b/src/submissions/templates/tooltip.html @@ -1,4 +1,4 @@ Sample name: {{ fields['submitter_id'] }}
{% if fields['organism'] %}Organism: {{ fields['organism'] }}
{% endif %} {% if fields['concentration'] %}Concentration: {{ fields['concentration'] }}
{% endif %} -Well: {{ fields['row'] }}{{ fields['column'] }} \ No newline at end of file +Well: {{ fields['well'] }} \ No newline at end of file diff --git a/src/submissions/tools.py b/src/submissions/tools.py index 9a3b6ef..2ac45b6 100644 --- a/src/submissions/tools.py +++ b/src/submissions/tools.py @@ -357,7 +357,7 @@ def copy_settings(settings_path:Path, settings:dict) -> dict: yaml.dump(settings, f) return settings -def jinja_template_loading(): +def jinja_template_loading() -> Environment: """ Returns jinja2 template environment.