landon/Submissions-App
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pending large code cleanup

Browse Source
This commit is contained in:
Landon Wark
2024-01-31 15:21:07 -06:00
parent 319f72cab2
commit eda62fba5a
19 changed files with 741 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGELOG.md
View File

@@ -1,3 +1,7 @@
## 202401.04
- Large scale database refactor to increase modularity.
## 202401.01
- Improved tooltips and form regeneration.

4
alembic.ini
View File

@@ -55,8 +55,8 @@ version_path_separator = os # Use os.pathsep. Default configuration used for ne
# are written from script.py.mako
# output_encoding = utf-8
; sqlalchemy.url = sqlite:///L:\Robotics Laboratory Support\Submissions\submissions.db
sqlalchemy.url = sqlite:///C:\Users\lwark\Documents\Archives\Submissions_app_backups\DB_backups\submissions-new.db
sqlalchemy.url = sqlite:///L:\Robotics Laboratory Support\Submissions\submissions.db
; sqlalchemy.url = sqlite:///C:\Users\lwark\Documents\Archives\Submissions_app_backups\DB_backups\submissions-new.db
; sqlalchemy.url = sqlite:///C:\Users\lwark\Documents\python\submissions\tests\test_assets\submissions-test.db

34
alembic/versions/70426df72f80_adding_gel_image_info_again.py Normal file
View File

@@ -0,0 +1,34 @@
"""adding gel image, info. Again
Revision ID: 70426df72f80
Revises: c4201b0ea9fe
Create Date: 2024-01-30 08:47:22.809841
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '70426df72f80'
down_revision = 'c4201b0ea9fe'
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('_wastewaterartic', schema=None) as batch_op:
batch_op.add_column(sa.Column('gel_image', sa.String(length=64), nullable=True))
batch_op.add_column(sa.Column('gel_info', sa.JSON(), nullable=True))
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('_wastewaterartic', schema=None) as batch_op:
batch_op.drop_column('gel_info')
batch_op.drop_column('gel_image')
# ### end Alembic commands ###

38
alembic/versions/70d5a751f579_tweaking_submission_sample_association.py Normal file
View File

@@ -0,0 +1,38 @@
"""tweaking submission sample association
Revision ID: 70d5a751f579
Revises: 97392dda5436
Create Date: 2024-01-25 13:39:34.163501
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '70d5a751f579'
down_revision = '97392dda5436'
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('_submissionsampleassociation', schema=None) as batch_op:
batch_op.alter_column('id',
existing_type=sa.INTEGER(),
nullable=False)
batch_op.create_unique_constraint("ssa_id_unique", ['id'])
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('_submissionsampleassociation', schema=None) as batch_op:
batch_op.drop_constraint("ssa_id_unique", type_='unique')
batch_op.alter_column('id',
existing_type=sa.INTEGER(),
nullable=False)
# ### end Alembic commands ###

50
alembic/versions/97392dda5436_update_to_submissionsampleassociation.py Normal file
View File

@@ -0,0 +1,50 @@
"""Update to submissionsampleassociation
Revision ID: 97392dda5436
Revises: e3f6770ef515
Create Date: 2024-01-25 09:10:04.384194
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '97392dda5436'
down_revision = 'e3f6770ef515'
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('_submissionsampleassociation', schema=None) as batch_op:
batch_op.add_column(sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=True))
batch_op.create_unique_constraint("submissionsampleassociation_id", ['id'])
with op.batch_alter_table('_wastewaterassociation', schema=None) as batch_op:
batch_op.add_column(sa.Column('id', sa.INTEGER(), nullable=False))
# batch_op.drop_constraint("sample_id", type_='foreignkey')
# batch_op.drop_constraint("submission_id", type_='foreignkey')
batch_op.create_foreign_key("fk_subsampassoc_id", '_submissionsampleassociation', ['id'], ['id'])
# batch_op.drop_column('sample_id')
# batch_op.drop_column('submission_id')
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('_wastewaterassociation', schema=None) as batch_op:
batch_op.add_column(sa.Column('submission_id', sa.INTEGER(), nullable=False))
batch_op.add_column(sa.Column('sample_id', sa.INTEGER(), nullable=False))
batch_op.drop_constraint(None, type_='foreignkey')
batch_op.create_foreign_key(None, '_submissionsampleassociation', ['submission_id'], ['submission_id'])
batch_op.create_foreign_key(None, '_submissionsampleassociation', ['sample_id'], ['sample_id'])
batch_op.drop_column('id')
with op.batch_alter_table('_submissionsampleassociation', schema=None) as batch_op:
batch_op.drop_constraint(None, type_='unique')
batch_op.drop_column('id')
# ### end Alembic commands ###

42
alembic/versions/c4201b0ea9fe_adding_gel_image_info.py Normal file
View File

@@ -0,0 +1,42 @@
"""adding gel image, info
Revision ID: c4201b0ea9fe
Revises: 70d5a751f579
Create Date: 2024-01-30 08:42:03.928933
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'c4201b0ea9fe'
down_revision = '70d5a751f579'
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('_submissionsampleassociation', schema=None) as batch_op:
batch_op.create_unique_constraint("unique_ssa_id", ['id'])
with op.batch_alter_table('_wastewaterassociation', schema=None) as batch_op:
batch_op.alter_column('id',
existing_type=sa.INTEGER(),
nullable=False)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
with op.batch_alter_table('_wastewaterassociation', schema=None) as batch_op:
batch_op.alter_column('id',
existing_type=sa.INTEGER(),
nullable=True)
with op.batch_alter_table('_submissionsampleassociation', schema=None) as batch_op:
batch_op.drop_constraint("unique_ssa_id", type_='unique')
# ### end Alembic commands ###

340
alembic/versions/e3f6770ef515_first_commit.py Normal file
View File

@@ -0,0 +1,340 @@
"""First Commit
Revision ID: e3f6770ef515
Revises:
Create Date: 2024-01-22 14:01:02.958292
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'e3f6770ef515'
down_revision = None
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('_basicsample',
sa.Column('id', sa.INTEGER(), nullable=False),
sa.Column('submitter_id', sa.String(length=64), nullable=False),
sa.Column('sample_type', sa.String(length=32), nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('submitter_id')
)
op.create_table('_contact',
sa.Column('id', sa.INTEGER(), nullable=False),
sa.Column('name', sa.String(length=64), nullable=True),
sa.Column('email', sa.String(length=64), nullable=True),
sa.Column('phone', sa.String(length=32), nullable=True),
sa.PrimaryKeyConstraint('id')
)
op.create_table('_controltype',
sa.Column('id', sa.INTEGER(), nullable=False),
sa.Column('name', sa.String(length=255), nullable=True),
sa.Column('targets', sa.JSON(), nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name')
)
op.create_table('_equipment',
sa.Column('id', sa.INTEGER(), nullable=False),
sa.Column('name', sa.String(length=64), nullable=True),
sa.Column('nickname', sa.String(length=64), nullable=True),
sa.Column('asset_number', sa.String(length=16), nullable=True),
sa.PrimaryKeyConstraint('id')
)
op.create_table('_equipmentrole',
sa.Column('id', sa.INTEGER(), nullable=False),
sa.Column('name', sa.String(length=32), nullable=True),
sa.PrimaryKeyConstraint('id')
)
op.create_table('_kittype',
sa.Column('id', sa.INTEGER(), nullable=False),
sa.Column('name', sa.String(length=64), nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name')
)
op.create_table('_organization',
sa.Column('id', sa.INTEGER(), nullable=False),
sa.Column('name', sa.String(length=64), nullable=True),
sa.Column('cost_centre', sa.String(), nullable=True),
sa.PrimaryKeyConstraint('id')
)
op.create_table('_process',
sa.Column('id', sa.INTEGER(), nullable=False),
sa.Column('name', sa.String(length=64), nullable=True),
sa.PrimaryKeyConstraint('id')
)
op.create_table('_reagenttype',
sa.Column('id', sa.INTEGER(), nullable=False),
sa.Column('name', sa.String(length=64), nullable=True),
sa.Column('eol_ext', sa.Interval(), nullable=True),
sa.PrimaryKeyConstraint('id')
)
op.create_table('_submissiontype',
sa.Column('id', sa.INTEGER(), nullable=False),
sa.Column('name', sa.String(length=128), nullable=True),
sa.Column('info_map', sa.JSON(), nullable=True),
sa.Column('template_file', sa.BLOB(), nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name')
)
op.create_table('_bacterialculturesample',
sa.Column('id', sa.INTEGER(), nullable=False),
sa.Column('organism', sa.String(length=64), nullable=True),
sa.Column('concentration', sa.String(length=16), nullable=True),
sa.ForeignKeyConstraint(['id'], ['_basicsample.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_table('_discount',
sa.Column('id', sa.INTEGER(), nullable=False),
sa.Column('kit_id', sa.INTEGER(), nullable=True),
sa.Column('client_id', sa.INTEGER(), nullable=True),
sa.Column('name', sa.String(length=128), nullable=True),
sa.Column('amount', sa.FLOAT(precision=2), nullable=True),
sa.ForeignKeyConstraint(['client_id'], ['_organization.id'], name='fk_org_id', ondelete='SET NULL'),
sa.ForeignKeyConstraint(['kit_id'], ['_kittype.id'], name='fk_kit_type_id', ondelete='SET NULL'),
sa.PrimaryKeyConstraint('id')
)
op.create_table('_equipment_processes',
sa.Column('process_id', sa.INTEGER(), nullable=True),
sa.Column('equipment_id', sa.INTEGER(), nullable=True),
sa.ForeignKeyConstraint(['equipment_id'], ['_equipment.id'], ),
sa.ForeignKeyConstraint(['process_id'], ['_process.id'], )
)
op.create_table('_equipmentroles_equipment',
sa.Column('equipment_id', sa.INTEGER(), nullable=True),
sa.Column('equipmentroles_id', sa.INTEGER(), nullable=True),
sa.ForeignKeyConstraint(['equipment_id'], ['_equipment.id'], ),
sa.ForeignKeyConstraint(['equipmentroles_id'], ['_equipmentrole.id'], )
)
op.create_table('_equipmentroles_processes',
sa.Column('process_id', sa.INTEGER(), nullable=True),
sa.Column('equipmentrole_id', sa.INTEGER(), nullable=True),
sa.ForeignKeyConstraint(['equipmentrole_id'], ['_equipmentrole.id'], ),
sa.ForeignKeyConstraint(['process_id'], ['_process.id'], )
)
op.create_table('_kittypereagenttypeassociation',
sa.Column('reagent_types_id', sa.INTEGER(), nullable=False),
sa.Column('kits_id', sa.INTEGER(), nullable=False),
sa.Column('submission_type_id', sa.INTEGER(), nullable=False),
sa.Column('uses', sa.JSON(), nullable=True),
sa.Column('required', sa.INTEGER(), nullable=True),
sa.Column('last_used', sa.String(length=32), nullable=True),
sa.ForeignKeyConstraint(['kits_id'], ['_kittype.id'], ),
sa.ForeignKeyConstraint(['reagent_types_id'], ['_reagenttype.id'], ),
sa.ForeignKeyConstraint(['submission_type_id'], ['_submissiontype.id'], ),
sa.PrimaryKeyConstraint('reagent_types_id', 'kits_id', 'submission_type_id')
)
op.create_table('_kittypes_processes',
sa.Column('process_id', sa.INTEGER(), nullable=True),
sa.Column('kit_id', sa.INTEGER(), nullable=True),
sa.ForeignKeyConstraint(['kit_id'], ['_kittype.id'], ),
sa.ForeignKeyConstraint(['process_id'], ['_process.id'], )
)
op.create_table('_orgs_contacts',
sa.Column('org_id', sa.INTEGER(), nullable=True),
sa.Column('contact_id', sa.INTEGER(), nullable=True),
sa.ForeignKeyConstraint(['contact_id'], ['_contact.id'], ),
sa.ForeignKeyConstraint(['org_id'], ['_organization.id'], )
)
op.create_table('_reagent',
sa.Column('id', sa.INTEGER(), nullable=False),
sa.Column('type_id', sa.INTEGER(), nullable=True),
sa.Column('name', sa.String(length=64), nullable=True),
sa.Column('lot', sa.String(length=64), nullable=True),
sa.Column('expiry', sa.TIMESTAMP(), nullable=True),
sa.ForeignKeyConstraint(['type_id'], ['_reagenttype.id'], name='fk_reagent_type_id', ondelete='SET NULL'),
sa.PrimaryKeyConstraint('id')
)
op.create_table('_submissiontypeequipmentroleassociation',
sa.Column('equipmentrole_id', sa.INTEGER(), nullable=False),
sa.Column('submissiontype_id', sa.INTEGER(), nullable=False),
sa.Column('uses', sa.JSON(), nullable=True),
sa.Column('static', sa.INTEGER(), nullable=True),
sa.ForeignKeyConstraint(['equipmentrole_id'], ['_equipmentrole.id'], ),
sa.ForeignKeyConstraint(['submissiontype_id'], ['_submissiontype.id'], ),
sa.PrimaryKeyConstraint('equipmentrole_id', 'submissiontype_id')
)
op.create_table('_submissiontypekittypeassociation',
sa.Column('submission_types_id', sa.INTEGER(), nullable=False),
sa.Column('kits_id', sa.INTEGER(), nullable=False),
sa.Column('mutable_cost_column', sa.FLOAT(precision=2), nullable=True),
sa.Column('mutable_cost_sample', sa.FLOAT(precision=2), nullable=True),
sa.Column('constant_cost', sa.FLOAT(precision=2), nullable=True),
sa.ForeignKeyConstraint(['kits_id'], ['_kittype.id'], ),
sa.ForeignKeyConstraint(['submission_types_id'], ['_submissiontype.id'], ),
sa.PrimaryKeyConstraint('submission_types_id', 'kits_id')
)
op.create_table('_submissiontypes_processes',
sa.Column('process_id', sa.INTEGER(), nullable=True),
sa.Column('equipmentroles_id', sa.INTEGER(), nullable=True),
sa.ForeignKeyConstraint(['equipmentroles_id'], ['_submissiontype.id'], ),
sa.ForeignKeyConstraint(['process_id'], ['_process.id'], )
)
op.create_table('_wastewatersample',
sa.Column('id', sa.INTEGER(), nullable=False),
sa.Column('ww_processing_num', sa.String(length=64), nullable=True),
sa.Column('ww_full_sample_id', sa.String(length=64), nullable=True),
sa.Column('rsl_number', sa.String(length=64), nullable=True),
sa.Column('collection_date', sa.TIMESTAMP(), nullable=True),
sa.Column('received_date', sa.TIMESTAMP(), nullable=True),
sa.Column('notes', sa.String(length=2000), nullable=True),
sa.Column('sample_location', sa.String(length=8), nullable=True),
sa.ForeignKeyConstraint(['id'], ['_basicsample.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_table('_basicsubmission',
sa.Column('id', sa.INTEGER(), nullable=False),
sa.Column('rsl_plate_num', sa.String(length=32), nullable=False),
sa.Column('submitter_plate_num', sa.String(length=127), nullable=True),
sa.Column('submitted_date', sa.TIMESTAMP(), nullable=True),
sa.Column('submitting_lab_id', sa.INTEGER(), nullable=True),
sa.Column('sample_count', sa.INTEGER(), nullable=True),
sa.Column('extraction_kit_id', sa.INTEGER(), nullable=True),
sa.Column('submission_type_name', sa.String(), nullable=True),
sa.Column('technician', sa.String(length=64), nullable=True),
sa.Column('reagents_id', sa.String(), nullable=True),
sa.Column('extraction_info', sa.JSON(), nullable=True),
sa.Column('pcr_info', sa.JSON(), nullable=True),
sa.Column('run_cost', sa.FLOAT(precision=2), nullable=True),
sa.Column('uploaded_by', sa.String(length=32), nullable=True),
sa.Column('comment', sa.JSON(), nullable=True),
sa.Column('submission_category', sa.String(length=64), nullable=True),
sa.ForeignKeyConstraint(['extraction_kit_id'], ['_kittype.id'], name='fk_BS_extkit_id', ondelete='SET NULL'),
sa.ForeignKeyConstraint(['reagents_id'], ['_reagent.id'], name='fk_BS_reagents_id', ondelete='SET NULL'),
sa.ForeignKeyConstraint(['submission_type_name'], ['_submissiontype.name'], name='fk_BS_subtype_name', ondelete='SET NULL'),
sa.ForeignKeyConstraint(['submitting_lab_id'], ['_organization.id'], name='fk_BS_sublab_id', ondelete='SET NULL'),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('rsl_plate_num'),
sa.UniqueConstraint('submitter_plate_num')
)
op.create_table('_reagenttypes_reagents',
sa.Column('reagent_id', sa.INTEGER(), nullable=True),
sa.Column('reagenttype_id', sa.INTEGER(), nullable=True),
sa.ForeignKeyConstraint(['reagent_id'], ['_reagent.id'], ),
sa.ForeignKeyConstraint(['reagenttype_id'], ['_reagenttype.id'], )
)
op.create_table('_bacterialculture',
sa.Column('id', sa.INTEGER(), nullable=False),
sa.ForeignKeyConstraint(['id'], ['_basicsubmission.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_table('_control',
sa.Column('id', sa.INTEGER(), nullable=False),
sa.Column('parent_id', sa.String(), nullable=True),
sa.Column('name', sa.String(length=255), nullable=True),
sa.Column('submitted_date', sa.TIMESTAMP(), nullable=True),
sa.Column('contains', sa.JSON(), nullable=True),
sa.Column('matches', sa.JSON(), nullable=True),
sa.Column('kraken', sa.JSON(), nullable=True),
sa.Column('submission_id', sa.INTEGER(), nullable=True),
sa.Column('refseq_version', sa.String(length=16), nullable=True),
sa.Column('kraken2_version', sa.String(length=16), nullable=True),
sa.Column('kraken2_db_version', sa.String(length=32), nullable=True),
sa.Column('sample_id', sa.INTEGER(), nullable=True),
sa.ForeignKeyConstraint(['parent_id'], ['_controltype.id'], name='fk_control_parent_id'),
sa.ForeignKeyConstraint(['sample_id'], ['_basicsample.id'], name='cont_BCS_id', ondelete='SET NULL'),
sa.ForeignKeyConstraint(['submission_id'], ['_basicsubmission.id'], ),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name')
)
op.create_table('_submissionequipmentassociation',
sa.Column('equipment_id', sa.INTEGER(), nullable=False),
sa.Column('submission_id', sa.INTEGER(), nullable=False),
sa.Column('role', sa.String(length=64), nullable=False),
sa.Column('process_id', sa.INTEGER(), nullable=True),
sa.Column('start_time', sa.TIMESTAMP(), nullable=True),
sa.Column('end_time', sa.TIMESTAMP(), nullable=True),
sa.Column('comments', sa.String(length=1024), nullable=True),
sa.ForeignKeyConstraint(['equipment_id'], ['_equipment.id'], ),
sa.ForeignKeyConstraint(['process_id'], ['_process.id'], name='SEA_Process_id', ondelete='SET NULL'),
sa.ForeignKeyConstraint(['submission_id'], ['_basicsubmission.id'], ),
sa.PrimaryKeyConstraint('equipment_id', 'submission_id', 'role')
)
op.create_table('_submissionreagentassociation',
sa.Column('reagent_id', sa.INTEGER(), nullable=False),
sa.Column('submission_id', sa.INTEGER(), nullable=False),
sa.Column('comments', sa.String(length=1024), nullable=True),
sa.ForeignKeyConstraint(['reagent_id'], ['_reagent.id'], ),
sa.ForeignKeyConstraint(['submission_id'], ['_basicsubmission.id'], ),
sa.PrimaryKeyConstraint('reagent_id', 'submission_id')
)
op.create_table('_submissionsampleassociation',
sa.Column('sample_id', sa.INTEGER(), nullable=False),
sa.Column('submission_id', sa.INTEGER(), nullable=False),
sa.Column('row', sa.INTEGER(), nullable=False),
sa.Column('column', sa.INTEGER(), nullable=False),
sa.Column('base_sub_type', sa.String(), nullable=True),
sa.ForeignKeyConstraint(['sample_id'], ['_basicsample.id'], ),
sa.ForeignKeyConstraint(['submission_id'], ['_basicsubmission.id'], ),
sa.PrimaryKeyConstraint('submission_id', 'row', 'column')
)
op.create_table('_wastewater',
sa.Column('id', sa.INTEGER(), nullable=False),
sa.Column('ext_technician', sa.String(length=64), nullable=True),
sa.Column('pcr_technician', sa.String(length=64), nullable=True),
sa.ForeignKeyConstraint(['id'], ['_basicsubmission.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_table('_wastewaterartic',
sa.Column('id', sa.INTEGER(), nullable=False),
sa.Column('artic_technician', sa.String(length=64), nullable=True),
sa.Column('dna_core_submission_number', sa.String(length=64), nullable=True),
sa.ForeignKeyConstraint(['id'], ['_basicsubmission.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_table('_wastewaterassociation',
sa.Column('sample_id', sa.INTEGER(), nullable=False),
sa.Column('submission_id', sa.INTEGER(), nullable=False),
sa.Column('ct_n1', sa.FLOAT(precision=2), nullable=True),
sa.Column('ct_n2', sa.FLOAT(precision=2), nullable=True),
sa.Column('n1_status', sa.String(length=32), nullable=True),
sa.Column('n2_status', sa.String(length=32), nullable=True),
sa.Column('pcr_results', sa.JSON(), nullable=True),
sa.ForeignKeyConstraint(['sample_id'], ['_submissionsampleassociation.sample_id'], ),
sa.ForeignKeyConstraint(['submission_id'], ['_submissionsampleassociation.submission_id'], ),
sa.PrimaryKeyConstraint('sample_id', 'submission_id')
)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('_wastewaterassociation')
op.drop_table('_wastewaterartic')
op.drop_table('_wastewater')
op.drop_table('_submissionsampleassociation')
op.drop_table('_submissionreagentassociation')
op.drop_table('_submissionequipmentassociation')
op.drop_table('_control')
op.drop_table('_bacterialculture')
op.drop_table('_reagenttypes_reagents')
op.drop_table('_basicsubmission')
op.drop_table('_wastewatersample')
op.drop_table('_submissiontypes_processes')
op.drop_table('_submissiontypekittypeassociation')
op.drop_table('_submissiontypeequipmentroleassociation')
op.drop_table('_reagent')
op.drop_table('_orgs_contacts')
op.drop_table('_kittypes_processes')
op.drop_table('_kittypereagenttypeassociation')
op.drop_table('_equipmentroles_processes')
op.drop_table('_equipmentroles_equipment')
op.drop_table('_equipment_processes')
op.drop_table('_discount')
op.drop_table('_bacterialculturesample')
op.drop_table('_submissiontype')
op.drop_table('_reagenttype')
op.drop_table('_process')
op.drop_table('_organization')
op.drop_table('_kittype')
op.drop_table('_equipmentrole')
op.drop_table('_equipment')
op.drop_table('_controltype')
op.drop_table('_contact')
op.drop_table('_basicsample')
# ### end Alembic commands ###

2
src/submissions/__init__.py
View File

@@ -4,7 +4,7 @@ from pathlib import Path
# Version of the realpython-reader package
__project__ = "submissions"
__version__ = "202401.2b"
__version__ = "202401.4b"
__author__ = {"name":"Landon Wark", "email":"Landon.Wark@phac-aspc.gc.ca"}
__copyright__ = "2022-2024, Government of Canada"

2
src/submissions/backend/db/models/__init__.py
View File

@@ -50,7 +50,7 @@ class BaseClass(Base):
return ctx.backup_path
def save(self):
logger.debug(f"Saving {self}")
# logger.debug(f"Saving {self}")
try:
self.__database_session__.add(self)
self.__database_session__.commit()

6
src/submissions/backend/db/models/controls.py
View File

@@ -78,20 +78,20 @@ class Control(BaseClass):
# __tablename__ = '_control_samples'
id = Column(INTEGER, primary_key=True) #: primary key
parent_id = Column(String, ForeignKey("_control_types.id", name="fk_control_parent_id")) #: primary key of control type
parent_id = Column(String, ForeignKey("_controltype.id", name="fk_control_parent_id")) #: primary key of control type
controltype = relationship("ControlType", back_populates="instances", foreign_keys=[parent_id]) #: reference to parent control type
name = Column(String(255), unique=True) #: Sample ID
submitted_date = Column(TIMESTAMP) #: Date submitted to Robotics
contains = Column(JSON) #: unstructured hashes in contains.tsv for each organism
matches = Column(JSON) #: unstructured hashes in matches.tsv for each organism
kraken = Column(JSON) #: unstructured output from kraken_report
submission_id = Column(INTEGER, ForeignKey("_submissions.id")) #: parent submission id
submission_id = Column(INTEGER, ForeignKey("_basicsubmission.id")) #: parent submission id
submission = relationship("BacterialCulture", back_populates="controls", foreign_keys=[submission_id]) #: parent submission
refseq_version = Column(String(16)) #: version of refseq used in fastq parsing
kraken2_version = Column(String(16)) #: version of kraken2 used in fastq parsing
kraken2_db_version = Column(String(32)) #: folder name of kraken2 db
sample = relationship("BacterialCultureSample", back_populates="control")
sample_id = Column(INTEGER, ForeignKey("_samples.id", ondelete="SET NULL", name="cont_BCS_id"))
sample_id = Column(INTEGER, ForeignKey("_basicsample.id", ondelete="SET NULL", name="cont_BCS_id"))
def __repr__(self) -> str:
return f"<Control({self.name})>"

45
src/submissions/backend/db/models/kits.py
View File

@@ -18,8 +18,8 @@ logger = logging.getLogger(f'submissions.{__name__}')
reagenttypes_reagents = Table(
"_reagenttypes_reagents",
Base.metadata,
Column("reagent_id", INTEGER, ForeignKey("_reagents.id")),
Column("reagenttype_id", INTEGER, ForeignKey("_reagent_types.id")),
Column("reagent_id", INTEGER, ForeignKey("_reagent.id")),
Column("reagenttype_id", INTEGER, ForeignKey("_reagenttype.id")),
extend_existing = True
)
@@ -27,7 +27,7 @@ equipmentroles_equipment = Table(
"_equipmentroles_equipment",
Base.metadata,
Column("equipment_id", INTEGER, ForeignKey("_equipment.id")),
Column("equipmentroles_id", INTEGER, ForeignKey("_equipment_roles.id")),
Column("equipmentroles_id", INTEGER, ForeignKey("_equipmentrole.id")),
extend_existing=True
)
@@ -43,7 +43,7 @@ equipmentroles_processes = Table(
"_equipmentroles_processes",
Base.metadata,
Column("process_id", INTEGER, ForeignKey("_process.id")),
Column("equipmentrole_id", INTEGER, ForeignKey("_equipment_roles.id")),
Column("equipmentrole_id", INTEGER, ForeignKey("_equipmentrole.id")),
extend_existing=True
)
@@ -51,7 +51,7 @@ submissiontypes_processes = Table(
"_submissiontypes_processes",
Base.metadata,
Column("process_id", INTEGER, ForeignKey("_process.id")),
Column("equipmentroles_id", INTEGER, ForeignKey("_submission_types.id")),
Column("equipmentroles_id", INTEGER, ForeignKey("_submissiontype.id")),
extend_existing=True
)
@@ -59,7 +59,7 @@ kittypes_processes = Table(
"_kittypes_processes",
Base.metadata,
Column("process_id", INTEGER, ForeignKey("_process.id")),
Column("kit_id", INTEGER, ForeignKey("_kits.id")),
Column("kit_id", INTEGER, ForeignKey("_kittype.id")),
extend_existing=True
)
@@ -304,7 +304,7 @@ class Reagent(BaseClass):
id = Column(INTEGER, primary_key=True) #: primary key
type = relationship("ReagentType", back_populates="instances", secondary=reagenttypes_reagents) #: joined parent reagent type
type_id = Column(INTEGER, ForeignKey("_reagent_types.id", ondelete='SET NULL', name="fk_reagent_type_id")) #: id of parent reagent type
type_id = Column(INTEGER, ForeignKey("_reagenttype.id", ondelete='SET NULL', name="fk_reagent_type_id")) #: id of parent reagent type
name = Column(String(64)) #: reagent name
lot = Column(String(64)) #: lot number of reagent
expiry = Column(TIMESTAMP) #: expiry date - extended by eol_ext of parent programmatically
@@ -442,9 +442,9 @@ class Discount(BaseClass):
id = Column(INTEGER, primary_key=True) #: primary key
kit = relationship("KitType") #: joined parent reagent type
kit_id = Column(INTEGER, ForeignKey("_kits.id", ondelete='SET NULL', name="fk_kit_type_id")) #: id of joined kit
kit_id = Column(INTEGER, ForeignKey("_kittype.id", ondelete='SET NULL', name="fk_kit_type_id")) #: id of joined kit
client = relationship("Organization") #: joined client lab
client_id = Column(INTEGER, ForeignKey("_organizations.id", ondelete='SET NULL', name="fk_org_id")) #: id of joined client
client_id = Column(INTEGER, ForeignKey("_organization.id", ondelete='SET NULL', name="fk_org_id")) #: id of joined client
name = Column(String(128)) #: Short description
amount = Column(FLOAT(2)) #: Dollar amount of discount
@@ -625,8 +625,9 @@ class SubmissionType(BaseClass):
"""
Adds this instances to the database and commits.
"""
self.__database_session__.add(self)
self.__database_session__.commit()
# self.__database_session__.add(self)
# self.__database_session__.commit()
super().save()
class SubmissionTypeKitTypeAssociation(BaseClass):
"""
@@ -634,8 +635,8 @@ class SubmissionTypeKitTypeAssociation(BaseClass):
"""
# __tablename__ = "_submissiontypes_kittypes"
submission_types_id = Column(INTEGER, ForeignKey("_submission_types.id"), primary_key=True) #: id of joined submission type
kits_id = Column(INTEGER, ForeignKey("_kits.id"), primary_key=True) #: id of joined kit
submission_types_id = Column(INTEGER, ForeignKey("_submissiontype.id"), primary_key=True) #: id of joined submission type
kits_id = Column(INTEGER, ForeignKey("_kittype.id"), primary_key=True) #: id of joined kit
mutable_cost_column = Column(FLOAT(2)) #: dollar amount per 96 well plate that can change with number of columns (reagents, tips, etc)
mutable_cost_sample = Column(FLOAT(2)) #: dollar amount that can change with number of samples (reagents, tips, etc)
constant_cost = Column(FLOAT(2)) #: dollar amount per plate that will remain constant (plates, man hours, etc)
@@ -707,9 +708,9 @@ class KitTypeReagentTypeAssociation(BaseClass):
"""
# __tablename__ = "_reagenttypes_kittypes"
reagent_types_id = Column(INTEGER, ForeignKey("_reagent_types.id"), primary_key=True) #: id of associated reagent type
kits_id = Column(INTEGER, ForeignKey("_kits.id"), primary_key=True) #: id of associated reagent type
submission_type_id = Column(INTEGER, ForeignKey("_submission_types.id"), primary_key=True)
reagent_types_id = Column(INTEGER, ForeignKey("_reagenttype.id"), primary_key=True) #: id of associated reagent type
kits_id = Column(INTEGER, ForeignKey("_kittype.id"), primary_key=True) #: id of associated reagent type
submission_type_id = Column(INTEGER, ForeignKey("_submissiontype.id"), primary_key=True)
uses = Column(JSON) #: map to location on excel sheets of different submission types
required = Column(INTEGER) #: whether the reagent type is required for the kit (Boolean 1 or 0)
last_used = Column(String(32)) #: last used lot number of this type of reagent
@@ -810,8 +811,8 @@ class SubmissionReagentAssociation(BaseClass):
# __tablename__ = "_reagents_submissions"
reagent_id = Column(INTEGER, ForeignKey("_reagents.id"), primary_key=True) #: id of associated sample
submission_id = Column(INTEGER, ForeignKey("_submissions.id"), primary_key=True)
reagent_id = Column(INTEGER, ForeignKey("_reagent.id"), primary_key=True) #: id of associated sample
submission_id = Column(INTEGER, ForeignKey("_basicsubmission.id"), primary_key=True)
comments = Column(String(1024))
submission = relationship("BasicSubmission", back_populates="submission_reagent_associations") #: associated submission
@@ -1060,7 +1061,7 @@ class SubmissionEquipmentAssociation(BaseClass):
# __tablename__ = "_equipment_submissions"
equipment_id = Column(INTEGER, ForeignKey("_equipment.id"), primary_key=True) #: id of associated equipment
submission_id = Column(INTEGER, ForeignKey("_submissions.id"), primary_key=True) #: id of associated submission
submission_id = Column(INTEGER, ForeignKey("_basicsubmission.id"), primary_key=True) #: id of associated submission
role = Column(String(64), primary_key=True) #: name of the role the equipment fills
# process = Column(String(64)) #: name of the process run on this equipment
process_id = Column(INTEGER, ForeignKey("_process.id",ondelete="SET NULL", name="SEA_Process_id"))
@@ -1090,8 +1091,8 @@ class SubmissionTypeEquipmentRoleAssociation(BaseClass):
# __tablename__ = "_submissiontype_equipmentrole"
equipmentrole_id = Column(INTEGER, ForeignKey("_equipment_roles.id"), primary_key=True) #: id of associated equipment
submissiontype_id = Column(INTEGER, ForeignKey("_submission_types.id"), primary_key=True) #: id of associated submission
equipmentrole_id = Column(INTEGER, ForeignKey("_equipmentrole.id"), primary_key=True) #: id of associated equipment
submissiontype_id = Column(INTEGER, ForeignKey("_submissiontype.id"), primary_key=True) #: id of associated submission
uses = Column(JSON) #: locations of equipment on the submission type excel sheet.
static = Column(INTEGER, default=1) #: if 1 this piece of equipment will always be used, otherwise it will need to be selected from list?
@@ -1156,7 +1157,7 @@ class Process(BaseClass):
@classmethod
@setup_lookup
def query(cls, name:str|None, limit:int=0):
def query(cls, name:str|None=None, limit:int=0):
query = cls.__database_session__.query(cls)
match name:
case str():

4
src/submissions/backend/db/models/organizations.py
View File

@@ -15,8 +15,8 @@ logger = logging.getLogger(f"submissions.{__name__}")
orgs_contacts = Table(
"_orgs_contacts",
Base.metadata,
Column("org_id", INTEGER, ForeignKey("_organizations.id")),
Column("contact_id", INTEGER, ForeignKey("_contacts.id")),
Column("org_id", INTEGER, ForeignKey("_organization.id")),
Column("contact_id", INTEGER, ForeignKey("_contact.id")),
# __table_args__ = {'extend_existing': True}
extend_existing = True
)

175
src/submissions/backend/db/models/submissions.py
View File

@@ -3,7 +3,8 @@ Models for the main submission types.
'''
from __future__ import annotations
from getpass import getuser
import math, json, logging, uuid, tempfile, re, yaml
import math, json, logging, uuid, tempfile, re, yaml, zipfile
import sys
from operator import attrgetter
from pprint import pformat
from . import Reagent, SubmissionType, KitType, Organization
@@ -13,9 +14,10 @@ from json.decoder import JSONDecodeError
from sqlalchemy.ext.associationproxy import association_proxy
import pandas as pd
from openpyxl import Workbook
from . import BaseClass, Equipment
from openpyxl.worksheet.worksheet import Worksheet
from . import BaseClass
from tools import check_not_nan, row_map, query_return, setup_lookup, jinja_template_loading
from datetime import datetime, date, time
from datetime import datetime, date
from typing import List, Any
from dateutil.parser import parse
from dateutil.parser._parser import ParserError
@@ -37,17 +39,16 @@ class BasicSubmission(BaseClass):
submitter_plate_num = Column(String(127), unique=True) #: The number given to the submission by the submitting lab
submitted_date = Column(TIMESTAMP) #: Date submission received
submitting_lab = relationship("Organization", back_populates="submissions") #: client org
submitting_lab_id = Column(INTEGER, ForeignKey("_organizations.id", ondelete="SET NULL", name="fk_BS_sublab_id")) #: client lab id from _organizations
submitting_lab_id = Column(INTEGER, ForeignKey("_organization.id", ondelete="SET NULL", name="fk_BS_sublab_id")) #: client lab id from _organizations
sample_count = Column(INTEGER) #: Number of samples in the submission
extraction_kit = relationship("KitType", back_populates="submissions") #: The extraction kit used
extraction_kit_id = Column(INTEGER, ForeignKey("_kits.id", ondelete="SET NULL", name="fk_BS_extkit_id")) #: id of joined extraction kit
submission_type_name = Column(String, ForeignKey("_submission_types.name", ondelete="SET NULL", name="fk_BS_subtype_name")) #: name of joined submission type
extraction_kit_id = Column(INTEGER, ForeignKey("_kittype.id", ondelete="SET NULL", name="fk_BS_extkit_id")) #: id of joined extraction kit
submission_type_name = Column(String, ForeignKey("_submissiontype.name", ondelete="SET NULL", name="fk_BS_subtype_name")) #: name of joined submission type
technician = Column(String(64)) #: initials of processing tech(s)
# Move this into custom types?
# reagents = relationship("Reagent", back_populates="submissions", secondary=reagents_submissions) #: relationship to reagents
reagents_id = Column(String, ForeignKey("_reagents.id", ondelete="SET NULL", name="fk_BS_reagents_id")) #: id of used reagents
reagents_id = Column(String, ForeignKey("_reagent.id", ondelete="SET NULL", name="fk_BS_reagents_id")) #: id of used reagents
extraction_info = Column(JSON) #: unstructured output from the extraction table logger.
pcr_info = Column(JSON) #: unstructured output from pcr table logger or user(Artic)
run_cost = Column(FLOAT(2)) #: total cost of running the plate. Set from constant and mutable kit costs at time of creation.
uploaded_by = Column(String(32)) #: user name of person who submitted the submission to the database.
comment = Column(JSON) #: user notes
@@ -132,19 +133,21 @@ class BasicSubmission(BaseClass):
logger.error(f"Json error in {self.rsl_plate_num}: {e}")
# Updated 2023-09 to use the extraction kit to pull reagents.
if full_data:
logger.debug(f"Attempting reagents.")
try:
reagents = [item.to_sub_dict(extraction_kit=self.extraction_kit) for item in self.submission_reagent_associations]
except Exception as e:
logger.error(f"We got an error retrieving reagents: {e}")
reagents = None
# samples = [item.sample.to_sub_dict(submission_rsl=self.rsl_plate_num) for item in self.submission_sample_associations]
logger.debug(f"Running samples.")
samples = self.adjust_to_dict_samples(backup=backup)
try:
equipment = [item.to_sub_dict() for item in self.submission_equipment_associations]
if len(equipment) == 0:
equipment = None
except Exception as e:
logger.error(f"Error setting equipment: {self.equipment}")
logger.error(f"Error setting equipment: {e}")
equipment = None
else:
reagents = None
@@ -155,7 +158,6 @@ class BasicSubmission(BaseClass):
except Exception as e:
logger.error(f"Error setting comment: {self.comment}")
comments = None
output = {
"id": self.id,
"Plate Number": self.rsl_plate_num,
@@ -440,6 +442,7 @@ class BasicSubmission(BaseClass):
def filename_template(cls) -> str:
"""
Constructs template for filename of this class.
Note: This is meant to be used with the dictionary constructed in self.to_dict(). Keys need to have spaces removed
Returns:
str: filename template in jinja friendly format.
@@ -462,6 +465,20 @@ class BasicSubmission(BaseClass):
logger.warning(f"Couldn't drop '{item}' column from submissionsheet df.")
return df
@classmethod
def custom_sample_autofill_row(cls, sample, worksheet:Worksheet) -> int:
"""
_summary_
Args:
sample (_type_): _description_
worksheet (Workbook): _description_
Returns:
int: _description_
"""
return None
def set_attribute(self, key:str, value):
"""
Performs custom attribute setting based on values.
@@ -547,7 +564,7 @@ class BasicSubmission(BaseClass):
"""
from backend.validators import PydSubmission, PydSample, PydReagent, PydEquipment
dicto = self.to_dict(full_data=True, backup=backup)
logger.debug(f"Backup dictionary: {pformat(dicto)}")
# logger.debug(f"Backup dictionary: {pformat(dicto)}")
# dicto['filepath'] = Path(tempfile.TemporaryFile().name)
new_dict = {}
for key, value in dicto.items():
@@ -572,7 +589,7 @@ class BasicSubmission(BaseClass):
# new_dict[key.lower().replace(" ", "_")]['value'] = value
# new_dict[key.lower().replace(" ", "_")]['missing'] = True
new_dict['filepath'] = Path(tempfile.TemporaryFile().name)
logger.debug(f"Dictionary coming into PydSubmission: {pformat(new_dict)}")
# logger.debug(f"Dictionary coming into PydSubmission: {pformat(new_dict)}")
# sys.exit()
return PydSubmission(**new_dict)
@@ -797,22 +814,15 @@ class BasicSubmission(BaseClass):
# logger.debug(f"Save result: {result}")
def add_equipment(self, obj):
# submission_type = submission.submission_type_name
from frontend.widgets.equipment_usage import EquipmentUsage
dlg = EquipmentUsage(parent=obj, submission_type=self.submission_type_name, submission=self)
dlg = EquipmentUsage(parent=obj, submission=self)
if dlg.exec():
equipment = dlg.parse_form()
logger.debug(f"We've got equipment: {equipment}")
for equip in equipment:
# e = Equipment.query(name=equip.name)
# assoc = SubmissionEquipmentAssociation(submission=submission, equipment=e)
# process = Process.query(name=equip.processes)
# assoc.process = process
# assoc.role = equip.role
logger.debug(f"Processing: {equip}")
_, assoc = equip.toSQL(submission=self)
# submission.submission_equipment_associations.append(assoc)
logger.debug(f"Appending SubmissionEquipmentAssociation: {assoc}")
# submission.save()
assoc.save()
else:
pass
@@ -825,12 +835,14 @@ class BasicSubmission(BaseClass):
fname (Path): Filename of xlsx file.
"""
logger.debug("Hello from backup.")
pyd = self.to_pydantic(backup=True)
if fname == None:
from frontend.widgets.functions import select_save_file
from backend.validators import RSLNamer
abbreviation = self.get_abbreviation()
file_data = dict(rsl_plate_num=self.rsl_plate_num, submission_type=self.submission_type_name, submitted_date=self.submitted_date, abbreviation=abbreviation)
fname = select_save_file(default_name=RSLNamer.construct_new_plate_name(data=file_data), extension="xlsx", obj=obj)
fname = select_save_file(default_name=pyd.construct_filename(), extension="xlsx", obj=obj)
logger.debug(fname.name)
if fname.name == "":
logger.debug(f"export cancelled.")
return
if full_backup:
backup = self.to_dict(full_data=True)
try:
@@ -838,7 +850,6 @@ class BasicSubmission(BaseClass):
yaml.dump(backup, f)
except KeyError as e:
logger.error(f"Problem saving yml backup file: {e}")
pyd = self.to_pydantic(backup=True)
wb = pyd.autofill_excel()
wb = pyd.autofill_samples(wb)
wb = pyd.autofill_equipment(wb)
@@ -856,14 +867,14 @@ class BacterialCulture(BasicSubmission):
polymorphic_load="inline",
inherit_condition=(id == BasicSubmission.id))
def to_dict(self, full_data:bool=False) -> dict:
def to_dict(self, full_data:bool=False, backup:bool=False) -> dict:
"""
Extends parent class method to add controls to dict
Returns:
dict: dictionary used in submissions summary
"""
output = super().to_dict(full_data=full_data)
output = super().to_dict(full_data=full_data, backup=backup)
if full_data:
output['controls'] = [item.to_sub_dict() for item in self.controls]
return output
@@ -996,6 +1007,22 @@ class BacterialCulture(BasicSubmission):
input_dict['submitted_date']['missing'] = True
return input_dict
@classmethod
def custom_sample_autofill_row(cls, sample, worksheet: Worksheet) -> int:
logger.debug(f"Checking {sample.well}")
logger.debug(f"here's the worksheet: {worksheet}")
row = super().custom_sample_autofill_row(sample, worksheet)
df = pd.DataFrame(list(worksheet.values))
# logger.debug(f"Here's the dataframe: {df}")
idx = df[df[0]==sample.well]
if idx.empty:
new = f"{sample.well[0]}{sample.well[1:].zfill(2)}"
logger.debug(f"Checking: {new}")
idx = df[df[0]==new]
logger.debug(f"Here is the row: {idx}")
row = idx.index.to_list()[0]
return row + 1
class Wastewater(BasicSubmission):
"""
derivative submission type from BasicSubmission
@@ -1003,11 +1030,13 @@ class Wastewater(BasicSubmission):
id = Column(INTEGER, ForeignKey('_basicsubmission.id'), primary_key=True)
ext_technician = Column(String(64))
pcr_technician = Column(String(64))
pcr_info = Column(JSON) #: unstructured output from pcr table logger or user(Artic)
__mapper_args__ = __mapper_args__ = dict(polymorphic_identity="Wastewater",
polymorphic_load="inline",
inherit_condition=(id == BasicSubmission.id))
def to_dict(self, full_data:bool=False) -> dict:
def to_dict(self, full_data:bool=False, backup:bool=False) -> dict:
"""
Extends parent class method to add controls to dict
@@ -1020,6 +1049,7 @@ class Wastewater(BasicSubmission):
except TypeError as e:
pass
output['Technician'] = f"Enr: {self.technician}, Ext: {self.ext_technician}, PCR: {self.pcr_technician}"
return output
@classmethod
@@ -1145,6 +1175,18 @@ class Wastewater(BasicSubmission):
samples = super().adjust_autofill_samples(samples)
return [item for item in samples if not item.submitter_id.startswith("EN")]
@classmethod
def custom_sample_autofill_row(cls, sample, worksheet: Worksheet) -> int:
logger.debug(f"Checking {sample.well}")
logger.debug(f"here's the worksheet: {worksheet}")
row = super().custom_sample_autofill_row(sample, worksheet)
df = pd.DataFrame(list(worksheet.values))
logger.debug(f"Here's the dataframe: {df}")
idx = df[df[1]==sample.sample_location]
logger.debug(f"Here is the row: {idx}")
row = idx.index.to_list()[0]
return row + 1
class WastewaterArtic(BasicSubmission):
"""
derivative submission type for artic wastewater
@@ -1155,6 +1197,9 @@ class WastewaterArtic(BasicSubmission):
inherit_condition=(id == BasicSubmission.id))
artic_technician = Column(String(64))
dna_core_submission_number = Column(String(64))
pcr_info = Column(JSON) #: unstructured output from pcr table logger or user(Artic)
gel_image = Column(String(64))
gel_info = Column(JSON)
def calculate_base_cost(self):
"""
@@ -1381,10 +1426,19 @@ class WastewaterArtic(BasicSubmission):
def gel_box(self, obj):
from frontend.widgets.gel_checker import GelBox
dlg = GelBox(parent=obj)
from frontend.widgets import select_open_file
fname = select_open_file(obj=obj, file_extension="jpg")
dlg = GelBox(parent=obj, img_path=fname)
if dlg.exec():
output = dlg.parse_form()
print(output)
img_path, output = dlg.parse_form()
self.gel_image = img_path.name
self.gel_info = output
with zipfile.ZipFile(self.__directory_path__.joinpath("submission_imgs.zip"), 'a') as zipf:
# Add a file located at the source_path to the destination within the zip
# file. It will overwrite existing files if the names collide, but it
# will give a warning
zipf.write(img_path, self.gel_image)
self.save()
# Sample Classes
@@ -1439,7 +1493,10 @@ class BasicSample(BaseClass):
return value
def __repr__(self) -> str:
try:
return f"<{self.sample_type.replace('_', ' ').title().replace(' ', '')}({self.submitter_id})>"
except AttributeError:
return f"<Sample({self.submitter_id})"
def to_sub_dict(self, submission_rsl:str) -> dict:
"""
@@ -1448,6 +1505,7 @@ class BasicSample(BaseClass):
Returns:
dict: well location and name (sample id, organism) NOTE: keys must sync with WWSample to_sub_dict above
"""
# logger.debug(f"Converting {self} to dict.")
sample = {}
sample['submitter_id'] = self.submitter_id
sample['sample_type'] = self.sample_type
@@ -1642,6 +1700,9 @@ class WastewaterSample(BasicSample):
"""
sample = super().to_sub_dict(submission_rsl=submission_rsl)
sample['ww_processing_num'] = self.ww_processing_num
sample['sample_location'] = self.sample_location
sample['received_date'] = self.received_date
sample['collection_date'] = self.collection_date
return sample
@classmethod
@@ -1722,8 +1783,9 @@ class SubmissionSampleAssociation(BaseClass):
# __tablename__ = "_submission_sample"
sample_id = Column(INTEGER, ForeignKey("_samples.id"), nullable=False) #: id of associated sample
submission_id = Column(INTEGER, ForeignKey("_submissions.id"), primary_key=True) #: id of associated submission
id = Column(INTEGER, unique=True, nullable=False)
sample_id = Column(INTEGER, ForeignKey("_basicsample.id"), nullable=False) #: id of associated sample
submission_id = Column(INTEGER, ForeignKey("_basicsubmission.id"), primary_key=True) #: id of associated submission
row = Column(INTEGER, primary_key=True) #: row on the 96 well plate
column = Column(INTEGER, primary_key=True) #: column on the 96 well plate
@@ -1743,14 +1805,23 @@ class SubmissionSampleAssociation(BaseClass):
"with_polymorphic": "*",
}
def __init__(self, submission:BasicSubmission=None, sample:BasicSample=None, row:int=1, column:int=1):
def __init__(self, submission:BasicSubmission=None, sample:BasicSample=None, row:int=1, column:int=1, id:int|None=None):
self.submission = submission
self.sample = sample
self.row = row
self.column = column
if id != None:
self.id = id
else:
self.id = self.__class__.autoincrement_id()
logger.debug(f"Using id: {self.id}")
def __repr__(self) -> str:
return f"<SubmissionSampleAssociation({self.submission.rsl_plate_num} & {self.sample.submitter_id})"
try:
return f"<{self.__class__.__name__}({self.submission.rsl_plate_num} & {self.sample.submitter_id})"
except AttributeError as e:
logger.error(f"Unable to construct __repr__ due to: {e}")
return super().__repr__()
def to_sub_dict(self) -> dict:
"""
@@ -1760,6 +1831,7 @@ class SubmissionSampleAssociation(BaseClass):
dict: Updated dictionary with row, column and well updated
"""
# Get sample info
# logger.debug(f"Running {self.__repr__()}")
sample = self.sample.to_sub_dict(submission_rsl=self.submission)
# sample = {}
sample['name'] = self.sample.submitter_id
@@ -1787,6 +1859,7 @@ class SubmissionSampleAssociation(BaseClass):
# Since there is no PCR, negliable result is necessary.
# assoc = [item for item in self.sample_submission_associations if item.submission.rsl_plate_num==submission_rsl][0]
sample = self.to_sub_dict()
logger.debug(f"Sample dict to hitpick: {sample}")
env = jinja_template_loading()
template = env.get_template("tooltip.html")
tooltip_text = template.render(fields=sample)
@@ -1801,6 +1874,14 @@ class SubmissionSampleAssociation(BaseClass):
sample.update(dict(name=self.sample.submitter_id[:10], tooltip=tooltip_text))
return sample
@classmethod
def autoincrement_id(cls):
try:
return max([item.id for item in cls.query()]) + 1
except ValueError as e:
logger.error(f"Problem incrementing id: {e}")
return 1
@classmethod
def find_polymorphic_subclass(cls, polymorphic_identity:str|None=None) -> SubmissionSampleAssociation:
"""
@@ -1890,6 +1971,7 @@ class SubmissionSampleAssociation(BaseClass):
association_type:str="Basic Association",
submission:BasicSubmission|str|None=None,
sample:BasicSample|str|None=None,
id:int|None=None,
**kwargs) -> SubmissionSampleAssociation:
"""
Queries for an association, if none exists creates a new one.
@@ -1931,7 +2013,7 @@ class SubmissionSampleAssociation(BaseClass):
instance = None
if instance == None:
used_cls = cls.find_polymorphic_subclass(polymorphic_identity=association_type)
instance = used_cls(submission=submission, sample=sample, **kwargs)
instance = used_cls(submission=submission, sample=sample, id=id, **kwargs)
return instance
def delete(self):
@@ -1941,8 +2023,8 @@ class WastewaterAssociation(SubmissionSampleAssociation):
"""
Derivative custom Wastewater/Submission Association... fancy.
"""
sample_id = Column(INTEGER, ForeignKey('_submissionsampleassociation.sample_id'), primary_key=True)
submission_id = Column(INTEGER, ForeignKey('_submissionsampleassociation.submission_id'), primary_key=True)
# sample_id = Column(INTEGER, ForeignKey('_submissionsampleassociation.sample_id'), primary_key=True)
id = Column(INTEGER, ForeignKey("_submissionsampleassociation.id"), primary_key=True)
ct_n1 = Column(FLOAT(2)) #: AKA ct for N1
ct_n2 = Column(FLOAT(2)) #: AKA ct for N2
n1_status = Column(String(32)) #: positive or negative for N1
@@ -1952,7 +2034,11 @@ class WastewaterAssociation(SubmissionSampleAssociation):
# __mapper_args__ = {"polymorphic_identity": "Wastewater Association", "polymorphic_load": "inline"}
__mapper_args__ = dict(polymorphic_identity="Wastewater Association",
polymorphic_load="inline",
inherit_condition=(sample_id == SubmissionSampleAssociation.sample_id))
# inherit_condition=(submission_id==SubmissionSampleAssociation.submission_id and
# row==SubmissionSampleAssociation.row and
# column==SubmissionSampleAssociation.column))
inherit_condition=(id==SubmissionSampleAssociation.id))
# inherit_foreign_keys=(sample_id == SubmissionSampleAssociation.sample_id, submission_id == SubmissionSampleAssociation.submission_id))
def to_sub_dict(self) -> dict:
sample = super().to_sub_dict()
@@ -1970,3 +2056,12 @@ class WastewaterAssociation(SubmissionSampleAssociation):
except (TypeError, AttributeError) as e:
logger.error(f"Couldn't set tooltip for {self.sample.rsl_number}. Looks like there isn't PCR data.")
return sample
@classmethod
def autoincrement_id(cls):
try:
parent = [base for base in cls.__bases__ if base.__name__=="SubmissionSampleAssociation"][0]
return max([item.id for item in parent.query()]) + 1
except ValueError as e:
logger.error(f"Problem incrementing id: {e}")
return 1

12
src/submissions/backend/validators/__init__.py
View File

@@ -3,6 +3,7 @@ from pathlib import Path
from openpyxl import load_workbook
from backend.db.models import BasicSubmission, SubmissionType
from datetime import date
from tools import jinja_template_loading
logger = logging.getLogger(f"submissions.{__name__}")
@@ -126,9 +127,20 @@ class RSLNamer(object):
today = parse(today.group())
except AttributeError:
today = datetime.now()
if "rsl_plate_num" in data.keys():
plate_number = data['rsl_plate_num'].split("-")[-1][0]
else:
previous = BasicSubmission.query(start_date=today, end_date=today, submission_type=data['submission_type'])
plate_number = len(previous) + 1
return f"RSL-{data['abbreviation']}-{today.year}{str(today.month).zfill(2)}{str(today.day).zfill(2)}-{plate_number}"
@classmethod
def construct_export_name(cls, template, **kwargs):
logger.debug(f"Kwargs: {kwargs}")
logger.debug(f"Template: {template}")
environment = jinja_template_loading()
template = environment.from_string(template)
return template.render(**kwargs)
from .pydant import *

43
src/submissions/backend/validators/pydant.py
View File

@@ -8,7 +8,7 @@ from pydantic import BaseModel, field_validator, Field
from datetime import date, datetime, timedelta
from dateutil.parser import parse
from dateutil.parser._parser import ParserError
from typing import List, Any, Tuple
from typing import List, Tuple
from . import RSLNamer
from pathlib import Path
from tools import check_not_nan, convert_nans_to_nones, jinja_template_loading, Report, Result, row_map
@@ -156,8 +156,9 @@ class PydSample(BaseModel, extra='allow'):
sample_type: str
row: int|List[int]|None
column: int|List[int]|None
assoc_id: int|List[int]|None = Field(default=None)
@field_validator("row", "column")
@field_validator("row", "column", "assoc_id")
@classmethod
def row_int_to_list(cls, value):
if isinstance(value, int):
@@ -193,14 +194,14 @@ class PydSample(BaseModel, extra='allow'):
out_associations = []
if submission != None:
assoc_type = self.sample_type.replace("Sample", "").strip()
for row, column in zip(self.row, self.column):
# logger.debug(f"Looking up association with identity: ({submission.submission_type_name} Association)")
for row, column, id in zip(self.row, self.column, self.assoc_id):
logger.debug(f"Looking up association with identity: ({submission.submission_type_name} Association)")
logger.debug(f"Looking up association with identity: ({assoc_type} Association)")
association = SubmissionSampleAssociation.query_or_create(association_type=f"{assoc_type} Association",
submission=submission,
sample=instance,
row=row, column=column)
logger.debug(f"Using submission_sample_association: {association}")
row=row, column=column, id=id)
# logger.debug(f"Using submission_sample_association: {association}")
try:
instance.sample_submission_associations.append(association)
out_associations.append(association)
@@ -254,7 +255,7 @@ class PydEquipment(BaseModel, extra='ignore'):
assoc.process = process
assoc.role = self.role
# equipment.equipment_submission_associations.append(assoc)
equipment.equipment_submission_associations.append(assoc)
# equipment.equipment_submission_associations.append(assoc)
else:
assoc = None
return equipment, assoc
@@ -275,7 +276,7 @@ class PydSubmission(BaseModel, extra='allow'):
comment: dict|None = Field(default=dict(value="", missing=True), validate_default=True)
reagents: List[dict]|List[PydReagent] = []
samples: List[PydSample]
equipment: List[PydEquipment]|None
equipment: List[PydEquipment]|None =[]
@field_validator('equipment', mode='before')
@classmethod
@@ -421,6 +422,16 @@ class PydSubmission(BaseModel, extra='allow'):
value['value'] = values.data['submission_type']['value']
return value
@field_validator("samples")
def assign_ids(cls, value, values):
starting_id = SubmissionSampleAssociation.autoincrement_id()
output = []
for iii, sample in enumerate(value, start=starting_id):
sample.assoc_id = [iii]
output.append(sample)
return output
def handle_duplicate_samples(self):
"""
Collapses multiple samples with same submitter id into one with lists for rows, columns.
@@ -428,14 +439,19 @@ class PydSubmission(BaseModel, extra='allow'):
"""
submitter_ids = list(set([sample.submitter_id for sample in self.samples]))
output = []
for id in submitter_ids:
for iii, id in enumerate(submitter_ids, start=1):
relevants = [item for item in self.samples if item.submitter_id==id]
if len(relevants) <= 1:
output += relevants
else:
rows = [item.row[0] for item in relevants]
columns = [item.column[0] for item in relevants]
ids = [item.assoc_id[0] for item in relevants]
# for jjj, rel in enumerate(relevants, start=1):
# starting_id += jjj
# ids.append(starting_id)
dummy = relevants[0]
dummy.assoc_id = ids
dummy.row = rows
dummy.column = columns
output.append(dummy)
@@ -663,14 +679,17 @@ class PydSubmission(BaseModel, extra='allow'):
logger.debug(f"Workbook sheets: {workbook.sheetnames}")
worksheet = workbook[sample_info["lookup_table"]['sheet']]
samples = sorted(self.samples, key=attrgetter('column', 'row'))
custom_sampler = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=self.submission_type).adjust_autofill_samples
samples = custom_sampler(samples=samples)
submission_obj = BasicSubmission.find_polymorphic_subclass(polymorphic_identity=self.submission_type)
samples = submission_obj.adjust_autofill_samples(samples=samples)
logger.debug(f"Samples: {pformat(samples)}")
# Fail safe against multiple instances of the same sample
for iii, sample in enumerate(samples, start=1):
logger.debug(f"Sample: {sample}")
row = submission_obj.custom_sample_autofill_row(sample, worksheet=worksheet)
logger.debug(f"Writing to {row}")
if row == None:
row = sample_info['lookup_table']['start_row'] + iii
fields = [field for field in list(sample.model_fields.keys()) + list(sample.model_extra.keys()) if field in sample_info['sample_columns'].keys()]
for field in fields:
column = sample_info['sample_columns'][field]
value = getattr(sample, field)

14
src/submissions/frontend/widgets/gel_checker.py
View File

@@ -1,7 +1,7 @@
# import required modules
from PyQt6.QtCore import Qt
# from PyQt6.QtCore import Qt
from PyQt6.QtWidgets import *
import sys
# import sys
from PyQt6.QtWidgets import QWidget
import numpy as np
import pyqtgraph as pg
@@ -13,12 +13,13 @@ import numpy as np
# Main window class
class GelBox(QDialog):
def __init__(self, parent):
def __init__(self, parent, img_path):
super().__init__(parent)
# setting title
self.setWindowTitle("PyQtGraph")
self.img_path = img_path
# setting geometry
self.setGeometry(100, 100, 600, 500)
self.setGeometry(50, 50, 1200, 900)
# icon
icon = QIcon("skin.png")
# setting icon to the window
@@ -35,7 +36,7 @@ class GelBox(QDialog):
pg.setConfigOptions(antialias=True)
# creating image view object
self.imv = pg.ImageView()
img = np.array(Image.open("C:\\Users\\lwark\\Desktop\\PLATE1_17012024_103607AM_1_4x26.jpg").rotate(-90).transpose(Image.FLIP_LEFT_RIGHT))
img = np.array(Image.open(self.img_path).rotate(-90).transpose(Image.FLIP_LEFT_RIGHT))
self.imv.setImage(img)#, xvals=np.linspace(1., 3., data.shape[0]))
layout = QGridLayout()
# setting this layout to the widget
@@ -54,7 +55,7 @@ class GelBox(QDialog):
self.setLayout(layout)
def parse_form(self):
return self.form.parse_form()
return self.img_path, self.form.parse_form()
class ControlsForm(QWidget):
@@ -79,6 +80,7 @@ class ControlsForm(QWidget):
for iii in range(3):
for jjj in range(3):
widge = QLineEdit()
widge.setText("Neg")
widge.setObjectName(f"{rows[iii]} : {columns[jjj]}")
self.layout.addWidget(widge, iii+1, jjj+2, 1, 1)
self.setLayout(self.layout)

11
src/submissions/frontend/widgets/submission_widget.py
View File

@@ -312,11 +312,12 @@ class SubmissionFormContainer(QWidget):
# update_last_used(reagent=reagent, kit=base_submission.extraction_kit)
reagent.update_last_used(kit=base_submission.extraction_kit)
# sys.exit()
logger.debug(f"Here is the final submission: {pformat(base_submission.__dict__)}")
logger.debug(f"Parsed reagents: {pformat(base_submission.reagents)}")
logger.debug(f"Sending submission: {base_submission.rsl_plate_num} to database.")
logger.debug(f"Samples from pyd: {pformat(self.pyd.samples)}")
logger.debug(f"Samples SQL: {pformat([item.__dict__ for item in base_submission.samples])}")
# logger.debug(f"Here is the final submission: {pformat(base_submission.__dict__)}")
# logger.debug(f"Parsed reagents: {pformat(base_submission.reagents)}")
# logger.debug(f"Sending submission: {base_submission.rsl_plate_num} to database.")
# logger.debug(f"Samples from pyd: {pformat(self.pyd.samples)}")
# logger.debug(f"Samples SQL: {pformat([item.__dict__ for item in base_submission.samples])}")
# logger.debug(f"")
base_submission.save()
# update summary sheet
self.app.table_widget.sub_wid.setData()

2
src/submissions/templates/tooltip.html
View File

@@ -1,4 +1,4 @@
Sample name: {{ fields['submitter_id'] }}<br>
{% if fields['organism'] %}Organism: {{ fields['organism'] }}<br>{% endif %}
{% if fields['concentration'] %}Concentration: {{ fields['concentration'] }}<br>{% endif %}
Well: {{ fields['row'] }}{{ fields['column'] }}
Well: {{ fields['well'] }}<!--{{ fields['column'] }}-->

2
src/submissions/tools.py
View File

@@ -357,7 +357,7 @@ def copy_settings(settings_path:Path, settings:dict) -> dict:
yaml.dump(settings, f)
return settings
def jinja_template_loading():
def jinja_template_loading() -> Environment:
"""
Returns jinja2 template environment.