Before making a big mistake.

CHANGELOG.md

@@ -1,3 +1,8 @@
+## 202311.03
+
+- Added in tabular log parser.
+- Split main_window_functions into object specific functions.
+
 ## 202311.02
 
 - Construct first strand integrated into Artic Import.

TODO.md

@@ -1,5 +1,7 @@
+- [ ] Buuuuuuhh. Split polymorphic objects into different tables... and rebuild DB.... FFFFF
+    - https://stackoverflow.com/questions/16910782/sqlalchemy-nested-inheritance-polymorphic-relationships
 - [x] Create a result object to facilitate returning function results.
-- [ ] Refactor main_window_functions into as many objects (forms, etc.) as possible to clean it up.
+- [x] Refactor main_window_functions into as many objects (forms, etc.) as possible to clean it up.
 - [x] Integrate 'Construct First Strand' into the Artic import.
 - [x] Clear out any unnecessary ctx passes now that queries are improved.
 - [x] Make a 'query or create' method in all db objects to go with new query.

alembic/env.py

@@ -24,7 +24,9 @@ if config.config_file_name is not None:
 # from myapp import mymodel
 # target_metadata = mymodel.Base.metadata
 from submissions.backend.db.models import Base
-target_metadata = [Base.metadata]
+# META_DATA = MetaData(bind=CONN, reflect=True)
+# base = ctx.database_session.get_bind()
+target_metadata = Base.metadata
 
 # other values from the config, defined by the needs of env.py,
 # can be acquired:

alembic/versions/3d9a88bd4ecd_added_in_other_ww_techs.py

@@ -1,34 +0,0 @@
-"""added in other ww techs
-
-Revision ID: 3d9a88bd4ecd
-Revises: f7f46e72f057
-Create Date: 2023-08-30 11:03:41.575219
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = '3d9a88bd4ecd'
-down_revision = 'f7f46e72f057'
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
-    with op.batch_alter_table('_submissions', schema=None) as batch_op:
-        batch_op.add_column(sa.Column('ext_technician', sa.String(length=64), nullable=True))
-        batch_op.add_column(sa.Column('pcr_technician', sa.String(length=64), nullable=True))
-
-    # ### end Alembic commands ###
-
-
-def downgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
-    with op.batch_alter_table('_submissions', schema=None) as batch_op:
-        batch_op.drop_column('pcr_technician')
-        batch_op.drop_column('ext_technician')
-
-    # ### end Alembic commands ###

alembic/versions/8a5bc2924ef9_adding_artic_technician_to_artic.py

@@ -1,32 +0,0 @@
-"""adding artic_technician to Artic
-
-Revision ID: 8a5bc2924ef9
-Revises: b95478ffb4a3
-Create Date: 2023-10-31 13:59:47.746122
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = '8a5bc2924ef9'
-down_revision = 'b95478ffb4a3'
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
-    with op.batch_alter_table('_submissions', schema=None) as batch_op:
-        batch_op.add_column(sa.Column('artic_technician', sa.String(length=64), nullable=True))
-
-    # ### end Alembic commands ###
-
-
-def downgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
-    with op.batch_alter_table('_submissions', schema=None) as batch_op:
-        batch_op.drop_column('artic_technician')
-
-    # ### end Alembic commands ###

alembic/versions/9a133efb3ffd_adjusting_reagents_reagenttypes_to_many_.py

@@ -1,33 +0,0 @@
-"""adjusting reagents/reagenttypes to many-to-many
-
-Revision ID: 9a133efb3ffd
-Revises: 3d9a88bd4ecd
-Create Date: 2023-09-01 10:28:22.335890
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = '9a133efb3ffd'
-down_revision = '3d9a88bd4ecd'
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('_reagenttypes_reagents',
-    sa.Column('reagent_id', sa.INTEGER(), nullable=True),
-    sa.Column('reagenttype_id', sa.INTEGER(), nullable=True),
-    sa.ForeignKeyConstraint(['reagent_id'], ['_reagents.id'], ),
-    sa.ForeignKeyConstraint(['reagenttype_id'], ['_reagent_types.id'], )
-    )
-    # ### end Alembic commands ###
-
-
-def downgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_table('_reagenttypes_reagents')
-    # ### end Alembic commands ###

alembic/versions/b95478ffb4a3_adding_submission_category.py

@@ -1,32 +0,0 @@
-"""adding submission category
-
-Revision ID: b95478ffb4a3
-Revises: 9a133efb3ffd
-Create Date: 2023-10-03 14:00:09.663055
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = 'b95478ffb4a3'
-down_revision = '9a133efb3ffd'
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
-    with op.batch_alter_table('_submissions', schema=None) as batch_op:
-        batch_op.add_column(sa.Column('submission_category', sa.String(length=64), nullable=True))
-
-    # ### end Alembic commands ###
-
-
-def downgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
-    with op.batch_alter_table('_submissions', schema=None) as batch_op:
-        batch_op.drop_column('submission_category')
-
-    # ### end Alembic commands ###

alembic/versions/f7f46e72f057_rebuild_database.py

@@ -1,204 +0,0 @@
-"""rebuild database
-
-Revision ID: f7f46e72f057
-Revises: 
-Create Date: 2023-08-30 09:47:18.071070
-
-"""
-from alembic import op
-import sqlalchemy as sa
-
-
-# revision identifiers, used by Alembic.
-revision = 'f7f46e72f057'
-down_revision = None
-branch_labels = None
-depends_on = None
-
-
-def upgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('_contacts',
-    sa.Column('id', sa.INTEGER(), nullable=False),
-    sa.Column('name', sa.String(length=64), nullable=True),
-    sa.Column('email', sa.String(length=64), nullable=True),
-    sa.Column('phone', sa.String(length=32), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('_control_types',
-    sa.Column('id', sa.INTEGER(), nullable=False),
-    sa.Column('name', sa.String(length=255), nullable=True),
-    sa.Column('targets', sa.JSON(), nullable=True),
-    sa.PrimaryKeyConstraint('id'),
-    sa.UniqueConstraint('name')
-    )
-    op.create_table('_kits',
-    sa.Column('id', sa.INTEGER(), nullable=False),
-    sa.Column('name', sa.String(length=64), nullable=True),
-    sa.PrimaryKeyConstraint('id'),
-    sa.UniqueConstraint('name')
-    )
-    op.create_table('_organizations',
-    sa.Column('id', sa.INTEGER(), nullable=False),
-    sa.Column('name', sa.String(length=64), nullable=True),
-    sa.Column('cost_centre', sa.String(), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('_reagent_types',
-    sa.Column('id', sa.INTEGER(), nullable=False),
-    sa.Column('name', sa.String(length=64), nullable=True),
-    sa.Column('eol_ext', sa.Interval(), nullable=True),
-    sa.Column('last_used', sa.String(length=32), nullable=True),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('_samples',
-    sa.Column('id', sa.INTEGER(), nullable=False),
-    sa.Column('submitter_id', sa.String(length=64), nullable=False),
-    sa.Column('sample_type', sa.String(length=32), nullable=True),
-    sa.Column('ww_processing_num', sa.String(length=64), nullable=True),
-    sa.Column('ww_sample_full_id', sa.String(length=64), nullable=True),
-    sa.Column('rsl_number', sa.String(length=64), nullable=True),
-    sa.Column('collection_date', sa.TIMESTAMP(), nullable=True),
-    sa.Column('received_date', sa.TIMESTAMP(), nullable=True),
-    sa.Column('notes', sa.String(length=2000), nullable=True),
-    sa.Column('sample_location', sa.String(length=8), nullable=True),
-    sa.Column('organism', sa.String(length=64), nullable=True),
-    sa.Column('concentration', sa.String(length=16), nullable=True),
-    sa.PrimaryKeyConstraint('id'),
-    sa.UniqueConstraint('submitter_id')
-    )
-    op.create_table('_submission_types',
-    sa.Column('id', sa.INTEGER(), nullable=False),
-    sa.Column('name', sa.String(length=128), nullable=True),
-    sa.Column('info_map', sa.JSON(), nullable=True),
-    sa.PrimaryKeyConstraint('id'),
-    sa.UniqueConstraint('name')
-    )
-    op.create_table('_discounts',
-    sa.Column('id', sa.INTEGER(), nullable=False),
-    sa.Column('kit_id', sa.INTEGER(), nullable=True),
-    sa.Column('client_id', sa.INTEGER(), nullable=True),
-    sa.Column('name', sa.String(length=128), nullable=True),
-    sa.Column('amount', sa.FLOAT(precision=2), nullable=True),
-    sa.ForeignKeyConstraint(['client_id'], ['_organizations.id'], name='fk_org_id', ondelete='SET NULL'),
-    sa.ForeignKeyConstraint(['kit_id'], ['_kits.id'], name='fk_kit_type_id', ondelete='SET NULL'),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('_orgs_contacts',
-    sa.Column('org_id', sa.INTEGER(), nullable=True),
-    sa.Column('contact_id', sa.INTEGER(), nullable=True),
-    sa.ForeignKeyConstraint(['contact_id'], ['_contacts.id'], ),
-    sa.ForeignKeyConstraint(['org_id'], ['_organizations.id'], )
-    )
-    op.create_table('_reagents',
-    sa.Column('id', sa.INTEGER(), nullable=False),
-    sa.Column('type_id', sa.INTEGER(), nullable=True),
-    sa.Column('name', sa.String(length=64), nullable=True),
-    sa.Column('lot', sa.String(length=64), nullable=True),
-    sa.Column('expiry', sa.TIMESTAMP(), nullable=True),
-    sa.ForeignKeyConstraint(['type_id'], ['_reagent_types.id'], name='fk_reagent_type_id', ondelete='SET NULL'),
-    sa.PrimaryKeyConstraint('id')
-    )
-    op.create_table('_reagenttypes_kittypes',
-    sa.Column('reagent_types_id', sa.INTEGER(), nullable=False),
-    sa.Column('kits_id', sa.INTEGER(), nullable=False),
-    sa.Column('uses', sa.JSON(), nullable=True),
-    sa.Column('required', sa.INTEGER(), nullable=True),
-    sa.ForeignKeyConstraint(['kits_id'], ['_kits.id'], ),
-    sa.ForeignKeyConstraint(['reagent_types_id'], ['_reagent_types.id'], ),
-    sa.PrimaryKeyConstraint('reagent_types_id', 'kits_id')
-    )
-    op.create_table('_submissiontypes_kittypes',
-    sa.Column('submission_types_id', sa.INTEGER(), nullable=False),
-    sa.Column('kits_id', sa.INTEGER(), nullable=False),
-    sa.Column('mutable_cost_column', sa.FLOAT(precision=2), nullable=True),
-    sa.Column('mutable_cost_sample', sa.FLOAT(precision=2), nullable=True),
-    sa.Column('constant_cost', sa.FLOAT(precision=2), nullable=True),
-    sa.ForeignKeyConstraint(['kits_id'], ['_kits.id'], ),
-    sa.ForeignKeyConstraint(['submission_types_id'], ['_submission_types.id'], ),
-    sa.PrimaryKeyConstraint('submission_types_id', 'kits_id')
-    )
-    op.create_table('_submissions',
-    sa.Column('id', sa.INTEGER(), nullable=False),
-    sa.Column('rsl_plate_num', sa.String(length=32), nullable=False),
-    sa.Column('submitter_plate_num', sa.String(length=127), nullable=True),
-    sa.Column('submitted_date', sa.TIMESTAMP(), nullable=True),
-    sa.Column('submitting_lab_id', sa.INTEGER(), nullable=True),
-    sa.Column('sample_count', sa.INTEGER(), nullable=True),
-    sa.Column('extraction_kit_id', sa.INTEGER(), nullable=True),
-    sa.Column('submission_type_name', sa.String(), nullable=True),
-    sa.Column('technician', sa.String(length=64), nullable=True),
-    sa.Column('reagents_id', sa.String(), nullable=True),
-    sa.Column('extraction_info', sa.JSON(), nullable=True),
-    sa.Column('run_cost', sa.FLOAT(precision=2), nullable=True),
-    sa.Column('uploaded_by', sa.String(length=32), nullable=True),
-    sa.Column('comment', sa.JSON(), nullable=True),
-    sa.Column('pcr_info', sa.JSON(), nullable=True),
-    sa.ForeignKeyConstraint(['extraction_kit_id'], ['_kits.id'], name='fk_BS_extkit_id', ondelete='SET NULL'),
-    sa.ForeignKeyConstraint(['reagents_id'], ['_reagents.id'], name='fk_BS_reagents_id', ondelete='SET NULL'),
-    sa.ForeignKeyConstraint(['submission_type_name'], ['_submission_types.name'], name='fk_BS_subtype_name', ondelete='SET NULL'),
-    sa.ForeignKeyConstraint(['submitting_lab_id'], ['_organizations.id'], name='fk_BS_sublab_id', ondelete='SET NULL'),
-    sa.PrimaryKeyConstraint('id'),
-    sa.UniqueConstraint('rsl_plate_num'),
-    sa.UniqueConstraint('submitter_plate_num')
-    )
-    op.create_table('_control_samples',
-    sa.Column('id', sa.INTEGER(), nullable=False),
-    sa.Column('parent_id', sa.String(), nullable=True),
-    sa.Column('name', sa.String(length=255), nullable=True),
-    sa.Column('submitted_date', sa.TIMESTAMP(), nullable=True),
-    sa.Column('contains', sa.JSON(), nullable=True),
-    sa.Column('matches', sa.JSON(), nullable=True),
-    sa.Column('kraken', sa.JSON(), nullable=True),
-    sa.Column('submission_id', sa.INTEGER(), nullable=True),
-    sa.Column('refseq_version', sa.String(length=16), nullable=True),
-    sa.Column('kraken2_version', sa.String(length=16), nullable=True),
-    sa.Column('kraken2_db_version', sa.String(length=32), nullable=True),
-    sa.ForeignKeyConstraint(['parent_id'], ['_control_types.id'], name='fk_control_parent_id'),
-    sa.ForeignKeyConstraint(['submission_id'], ['_submissions.id'], ),
-    sa.PrimaryKeyConstraint('id'),
-    sa.UniqueConstraint('name')
-    )
-    op.create_table('_reagents_submissions',
-    sa.Column('reagent_id', sa.INTEGER(), nullable=True),
-    sa.Column('submission_id', sa.INTEGER(), nullable=True),
-    sa.ForeignKeyConstraint(['reagent_id'], ['_reagents.id'], ),
-    sa.ForeignKeyConstraint(['submission_id'], ['_submissions.id'], )
-    )
-    op.create_table('_submission_sample',
-    sa.Column('sample_id', sa.INTEGER(), nullable=False),
-    sa.Column('submission_id', sa.INTEGER(), nullable=False),
-    sa.Column('row', sa.INTEGER(), nullable=False),
-    sa.Column('column', sa.INTEGER(), nullable=False),
-    sa.Column('base_sub_type', sa.String(), nullable=True),
-    sa.Column('ct_n1', sa.FLOAT(precision=2), nullable=True),
-    sa.Column('ct_n2', sa.FLOAT(precision=2), nullable=True),
-    sa.Column('n1_status', sa.String(length=32), nullable=True),
-    sa.Column('n2_status', sa.String(length=32), nullable=True),
-    sa.Column('pcr_results', sa.JSON(), nullable=True),
-    sa.ForeignKeyConstraint(['sample_id'], ['_samples.id'], ),
-    sa.ForeignKeyConstraint(['submission_id'], ['_submissions.id'], ),
-    sa.PrimaryKeyConstraint('submission_id', 'row', 'column')
-    )
-    # ### end Alembic commands ###
-
-
-def downgrade() -> None:
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_table('_submission_sample')
-    op.drop_table('_reagents_submissions')
-    op.drop_table('_control_samples')
-    op.drop_table('_submissions')
-    op.drop_table('_submissiontypes_kittypes')
-    op.drop_table('_reagenttypes_kittypes')
-    op.drop_table('_reagents')
-    op.drop_table('_orgs_contacts')
-    op.drop_table('_discounts')
-    op.drop_table('_submission_types')
-    op.drop_table('_samples')
-    op.drop_table('_reagent_types')
-    op.drop_table('_organizations')
-    op.drop_table('_kits')
-    op.drop_table('_control_types')
-    op.drop_table('_contacts')
-    # ### end Alembic commands ###

src/submissions/backend/db/__init__.py

@@ -1,5 +1,5 @@
 '''
 All database related operations.
 '''
-from .models import *
 from .functions import *
+from .models import *

src/submissions/backend/db/models/__init__.py

@@ -1,10 +1,10 @@
 '''
 Contains all models for sqlalchemy
 '''
-from .controls import Control, ControlType
-# import order must go: orgs, kit, subs due to circular import issues
-from .organizations import Organization, Contact
-from .kits import KitType, ReagentType, Reagent, Discount, KitTypeReagentTypeAssociation, SubmissionType, SubmissionTypeKitTypeAssociation
-from .submissions import (BasicSubmission, BacterialCulture, Wastewater, WastewaterArtic, WastewaterSample, BacterialCultureSample, 
-                          BasicSample, SubmissionSampleAssociation, WastewaterAssociation)
 
+from tools import Base
+from .controls import *
+# import order must go: orgs, kit, subs due to circular import issues
+from .organizations import *
+from .kits import *
+from .submissions import *

src/submissions/backend/db/models/controls.py

@@ -7,7 +7,8 @@ from sqlalchemy.orm import relationship, Query
 import logging
 from operator import itemgetter
 import json
-from tools import Base, setup_lookup, query_return
+from . import Base
+from tools import setup_lookup, query_return
 from datetime import date, datetime
 from typing import List
 from dateutil.parser import parse
@@ -19,7 +20,7 @@ class ControlType(Base):
     Base class of a control archetype.
     """    
     __tablename__ = '_control_types'
-    
+    __table_args__ = {'extend_existing': True} 
     
     id = Column(INTEGER, primary_key=True) #: primary key   
     name = Column(String(255), unique=True) #: controltype name (e.g. MCS)
@@ -58,6 +59,7 @@ class Control(Base):
     """    
 
     __tablename__ = '_control_samples'
+    __table_args__ = {'extend_existing': True} 
     
     id = Column(INTEGER, primary_key=True) #: primary key
     parent_id = Column(String, ForeignKey("_control_types.id", name="fk_control_parent_id")) #: primary key of control type

src/submissions/backend/db/models/kits.py

@@ -2,24 +2,31 @@
 All kit and reagent related models
 '''
 from __future__ import annotations
-from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey, Interval, Table, FLOAT, func
+from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey, Interval, Table, FLOAT, func, BLOB
 from sqlalchemy.orm import relationship, validates, Query
 from sqlalchemy.ext.associationproxy import association_proxy
 from datetime import date
 import logging
-from tools import check_authorization, Base, setup_lookup, query_return, Report, Result
+from tools import check_authorization, setup_lookup, query_return, Report, Result
 from typing import List
-from . import Organization
+from . import Base, Organization
 
 logger = logging.getLogger(f'submissions.{__name__}')
 
-reagenttypes_reagents = Table("_reagenttypes_reagents", Base.metadata, Column("reagent_id", INTEGER, ForeignKey("_reagents.id")), Column("reagenttype_id", INTEGER, ForeignKey("_reagent_types.id")))
+reagenttypes_reagents = Table(
+                                "_reagenttypes_reagents", 
+                                Base.metadata, 
+                                Column("reagent_id", INTEGER, ForeignKey("_reagents.id")), 
+                                Column("reagenttype_id", INTEGER, ForeignKey("_reagent_types.id")),
+                                extend_existing = True
+                                )
 
 class KitType(Base):
     """
     Base of kits used in submission processing
     """    
     __tablename__ = "_kits"
+    __table_args__ = {'extend_existing': True} 
 
     id = Column(INTEGER, primary_key=True) #: primary key  
     name = Column(String(64), unique=True) #: name of kit
@@ -162,6 +169,7 @@ class ReagentType(Base):
     Base of reagent type abstract
     """    
     __tablename__ = "_reagent_types"
+    __table_args__ = {'extend_existing': True} 
 
     id = Column(INTEGER, primary_key=True) #: primary key  
     name = Column(String(64)) #: name of reagent type
@@ -251,6 +259,8 @@ class KitTypeReagentTypeAssociation(Base):
     DOC: https://docs.sqlalchemy.org/en/14/orm/extensions/associationproxy.html
     """    
     __tablename__ = "_reagenttypes_kittypes"
+    __table_args__ = {'extend_existing': True} 
+
     reagent_types_id = Column(INTEGER, ForeignKey("_reagent_types.id"), primary_key=True)
     kits_id = Column(INTEGER, ForeignKey("_kits.id"), primary_key=True)
     uses = Column(JSON)
@@ -333,6 +343,7 @@ class Reagent(Base):
     Concrete reagent instance
     """
     __tablename__ = "_reagents"
+    __table_args__ = {'extend_existing': True} 
 
     id = Column(INTEGER, primary_key=True) #: primary key
     type = relationship("ReagentType", back_populates="instances", secondary=reagenttypes_reagents) #: joined parent reagent type
@@ -491,6 +502,7 @@ class Discount(Base):
     Relationship table for client labs for certain kits.
     """
     __tablename__ = "_discounts"
+    __table_args__ = {'extend_existing': True} 
 
     id = Column(INTEGER, primary_key=True) #: primary key
     kit = relationship("KitType") #: joined parent reagent type
@@ -558,12 +570,14 @@ class SubmissionType(Base):
     Abstract of types of submissions.
     """    
     __tablename__ = "_submission_types"
+    __table_args__ = {'extend_existing': True} 
 
     id = Column(INTEGER, primary_key=True) #: primary key
     name = Column(String(128), unique=True) #: name of submission type
     info_map = Column(JSON) #: Where basic information is found in the excel workbook corresponding to this type.
     instances = relationship("BasicSubmission", backref="submission_type")
     # regex = Column(String(512))
+    template_file = Column(BLOB)
     
     submissiontype_kit_associations = relationship(
         "SubmissionTypeKitTypeAssociation",
@@ -619,6 +633,8 @@ class SubmissionTypeKitTypeAssociation(Base):
     Abstract of relationship between kits and their submission type.
     """    
     __tablename__ = "_submissiontypes_kittypes"
+    __table_args__ = {'extend_existing': True} 
+
     submission_types_id = Column(INTEGER, ForeignKey("_submission_types.id"), primary_key=True)
     kits_id = Column(INTEGER, ForeignKey("_kits.id"), primary_key=True)
     mutable_cost_column = Column(FLOAT(2)) #: dollar amount per 96 well plate that can change with number of columns (reagents, tips, etc)

src/submissions/backend/db/models/organizations.py

@@ -4,20 +4,29 @@ All client organization related models.
 from __future__ import annotations
 from sqlalchemy import Column, String, INTEGER, ForeignKey, Table
 from sqlalchemy.orm import relationship, Query
-from tools import Base, check_authorization, setup_lookup, query_return
+from . import Base
+from tools import check_authorization, setup_lookup, query_return
 from typing import List
 import logging
 
 logger = logging.getLogger(f"submissions.{__name__}")
 
 # table containing organization/contact relationship
-orgs_contacts = Table("_orgs_contacts", Base.metadata, Column("org_id", INTEGER, ForeignKey("_organizations.id")), Column("contact_id", INTEGER, ForeignKey("_contacts.id")))
+orgs_contacts = Table(
+                        "_orgs_contacts", 
+                        Base.metadata, 
+                        Column("org_id", INTEGER, ForeignKey("_organizations.id")), 
+                        Column("contact_id", INTEGER, ForeignKey("_contacts.id")), 
+                        # __table_args__ = {'extend_existing': True} 
+                        extend_existing = True
+                    )
 
 class Organization(Base):
     """
     Base of organization
     """
     __tablename__ = "_organizations"
+    __table_args__ = {'extend_existing': True} 
 
     id = Column(INTEGER, primary_key=True) #: primary key  
     name = Column(String(64)) #: organization name
@@ -76,6 +85,7 @@ class Contact(Base):
     Base of Contact
     """
     __tablename__ = "_contacts"
+    __table_args__ = {'extend_existing': True} 
 
     id = Column(INTEGER, primary_key=True) #: primary key  
     name = Column(String(64)) #: contact name
@@ -127,3 +137,4 @@ class Contact(Base):
             case _:
                 pass
         return query_return(query=query, limit=limit)
+    

src/submissions/backend/db/models/submissions.py

@@ -17,7 +17,8 @@ import uuid
 import re
 import pandas as pd
 from openpyxl import Workbook
-from tools import check_not_nan, row_map, Base, query_return, setup_lookup
+from . import Base
+from tools import check_not_nan, row_map,  query_return, setup_lookup
 from datetime import datetime, date
 from typing import List
 from dateutil.parser import parse
@@ -29,13 +30,21 @@ from sqlite3 import OperationalError as SQLOperationalError, IntegrityError as S
 logger = logging.getLogger(f"submissions.{__name__}")
 
 # table containing reagents/submission relationships
-reagents_submissions = Table("_reagents_submissions", Base.metadata, Column("reagent_id", INTEGER, ForeignKey("_reagents.id")), Column("submission_id", INTEGER, ForeignKey("_submissions.id")))
+reagents_submissions = Table(
+                                "_reagents_submissions", 
+                                Base.metadata, 
+                                Column("reagent_id", INTEGER, ForeignKey("_reagents.id")), 
+                                Column("submission_id", INTEGER, ForeignKey("_submissions.id")),
+                                extend_existing = True
+                            )
 
 class BasicSubmission(Base):
     """
     Concrete of basic submission which polymorphs into BacterialCulture and Wastewater
     """
+    
     __tablename__ = "_submissions"
+    __table_args__ = {'extend_existing': True} 
 
     id = Column(INTEGER, primary_key=True) #: primary key   
     rsl_plate_num = Column(String(32), unique=True, nullable=False) #: RSL name (e.g. RSL-22-0012)
@@ -705,7 +714,6 @@ class Wastewater(BasicSubmission):
     """
     derivative submission type from BasicSubmission
     """    
-    # pcr_info = Column(JSON)
     ext_technician = Column(String(64))
     pcr_technician = Column(String(64))
     __mapper_args__ = {"polymorphic_identity": "Wastewater", "polymorphic_load": "inline"}
@@ -948,6 +956,7 @@ class BasicSample(Base):
     """    
 
     __tablename__ = "_samples"
+    __table_args__ = {'extend_existing': True} 
 
     id = Column(INTEGER, primary_key=True) #: primary key
     submitter_id = Column(String(64), nullable=False, unique=True) #: identification from submitter
@@ -1226,6 +1235,8 @@ class SubmissionSampleAssociation(Base):
     DOC: https://docs.sqlalchemy.org/en/14/orm/extensions/associationproxy.html
     """    
     __tablename__ = "_submission_sample"
+    __table_args__ = {'extend_existing': True} 
+
     sample_id = Column(INTEGER, ForeignKey("_samples.id"), nullable=False)
     submission_id = Column(INTEGER, ForeignKey("_submissions.id"), primary_key=True)
     row = Column(INTEGER, primary_key=True) #: row on the 96 well plate

src/submissions/backend/validators/__init__.py

@@ -1,7 +1,7 @@
 import logging, re
 from pathlib import Path
 from openpyxl import load_workbook
-from backend.db import BasicSubmission, SubmissionType
+from backend.db.models import BasicSubmission, SubmissionType
 
 
 logger = logging.getLogger(f"submissions.{__name__}")

src/submissions/frontend/widgets/app.py

@@ -16,7 +16,7 @@ from backend.validators import PydReagent
 # )
 from tools import check_if_app, Settings, Report
 from .pop_ups import  AlertPop
-from .misc import AddReagentForm
+from .misc import AddReagentForm, LogParser
 import logging
 from datetime import date
 import webbrowser
@@ -69,7 +69,7 @@ class App(QMainWindow):
         menuBar = self.menuBar()
         fileMenu = menuBar.addMenu("&File")
         # Creating menus using a title
-        # methodsMenu = menuBar.addMenu("&Methods")
+        methodsMenu = menuBar.addMenu("&Methods")
         reportMenu = menuBar.addMenu("&Reports")
         maintenanceMenu = menuBar.addMenu("&Monthly")
         helpMenu = menuBar.addMenu("&Help")
@@ -77,7 +77,7 @@ class App(QMainWindow):
         helpMenu.addAction(self.docsAction)
         fileMenu.addAction(self.importAction)
         fileMenu.addAction(self.importPCRAction)
-        # methodsMenu.addAction(self.constructFS)
+        methodsMenu.addAction(self.searchLog)
         reportMenu.addAction(self.generateReportAction)
         maintenanceMenu.addAction(self.joinExtractionAction)
         maintenanceMenu.addAction(self.joinPCRAction)
@@ -108,7 +108,7 @@ class App(QMainWindow):
         self.joinPCRAction = QAction("Link PCR Logs")
         self.helpAction = QAction("&About", self)
         self.docsAction = QAction("&Docs", self)
-        # self.constructFS = QAction("Make First Strand", self)
+        self.searchLog = QAction("Search Log", self)
 
     def _connectActions(self):
         """
@@ -127,6 +127,7 @@ class App(QMainWindow):
         self.docsAction.triggered.connect(self.openDocs)
         # self.constructFS.triggered.connect(self.construct_first_strand)
         # self.table_widget.formwidget.import_drag.connect(self.importSubmission)
+        self.searchLog.triggered.connect(self.runSearch)
 
     def showAbout(self):
         """
@@ -335,13 +336,16 @@ class App(QMainWindow):
     #     # from .main_window_functions import export_csv_function
     #     export_csv_function(self, fname)
 
+    def runSearch(self):
+        dlg = LogParser(self)
+        dlg.exec()
+
 class AddSubForm(QWidget):
     
     def __init__(self, parent:QWidget):
         logger.debug(f"Initializating subform...")
         super(QWidget, self).__init__(parent)
         self.layout = QVBoxLayout(self)
-        self.parent = parent
         # Initialize tab screen
         self.tabs = QTabWidget()
         self.tab1 = QWidget()

src/submissions/frontend/widgets/controls_chart.py

@@ -17,7 +17,7 @@ class ControlsViewer(QWidget):
 
     def __init__(self, parent: QWidget) -> None:
         super().__init__(parent)
-        self.app = self.parent().parent
+        self.app = self.parent().parent()
         print(f"\n\n{self.app}\n\n")
         self.report = Report()
         self.datepicker = ControlsDatePicker()

src/submissions/frontend/widgets/functions.py

@@ -28,7 +28,7 @@ def select_open_file(obj:QMainWindow, file_extension:str) -> Path:
         home_dir = obj.app.last_dir.resolve().__str__()
     fname = Path(QFileDialog.getOpenFileName(obj, 'Open file', home_dir, filter = f"{file_extension}(*.{file_extension})")[0])
     # fname = Path(QFileDialog.getOpenFileName(obj, 'Open file', filter = f"{file_extension}(*.{file_extension})")[0])
-    obj.last_file = fname
+    obj.last_dir = fname.parent
     return fname
 
 def select_save_file(obj:QMainWindow, default_name:str, extension:str) -> Path:

src/submissions/frontend/widgets/misc.py

@@ -2,27 +2,18 @@
 Contains miscellaneous widgets for frontend functions
 '''
 from datetime import date
-from pprint import pformat
-from PyQt6 import QtCore
 from PyQt6.QtWidgets import (
     QLabel, QVBoxLayout,
     QLineEdit, QComboBox, QDialog, 
-    QDialogButtonBox, QDateEdit, QSizePolicy, QWidget,
-    QGridLayout, QPushButton, QSpinBox, QDoubleSpinBox,
-    QHBoxLayout, QScrollArea, QFormLayout
+    QDialogButtonBox, QDateEdit, QPushButton, QFormLayout
 )
-from PyQt6.QtCore import Qt, QDate, QSize, pyqtSignal
-from tools import check_not_nan, jinja_template_loading, Settings, Result
+from PyQt6.QtCore import Qt, QDate
+from tools import jinja_template_loading, Settings
 from backend.db.models import *
-from sqlalchemy import FLOAT, INTEGER
 import logging
-import numpy as np
-from .pop_ups import AlertPop, QuestionAsker
-from backend.validators import PydReagent, PydKit, PydReagentType, PydSubmission
-from typing import Tuple, List
-from pprint import pformat
-import difflib
-
+from .pop_ups import AlertPop
+from .functions import select_open_file
+from tools import readInChunks
 
 logger = logging.getLogger(f"submissions.{__name__}")
 
@@ -137,235 +128,6 @@ class ReportDatePicker(QDialog):
     def parse_form(self):
         return dict(start_date=self.start_date.date().toPyDate(), end_date = self.end_date.date().toPyDate())
 
-# class KitAdder(QWidget):
-#     """
-#     dialog to get information to add kit
-#     """    
-#     def __init__(self) -> None:
-#         super().__init__()
-#         # self.ctx = parent_ctx
-#         main_box = QVBoxLayout(self)
-#         scroll = QScrollArea(self)
-#         main_box.addWidget(scroll)
-#         scroll.setWidgetResizable(True)    
-#         scrollContent = QWidget(scroll)
-#         self.grid = QGridLayout()
-#         # self.setLayout(self.grid)
-#         scrollContent.setLayout(self.grid)
-#         # insert submit button at top
-#         self.submit_btn = QPushButton("Submit")
-#         self.grid.addWidget(self.submit_btn,0,0,1,1)
-#         self.grid.addWidget(QLabel("Kit Name:"),2,0)
-#         # widget to get kit name
-#         kit_name = QLineEdit()
-#         kit_name.setObjectName("kit_name")
-#         self.grid.addWidget(kit_name,2,1)
-#         self.grid.addWidget(QLabel("Used For Submission Type:"),3,0)
-#         # widget to get uses of kit
-#         used_for = QComboBox()
-#         used_for.setObjectName("used_for")
-#         # Insert all existing sample types
-#         # used_for.addItems([item.name for item in lookup_submission_type(ctx=parent_ctx)])
-#         used_for.addItems([item.name for item in SubmissionType.query()])
-#         used_for.setEditable(True)
-#         self.grid.addWidget(used_for,3,1)
-#         # Get all fields in SubmissionTypeKitTypeAssociation
-#         self.columns = [item for item in SubmissionTypeKitTypeAssociation.__table__.columns if len(item.foreign_keys) == 0]
-#         for iii, column in enumerate(self.columns):
-#             idx = iii + 4
-#             # convert field name to human readable.
-#             field_name = column.name.replace("_", " ").title()
-#             self.grid.addWidget(QLabel(field_name),idx,0)
-#             match column.type:
-#                 case FLOAT():
-#                     add_widget = QDoubleSpinBox()
-#                     add_widget.setMinimum(0)
-#                     add_widget.setMaximum(9999)
-#                 case INTEGER():
-#                     add_widget = QSpinBox()
-#                     add_widget.setMinimum(0)
-#                     add_widget.setMaximum(9999)
-#                 case _:
-#                     add_widget = QLineEdit()
-#             add_widget.setObjectName(column.name)
-#             self.grid.addWidget(add_widget, idx,1)
-#         self.add_RT_btn = QPushButton("Add Reagent Type")
-#         self.grid.addWidget(self.add_RT_btn)
-#         self.add_RT_btn.clicked.connect(self.add_RT)
-#         self.submit_btn.clicked.connect(self.submit)
-#         scroll.setWidget(scrollContent)
-#         self.ignore = [None, "", "qt_spinbox_lineedit", "qt_scrollarea_viewport", "qt_scrollarea_hcontainer",
-#                        "qt_scrollarea_vcontainer", "submit_btn"
-#                        ]
-        
-#     def add_RT(self) -> None:
-#         """
-#         insert new reagent type row
-#         """        
-#         # get bottommost row
-#         maxrow = self.grid.rowCount()
-#         reg_form = ReagentTypeForm()
-#         reg_form.setObjectName(f"ReagentForm_{maxrow}")
-#         # self.grid.addWidget(reg_form, maxrow + 1,0,1,2)
-#         self.grid.addWidget(reg_form, maxrow,0,1,4)
-        
-
-
-#     def submit(self) -> None:
-#         """
-#         send kit to database
-#         """        
-#         # get form info
-#         info, reagents = self.parse_form()
-#         # info, reagents = extract_form_info(self)
-#         info = {k:v for k,v in info.items() if k in [column.name for column in self.columns] + ['kit_name', 'used_for']}
-#         logger.debug(f"kit info: {pformat(info)}")
-#         logger.debug(f"kit reagents: {pformat(reagents)}")
-#         info['reagent_types'] = reagents
-#         logger.debug(pformat(info))
-#         # send to kit constructor
-#         kit = PydKit(name=info['kit_name'])
-#         for reagent in info['reagent_types']:
-#             uses = {
-#                 info['used_for']:
-#                     {'sheet':reagent['sheet'],
-#                      'name':reagent['name'],
-#                      'lot':reagent['lot'],
-#                      'expiry':reagent['expiry']
-#                     }}
-#             kit.reagent_types.append(PydReagentType(name=reagent['rtname'], eol_ext=reagent['eol'], uses=uses))
-#         logger.debug(f"Output pyd object: {kit.__dict__}")
-#         # result = construct_kit_from_yaml(ctx=self.ctx, kit_dict=info)
-#         sqlobj, result = kit.toSQL(self.ctx)
-#         sqlobj.save()
-#         msg = AlertPop(message=result['message'], status=result['status'])
-#         msg.exec()
-#         self.__init__(self.ctx)
-
-#     def parse_form(self) -> Tuple[dict, list]:
-#         logger.debug(f"Hello from {self.__class__} parser!")
-#         info = {}
-#         reagents = []
-#         widgets = [widget for widget in self.findChildren(QWidget) if widget.objectName() not in self.ignore and not isinstance(widget.parent(), ReagentTypeForm)]
-#         for widget in widgets:
-#             # logger.debug(f"Parsed widget: {widget.objectName()} of type {type(widget)} with parent {widget.parent()}")
-#             match widget:
-#                 case ReagentTypeForm():
-#                     reagents.append(widget.parse_form())
-#                 case QLineEdit():
-#                     info[widget.objectName()] = widget.text()
-#                 case QComboBox():
-#                     info[widget.objectName()] = widget.currentText()
-#                 case QDateEdit():
-#                     info[widget.objectName()] = widget.date().toPyDate()
-#         return info, reagents
-        
-# class ReagentTypeForm(QWidget):
-#     """
-#     custom widget to add information about a new reagenttype
-#     """    
-#     def __init__(self) -> None:
-#         super().__init__()
-#         grid = QGridLayout()
-#         self.setLayout(grid)
-#         grid.addWidget(QLabel("Reagent Type Name"),0,0)
-#         # Widget to get reagent info
-#         self.reagent_getter = QComboBox()
-#         self.reagent_getter.setObjectName("rtname")
-#         # lookup all reagent type names from db
-#         # lookup = lookup_reagent_types(ctx=ctx)
-#         lookup = ReagentType.query()
-#         logger.debug(f"Looked up ReagentType names: {lookup}")
-#         self.reagent_getter.addItems([item.__str__() for item in lookup])
-#         self.reagent_getter.setEditable(True)
-#         grid.addWidget(self.reagent_getter,0,1)
-#         grid.addWidget(QLabel("Extension of Life (months):"),0,2)
-#         # widget to get extension of life
-#         self.eol = QSpinBox()
-#         self.eol.setObjectName('eol')
-#         self.eol.setMinimum(0)
-#         grid.addWidget(self.eol, 0,3)
-#         grid.addWidget(QLabel("Excel Location Sheet Name:"),1,0)
-#         self.location_sheet_name = QLineEdit()
-#         self.location_sheet_name.setObjectName("sheet")
-#         self.location_sheet_name.setText("e.g. 'Reagent Info'")
-#         grid.addWidget(self.location_sheet_name, 1,1)
-#         for iii, item in enumerate(["Name", "Lot", "Expiry"]):
-#             idx = iii + 2
-#             grid.addWidget(QLabel(f"{item} Row:"), idx, 0)
-#             row = QSpinBox()
-#             row.setFixedWidth(50)
-#             row.setObjectName(f'{item.lower()}_row')
-#             row.setMinimum(0)
-#             grid.addWidget(row, idx, 1)
-#             grid.addWidget(QLabel(f"{item} Column:"), idx, 2)
-#             col = QSpinBox()
-#             col.setFixedWidth(50)
-#             col.setObjectName(f'{item.lower()}_column')
-#             col.setMinimum(0)
-#             grid.addWidget(col, idx, 3)
-#         self.setFixedHeight(175)
-#         max_row = grid.rowCount()
-#         self.r_button = QPushButton("Remove")
-#         self.r_button.clicked.connect(self.remove)
-#         grid.addWidget(self.r_button,max_row,0,1,1)
-#         self.ignore = [None, "", "qt_spinbox_lineedit", "qt_scrollarea_viewport", "qt_scrollarea_hcontainer",
-#                        "qt_scrollarea_vcontainer", "submit_btn", "eol", "sheet", "rtname"
-#                        ]
-
-#     def remove(self):
-#         self.setParent(None)
-#         self.destroy()
-
-#     def parse_form(self) -> dict:
-#         logger.debug(f"Hello from {self.__class__} parser!")
-#         info = {}
-#         info['eol'] = self.eol.value()
-#         info['sheet'] = self.location_sheet_name.text()
-#         info['rtname'] = self.reagent_getter.currentText()
-#         widgets = [widget for widget in self.findChildren(QWidget) if widget.objectName() not in self.ignore]
-#         for widget in widgets:
-#             logger.debug(f"Parsed widget: {widget.objectName()} of type {type(widget)} with parent {widget.parent()}")
-#             match widget:
-#                 case QLineEdit():
-#                     info[widget.objectName()] = widget.text()
-#                 case QComboBox():
-#                     info[widget.objectName()] = widget.currentText()
-#                 case QDateEdit():
-#                     info[widget.objectName()] = widget.date().toPyDate()
-#                 case QSpinBox() | QDoubleSpinBox():
-#                     if "_" in widget.objectName():
-#                         key, sub_key = widget.objectName().split("_")
-#                         if key not in info.keys():
-#                             info[key] = {}
-#                             logger.debug(f"Adding key {key}, {sub_key} and value {widget.value()} to {info}")
-#                         info[key][sub_key] = widget.value()        
-#         return info
-
-# class ControlsDatePicker(QWidget):
-#     """
-#     custom widget to pick start and end dates for controls graphs
-#     """    
-#     def __init__(self) -> None:
-#         super().__init__()
-
-#         self.start_date = QDateEdit(calendarPopup=True)
-#         # start date is two months prior to end date by default
-#         twomonthsago = QDate.currentDate().addDays(-60)
-#         self.start_date.setDate(twomonthsago)
-#         self.end_date = QDateEdit(calendarPopup=True)
-#         self.end_date.setDate(QDate.currentDate())
-#         self.layout = QHBoxLayout()
-#         self.layout.addWidget(QLabel("Start Date"))
-#         self.layout.addWidget(self.start_date)
-#         self.layout.addWidget(QLabel("End Date"))
-#         self.layout.addWidget(self.end_date)
-#         self.setLayout(self.layout)
-#         self.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed)
-
-#     def sizeHint(self) -> QSize:
-#         return QSize(80,20)  
-
 class FirstStrandSalvage(QDialog):
 
     def __init__(self, ctx:Settings, submitter_id:str, rsl_plate_num:str|None=None) -> None:
@@ -429,321 +191,46 @@ class FirstStrandPlateList(QDialog):
             output.append(plate.currentText())
         return output
 
-# class ReagentFormWidget(QWidget):
+class LogParser(QDialog):
 
-#     def __init__(self, parent:QWidget, reagent:PydReagent, extraction_kit:str):
-#         super().__init__(parent)
-#         # self.setParent(parent)
-#         self.reagent = reagent
-#         self.extraction_kit = extraction_kit
-#         # self.ctx = reagent.ctx
-#         layout = QVBoxLayout()
-#         self.label = self.ReagentParsedLabel(reagent=reagent)
-#         layout.addWidget(self.label)
-#         self.lot = self.ReagentLot(reagent=reagent, extraction_kit=extraction_kit)
-#         layout.addWidget(self.lot)
-#         # Remove spacing between reagents
-#         layout.setContentsMargins(0,0,0,0)
-#         self.setLayout(layout)
-#         self.setObjectName(reagent.name)
-#         self.missing = reagent.missing
-#         # If changed set self.missing to True and update self.label
-#         self.lot.currentTextChanged.connect(self.updated)
-
-#     def parse_form(self) -> Tuple[PydReagent, dict]:
-#         lot = self.lot.currentText()
-#         # wanted_reagent = lookup_reagents(ctx=self.ctx, lot_number=lot, reagent_type=self.reagent.type)
-#         wanted_reagent = Reagent.query(lot_number=lot, reagent_type=self.reagent.type)
-#         # if reagent doesn't exist in database, off to add it (uses App.add_reagent)
-#         if wanted_reagent == None:
-#             dlg = QuestionAsker(title=f"Add {lot}?", message=f"Couldn't find reagent type {self.reagent.type}: {lot} in the database.\n\nWould you like to add it?")
-#             if dlg.exec():
-#                 wanted_reagent = self.parent().parent().parent().parent().parent().parent().parent().parent().parent.add_reagent(reagent_lot=lot, reagent_type=self.reagent.type, expiry=self.reagent.expiry, name=self.reagent.name)
-#                 return wanted_reagent, None
-#             else:
-#                 # In this case we will have an empty reagent and the submission will fail kit integrity check
-#                 logger.debug("Will not add reagent.")
-#                 return None, Result(msg="Failed integrity check", status="Critical")
-#         else:
-#             # Since this now gets passed in directly from the parser -> pyd -> form and the parser gets the name
-#             # from the db, it should no longer be necessary to query the db with reagent/kit, but with rt name directly.
-#             # rt = lookup_reagent_types(ctx=self.ctx, name=self.reagent.type)
-#             # rt = lookup_reagent_types(ctx=self.ctx, kit_type=self.extraction_kit, reagent=wanted_reagent)
-#             rt = ReagentType.query(name=self.reagent.type)
-#             if rt == None:
-#                 # rt = lookup_reagent_types(ctx=self.ctx, kit_type=self.extraction_kit, reagent=wanted_reagent)
-#                 rt = ReagentType.query(kit_type=self.extraction_kit, reagent=wanted_reagent)
-#             return PydReagent(name=wanted_reagent.name, lot=wanted_reagent.lot, type=rt.name, expiry=wanted_reagent.expiry, parsed=not self.missing), None
-
-#     def updated(self):
-#         self.missing = True
-#         self.label.updated(self.reagent.type)
+    def __init__(self, parent):
+        super().__init__(parent)
+        self.app = self.parent()
+        self.filebutton = QPushButton(self)
+        self.filebutton.setText("Import File")
+        self.phrase_looker = QComboBox(self)
+        self.phrase_looker.setEditable(True)
+        self.btn = QPushButton(self)
+        self.btn.setText("Search")
+        self.layout = QFormLayout(self)
+        self.layout.addRow(self.tr("&File:"), self.filebutton)
+        self.layout.addRow(self.tr("&Search Term:"), self.phrase_looker)
+        self.layout.addRow(self.btn)
+        self.filebutton.clicked.connect(self.filelookup)
+        self.btn.clicked.connect(self.runsearch)
+        self.setMinimumWidth(400)
 
 
-#     class ReagentParsedLabel(QLabel):
+    def filelookup(self):
+        self.fname = select_open_file(self, "tabular")
 
-#         def __init__(self, reagent:PydReagent):
-#             super().__init__()
-#             try:
-#                 check = not reagent.missing
-#             except:
-#                 check = False
-#             self.setObjectName(f"{reagent.type}_label")
-#             if check:
-#                 self.setText(f"Parsed {reagent.type}")
-#             else:
-#                 self.setText(f"MISSING {reagent.type}")
-        
-#         def updated(self, reagent_type:str):
-#             self.setText(f"UPDATED {reagent_type}")
-
-#     class ReagentLot(QComboBox):
-
-#         def __init__(self, reagent, extraction_kit:str) -> None:
-#             super().__init__()
-#             # self.ctx = reagent.ctx
-#             self.setEditable(True)
-#             # if reagent.parsed:
-#             #     pass
-#             logger.debug(f"Attempting lookup of reagents by type: {reagent.type}")
-#             # below was lookup_reagent_by_type_name_and_kit_name, but I couldn't get it to work.
-#             # lookup = lookup_reagents(ctx=self.ctx, reagent_type=reagent.type)
-#             lookup = Reagent.query(reagent_type=reagent.type)
-#             relevant_reagents = [item.__str__() for item in lookup]
-#             output_reg = []
-#             for rel_reagent in relevant_reagents:
-#             # extract strings from any sets.
-#                 if isinstance(rel_reagent, set):
-#                     for thing in rel_reagent:
-#                         output_reg.append(thing)
-#                 elif isinstance(rel_reagent, str):
-#                     output_reg.append(rel_reagent)
-#             relevant_reagents = output_reg
-#             # if reagent in sheet is not found insert it into the front of relevant reagents so it shows 
-#             logger.debug(f"Relevant reagents for {reagent.lot}: {relevant_reagents}")
-#             if str(reagent.lot) not in relevant_reagents:
-#                 if check_not_nan(reagent.lot):
-#                     relevant_reagents.insert(0, str(reagent.lot))
-#                 else:
-#                     # TODO: look up the last used reagent of this type in the database
-#                     # looked_up_rt = lookup_reagenttype_kittype_association(ctx=self.ctx, reagent_type=reagent.type, kit_type=extraction_kit)
-#                     looked_up_rt = KitTypeReagentTypeAssociation.query(reagent_type=reagent.type, kit_type=extraction_kit)
-#                     try:
-#                         # looked_up_reg = lookup_reagents(ctx=self.ctx, lot_number=looked_up_rt.last_used)
-#                         looked_up_reg = Reagent.query(lot_number=looked_up_rt.last_used)
-#                     except AttributeError:
-#                         looked_up_reg = None
-#                     logger.debug(f"Because there was no reagent listed for {reagent.lot}, we will insert the last lot used: {looked_up_reg}")
-#                     if looked_up_reg != None:
-#                         relevant_reagents.remove(str(looked_up_reg.lot))
-#                         relevant_reagents.insert(0, str(looked_up_reg.lot))
-#             else:
-#                 if len(relevant_reagents) > 1:
-#                     logger.debug(f"Found {reagent.lot} in relevant reagents: {relevant_reagents}. Moving to front of list.")
-#                     idx = relevant_reagents.index(str(reagent.lot))
-#                     logger.debug(f"The index we got for {reagent.lot} in {relevant_reagents} was {idx}")
-#                     moved_reag = relevant_reagents.pop(idx)
-#                     relevant_reagents.insert(0, moved_reag)
-#                 else:
-#                     logger.debug(f"Found {reagent.lot} in relevant reagents: {relevant_reagents}. But no need to move due to short list.")
-#             logger.debug(f"New relevant reagents: {relevant_reagents}")
-#             self.setObjectName(f"lot_{reagent.type}")
-#             self.addItems(relevant_reagents)
-
-# class SubmissionFormWidget(QWidget):
-
-#     def __init__(self, parent: QWidget, **kwargs) -> None:
-#         super().__init__(parent)
-#         # self.ignore = [None, "", "qt_spinbox_lineedit", "qt_scrollarea_viewport", "qt_scrollarea_hcontainer",
-#         #                "qt_scrollarea_vcontainer", "submit_btn"
-#         #                ]
-#         self.ignore = ['filepath', 'samples', 'reagents', 'csv', 'ctx']
-#         layout = QVBoxLayout()
-#         for k, v in kwargs.items():
-#             if k not in self.ignore:
-#                 add_widget = self.create_widget(key=k, value=v, submission_type=kwargs['submission_type'])
-#                 if add_widget != None:
-#                     layout.addWidget(add_widget)
-#             else:
-#                 setattr(self, k, v)
-        
-#         self.setLayout(layout)
-
-#     def create_widget(self, key:str, value:dict, submission_type:str|None=None):
-#         if key not in self.ignore:
-#             return self.InfoItem(self, key=key, value=value, submission_type=submission_type)
-#         return None
-        
-#     def clear_form(self):
-#         for item in self.findChildren(QWidget):
-#             item.setParent(None)
-
-#     def find_widgets(self, object_name:str|None=None) -> List[QWidget]:
-#         query = self.findChildren(QWidget)
-#         if object_name != None:
-#             query = [widget for widget in query if widget.objectName()==object_name]
-#         return query
-    
-#     def parse_form(self) -> PydSubmission:
-#         logger.debug(f"Hello from form parser!")
-#         info = {}
-#         reagents = []
-#         if hasattr(self, 'csv'):
-#             info['csv'] = self.csv
-#         for widget in self.findChildren(QWidget):
-#             # logger.debug(f"Parsed widget of type {type(widget)}")
-#             match widget:
-#                 case ReagentFormWidget():
-#                     reagent, _ = widget.parse_form()
-#                     if reagent != None:
-#                         reagents.append(reagent)
-#                 case self.InfoItem():
-#                     field, value = widget.parse_form()
-#                     if field != None:
-#                         info[field] = value
-#         logger.debug(f"Info: {pformat(info)}")
-#         logger.debug(f"Reagents: {pformat(reagents)}")
-#         # app = self.parent().parent().parent().parent().parent().parent().parent().parent
-#         submission = PydSubmission(filepath=self.filepath, reagents=reagents, samples=self.samples, **info)
-#         return submission
-    
-#     class InfoItem(QWidget):
-
-#         def __init__(self, parent: QWidget, key:str, value:dict, submission_type:str|None=None) -> None:
-#             super().__init__(parent)
-#             layout = QVBoxLayout()
-#             self.label = self.ParsedQLabel(key=key, value=value)
-#             self.input: QWidget = self.set_widget(parent=self, key=key, value=value, submission_type=submission_type['value'])
-#             self.setObjectName(key)
-#             try:
-#                 self.missing:bool = value['missing']
-#             except (TypeError, KeyError):
-#                 self.missing:bool = True
-#             if self.input != None:
-#                 layout.addWidget(self.label)
-#                 layout.addWidget(self.input)
-#             layout.setContentsMargins(0,0,0,0)
-#             self.setLayout(layout)
-#             match self.input:
-#                 case QComboBox():
-#                     self.input.currentTextChanged.connect(self.update_missing)
-#                 case QDateEdit():
-#                     self.input.dateChanged.connect(self.update_missing)
-#                 case QLineEdit():
-#                     self.input.textChanged.connect(self.update_missing)
-            
-#         def parse_form(self):
-#             match self.input:
-#                 case QLineEdit():
-#                     value = self.input.text()
-#                 case QComboBox():
-#                     value = self.input.currentText()
-#                 case QDateEdit():
-#                     value = self.input.date().toPyDate()
-#                 case _:
-#                     return None, None
-#             return self.input.objectName(), dict(value=value, missing=self.missing)
-        
-#         def set_widget(self, parent: QWidget, key:str, value:dict, submission_type:str|None=None) -> QWidget:
-#             try:
-#                 value = value['value']
-#             except (TypeError, KeyError):
-#                 pass
-#             obj = parent.parent().parent()
-#             logger.debug(f"Creating widget for: {key}")
-#             match key:
-#                 case 'submitting_lab':
-#                     add_widget = QComboBox()
-#                     # lookup organizations suitable for submitting_lab (ctx: self.InfoItem.SubmissionFormWidget.SubmissionFormContainer.AddSubForm )
-#                     labs = [item.__str__() for item in Organization.query()]
-#                     # try to set closest match to top of list
-#                     try:
-#                         labs = difflib.get_close_matches(value, labs, len(labs), 0)
-#                     except (TypeError, ValueError):
-#                         pass
-#                     # set combobox values to lookedup values
-#                     add_widget.addItems(labs)
-#                 case 'extraction_kit':
-#                     # if extraction kit not available, all other values fail
-#                     if not check_not_nan(value):
-#                         msg = AlertPop(message="Make sure to check your extraction kit in the excel sheet!", status="warning")
-#                         msg.exec()
-#                     # create combobox to hold looked up kits
-#                     add_widget = QComboBox()
-#                     # lookup existing kits by 'submission_type' decided on by sheetparser
-#                     logger.debug(f"Looking up kits used for {submission_type}")
-#                     uses = [item.__str__() for item in KitType.query(used_for=submission_type)]
-#                     obj.uses = uses
-#                     logger.debug(f"Kits received for {submission_type}: {uses}")
-#                     if check_not_nan(value):
-#                         logger.debug(f"The extraction kit in parser was: {value}")
-#                         uses.insert(0, uses.pop(uses.index(value)))
-#                         obj.ext_kit = value
-#                     else:
-#                         logger.error(f"Couldn't find {obj.prsr.sub['extraction_kit']}")
-#                         obj.ext_kit = uses[0]
-#                     add_widget.addItems(uses)
-#                     # Run reagent scraper whenever extraction kit is changed.
-#                     # add_widget.currentTextChanged.connect(obj.scrape_reagents)
-#                 case 'submitted_date':
-#                     # uses base calendar
-#                     add_widget = QDateEdit(calendarPopup=True)
-#                     # sets submitted date based on date found in excel sheet
-#                     try:
-#                         add_widget.setDate(value)
-#                     # if not found, use today
-#                     except:
-#                         add_widget.setDate(date.today())
-#                 case 'submission_category':
-#                     add_widget = QComboBox()
-#                     cats = ['Diagnostic', "Surveillance", "Research"]
-#                     # cats += [item.name for item in lookup_submission_type(ctx=obj.ctx)]
-#                     cats += [item.name for item in SubmissionType.query()]
-#                     try:
-#                         cats.insert(0, cats.pop(cats.index(value)))
-#                     except ValueError:
-#                         cats.insert(0, cats.pop(cats.index(submission_type)))
-#                     add_widget.addItems(cats)
-#                 case _:
-#                     # anything else gets added in as a line edit
-#                     add_widget = QLineEdit()
-#                     logger.debug(f"Setting widget text to {str(value).replace('_', ' ')}")
-#                     add_widget.setText(str(value).replace("_", " "))
-#             if add_widget != None:
-#                 add_widget.setObjectName(key)
-#                 add_widget.setParent(parent)
-                
-#             return add_widget
-            
-#         def update_missing(self):
-#             self.missing = True
-#             self.label.updated(self.objectName())
-
-#         class ParsedQLabel(QLabel):
-
-#             def __init__(self, key:str, value:dict, title:bool=True, label_name:str|None=None):
-#                 super().__init__()
-#                 try:
-#                     check = not value['missing']
-#                 except:
-#                     check = True
-#                 if label_name != None:
-#                     self.setObjectName(label_name)
-#                 else:
-#                     self.setObjectName(f"{key}_label")
-#                 if title:
-#                     output = key.replace('_', ' ').title()
-#                 else:
-#                     output = key.replace('_', ' ')
-#                 if check:
-#                     self.setText(f"Parsed {output}")
-#                 else:
-#                     self.setText(f"MISSING {output}")
-
-#             def updated(self, key:str, title:bool=True):
-#                 if title:
-#                     output = key.replace('_', ' ').title()
-#                 else:
-#                     output = key.replace('_', ' ')
-#                 self.setText(f"UPDATED {output}")
+    def runsearch(self):
+        count: int = 0
+        total: int = 0
+        logger.debug(f"Current search term: {self.phrase_looker.currentText()}")
+        try:
+            with open(self.fname, "r") as f:
+                for chunk in readInChunks(fileObj=f):
+                    total += len(chunk)
+                    for line in chunk:
+                        if self.phrase_looker.currentText().lower() in line.lower():
+                            count += 1
+            percent = (count/total)*100
+            msg = f"I found {count} instances of the search phrase out of {total} = {percent:.2f}%."
+            status = "Information"
+        except AttributeError:
+            msg = f"No file was selected."
+            status = "Error"
+        dlg = AlertPop(message=msg, status=status)
+        dlg.exec()
 

src/submissions/frontend/widgets/submission_widget.py

@@ -12,7 +12,7 @@ from backend.excel.parser import SheetParser, PCRParser
 from backend.validators import PydSubmission, PydReagent
 from backend.db import (
     check_kit_integrity, KitType, Organization, SubmissionType, Reagent, 
-    ReagentType, KitTypeReagentTypeAssociation, BasicSubmission, update_subsampassoc_with_pcr
+    ReagentType, KitTypeReagentTypeAssociation, BasicSubmission
 )
 from pprint import pformat
 from .pop_ups import QuestionAsker, AlertPop
@@ -22,7 +22,6 @@ import difflib
 from datetime import date
 import inspect
 import json
-import sys
 
 logger = logging.getLogger(f"submissions.{__name__}")
 

src/submissions/tools.py

@@ -224,7 +224,6 @@ class Settings(BaseSettings):
             engine = create_engine(f"sqlite:///{database_path}")#, echo=True, future=True)
             session = Session(engine)
             metadata.session = session
-            
             return session
 
     @field_validator('package', mode="before")
@@ -513,4 +512,14 @@ class Report(BaseModel):
             case _:
                 pass
 
+def readInChunks(fileObj, chunkSize=2048):
+    """
+    Lazy function to read a file piece by piece.
+    Default chunk size: 2kB.
 
+    """
+    while True:
+        data = fileObj.readlines(chunkSize)
+        if not data:
+            break
+        yield data