diff --git a/alembic/versions/3d9a88bd4ecd_added_in_other_ww_techs.py b/alembic/versions/3d9a88bd4ecd_added_in_other_ww_techs.py new file mode 100644 index 0000000..2160586 --- /dev/null +++ b/alembic/versions/3d9a88bd4ecd_added_in_other_ww_techs.py @@ -0,0 +1,34 @@ +"""added in other ww techs + +Revision ID: 3d9a88bd4ecd +Revises: f7f46e72f057 +Create Date: 2023-08-30 11:03:41.575219 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '3d9a88bd4ecd' +down_revision = 'f7f46e72f057' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('_submissions', schema=None) as batch_op: + batch_op.add_column(sa.Column('ext_technician', sa.String(length=64), nullable=True)) + batch_op.add_column(sa.Column('pcr_technician', sa.String(length=64), nullable=True)) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('_submissions', schema=None) as batch_op: + batch_op.drop_column('pcr_technician') + batch_op.drop_column('ext_technician') + + # ### end Alembic commands ### diff --git a/alembic/versions/9a133efb3ffd_adjusting_reagents_reagenttypes_to_many_.py b/alembic/versions/9a133efb3ffd_adjusting_reagents_reagenttypes_to_many_.py new file mode 100644 index 0000000..1729bc5 --- /dev/null +++ b/alembic/versions/9a133efb3ffd_adjusting_reagents_reagenttypes_to_many_.py @@ -0,0 +1,33 @@ +"""adjusting reagents/reagenttypes to many-to-many + +Revision ID: 9a133efb3ffd +Revises: 3d9a88bd4ecd +Create Date: 2023-09-01 10:28:22.335890 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '9a133efb3ffd' +down_revision = '3d9a88bd4ecd' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('_reagenttypes_reagents', + sa.Column('reagent_id', sa.INTEGER(), nullable=True), + sa.Column('reagenttype_id', sa.INTEGER(), nullable=True), + sa.ForeignKeyConstraint(['reagent_id'], ['_reagents.id'], ), + sa.ForeignKeyConstraint(['reagenttype_id'], ['_reagent_types.id'], ) + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('_reagenttypes_reagents') + # ### end Alembic commands ### diff --git a/alembic/versions/cac89ced412b_rebuild_database.py b/alembic/versions/f7f46e72f057_rebuild_database.py similarity index 97% rename from alembic/versions/cac89ced412b_rebuild_database.py rename to alembic/versions/f7f46e72f057_rebuild_database.py index b73e843..aa1415b 100644 --- a/alembic/versions/cac89ced412b_rebuild_database.py +++ b/alembic/versions/f7f46e72f057_rebuild_database.py @@ -1,8 +1,8 @@ """rebuild database -Revision ID: cac89ced412b +Revision ID: f7f46e72f057 Revises: -Create Date: 2023-08-25 14:03:48.883090 +Create Date: 2023-08-30 09:47:18.071070 """ from alembic import op @@ -10,7 +10,7 @@ import sqlalchemy as sa # revision identifiers, used by Alembic. -revision = 'cac89ced412b' +revision = 'f7f46e72f057' down_revision = None branch_labels = None depends_on = None @@ -168,8 +168,8 @@ def upgrade() -> None: op.create_table('_submission_sample', sa.Column('sample_id', sa.INTEGER(), nullable=False), sa.Column('submission_id', sa.INTEGER(), nullable=False), - sa.Column('row', sa.INTEGER(), nullable=True), - sa.Column('column', sa.INTEGER(), nullable=True), + sa.Column('row', sa.INTEGER(), nullable=False), + sa.Column('column', sa.INTEGER(), nullable=False), sa.Column('base_sub_type', sa.String(), nullable=True), sa.Column('ct_n1', sa.FLOAT(precision=2), nullable=True), sa.Column('ct_n2', sa.FLOAT(precision=2), nullable=True), @@ -178,7 +178,7 @@ def upgrade() -> None: sa.Column('pcr_results', sa.JSON(), nullable=True), sa.ForeignKeyConstraint(['sample_id'], ['_samples.id'], ), sa.ForeignKeyConstraint(['submission_id'], ['_submissions.id'], ), - sa.PrimaryKeyConstraint('sample_id', 'submission_id') + sa.PrimaryKeyConstraint('submission_id', 'row', 'column') ) # ### end Alembic commands ### diff --git a/src/submissions/backend/db/functions.py b/src/submissions/backend/db/functions.py index 8320205..a80c211 100644 --- a/src/submissions/backend/db/functions.py +++ b/src/submissions/backend/db/functions.py @@ -2,6 +2,7 @@ Convenience functions for interacting with the database. ''' +import pprint from . import models # from .models.kits import KitType # from .models.submissions import BasicSample, reagents_submissions, BasicSubmission, SubmissionSampleAssociation @@ -33,7 +34,6 @@ def set_sqlite_pragma(dbapi_connection, connection_record): cursor.execute("PRAGMA foreign_keys=ON") cursor.close() - def store_submission(ctx:Settings, base_submission:models.BasicSubmission, samples:List[dict]=[]) -> None|dict: """ Upserts submissions into database @@ -206,10 +206,12 @@ def construct_submission_info(ctx:Settings, info_dict:dict) -> models.BasicSubmi try: with ctx.database_session.no_autoflush: try: - logger.debug(f"Here is the sample instance type: {sample_instance.sample_type}") + sample_query = sample_instance.sample_type.replace('Sample', '').strip() + logger.debug(f"Here is the sample instance type: {sample_query}") try: - assoc = getattr(models, f"{sample_instance.sample_type.replace('_sample', '').replace('_', ' ').title().replace(' ', '')}Association") + assoc = getattr(models, f"{sample_query}Association") except AttributeError as e: + logger.error(f"Couldn't get type specific association. Getting generic.") assoc = models.SubmissionSampleAssociation # assoc = models.SubmissionSampleAssociation(submission=instance, sample=sample_instance, row=sample['row'], column=sample['column']) assoc = assoc(submission=instance, sample=sample_instance, row=sample['row'], column=sample['column']) @@ -287,7 +289,9 @@ def construct_reagent(ctx:Settings, info_dict:dict) -> models.Reagent: case "expiry": reagent.expiry = info_dict[item] case "type": - reagent.type = lookup_reagenttype_by_name(ctx=ctx, rt_name=info_dict[item]) + reagent_type = lookup_reagenttype_by_name(ctx=ctx, rt_name=info_dict[item]) + if reagent_type != None: + reagent.type.append(reagent_type) case "name": if item == None: reagent.name = reagent.type.name @@ -420,7 +424,7 @@ def lookup_regent_by_type_name_and_kit_name(ctx:Settings, type_name:str, kit_nam output = rt_types.instances return output -def lookup_all_submissions_by_type(ctx:Settings, sub_type:str|None=None) -> list[models.BasicSubmission]: +def lookup_all_submissions_by_type(ctx:Settings, sub_type:str|None=None, chronologic:bool=False) -> list[models.BasicSubmission]: """ Get all submissions, filtering by type if given @@ -433,11 +437,13 @@ def lookup_all_submissions_by_type(ctx:Settings, sub_type:str|None=None) -> list """ if sub_type == None: # subs = ctx['database_session'].query(models.BasicSubmission).all() - subs = ctx.database_session.query(models.BasicSubmission).all() + subs = ctx.database_session.query(models.BasicSubmission) else: # subs = ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.submission_type==sub_type.lower().replace(" ", "_")).all() - subs = ctx.database_session.query(models.BasicSubmission).filter(models.BasicSubmission.submission_type==sub_type.lower().replace(" ", "_")).all() - return subs + subs = ctx.database_session.query(models.BasicSubmission).filter(models.BasicSubmission.submission_type==sub_type.lower().replace(" ", "_")) + if chronologic: + subs.order_by(models.BasicSubmission.submitted_date) + return subs.all() def lookup_all_orgs(ctx:Settings) -> list[models.Organization]: """ @@ -480,7 +486,7 @@ def submissions_to_df(ctx:Settings, sub_type:str|None=None) -> pd.DataFrame: """ logger.debug(f"Type: {sub_type}") # use lookup function to create list of dicts - subs = [item.to_dict() for item in lookup_all_submissions_by_type(ctx=ctx, sub_type=sub_type)] + subs = [item.to_dict() for item in lookup_all_submissions_by_type(ctx=ctx, sub_type=sub_type, chronologic=True)] # make df from dicts (records) in list df = pd.DataFrame.from_records(subs) # Exclude sub information @@ -569,7 +575,9 @@ def create_kit_from_yaml(ctx:Settings, exp:dict) -> dict: # continue # A submission type may use multiple kits. for kt in exp[type]['kits']: + logger.debug(f"Looking up submission type: {type}") submission_type = lookup_submissiontype_by_name(ctx=ctx, type_name=type) + logger.debug(f"Looked up submission type: {submission_type}") kit = models.KitType(name=kt, # constant_cost=exp[type]["kits"][kt]["constant_cost"], # mutable_cost_column=exp[type]["kits"][kt]["mutable_cost_column"], @@ -588,7 +596,7 @@ def create_kit_from_yaml(ctx:Settings, exp:dict) -> dict: look_up = ctx.database_session.query(models.ReagentType).filter(models.ReagentType.name==r).first() if look_up == None: # rt = models.ReagentType(name=r.replace(" ", "_").lower(), eol_ext=timedelta(30*exp[type]['kits'][kt]['reagenttypes'][r]['eol_ext']), kits=[kit], required=1) - rt = models.ReagentType(name=r.replace(" ", "_").lower().strip(), eol_ext=timedelta(30*exp[type]['kits'][kt]['reagenttypes'][r]['eol_ext']), last_used="") + rt = models.ReagentType(name=r.strip(), eol_ext=timedelta(30*exp[type]['kits'][kt]['reagenttypes'][r]['eol_ext']), last_used="") else: rt = look_up # rt.kits.append(kit) @@ -893,9 +901,10 @@ def update_ww_sample(ctx:Settings, sample_obj:dict): sample_obj (dict): dictionary representing new values for database object """ # ww_samp = lookup_ww_sample_by_rsl_sample_number(ctx=ctx, rsl_number=sample_obj['sample']) + logger.debug(f"dictionary to use for update: {pprint.pformat(sample_obj)}") logger.debug(f"Looking up {sample_obj['sample']} in plate {sample_obj['plate_rsl']}") # ww_samp = lookup_ww_sample_by_sub_sample_rsl(ctx=ctx, sample_rsl=sample_obj['sample'], plate_rsl=sample_obj['plate_rsl']) - assoc = lookup_ww_association_by_plate_sample(ctx=ctx, rsl_plate_num=sample_obj['plate_rsl'], rsl_sample_num=sample_obj['sample']) + assoc = lookup_subsamp_association_by_plate_sample(ctx=ctx, rsl_plate_num=sample_obj['plate_rsl'], rsl_sample_num=sample_obj['sample']) # ww_samp = lookup_ww_sample_by_sub_sample_well(ctx=ctx, sample_rsl=sample_obj['sample'], well_num=sample_obj['well_num'], plate_rsl=sample_obj['plate_rsl']) if assoc != None: # del sample_obj['well_number'] @@ -903,11 +912,16 @@ def update_ww_sample(ctx:Settings, sample_obj:dict): # set attribute 'key' to 'value' try: check = getattr(assoc, key) - except AttributeError: + except AttributeError as e: + logger.error(f"Item doesn't have field {key} due to {e}") continue - if check == None: - logger.debug(f"Setting {key} to {value}") - setattr(assoc, key, value) + if check != value: + logger.debug(f"Setting association key: {key} to {value}") + try: + setattr(assoc, key, value) + except AttributeError as e: + logger.error(f"Can't set field {key} to {value} due to {e}") + continue else: logger.error(f"Unable to find sample {sample_obj['sample']}") return @@ -1059,16 +1073,22 @@ def check_kit_integrity(sub:models.BasicSubmission|models.KitType, reagenttypes: """ logger.debug(type(sub)) # What type is sub? + reagenttypes = [] match sub: case models.BasicSubmission(): # Get all required reagent types for this kit. # ext_kit_rtypes = [reagenttype.name for reagenttype in sub.extraction_kit.reagent_types if reagenttype.required == 1] ext_kit_rtypes = [item.name for item in sub.extraction_kit.get_reagents(required=True)] # Overwrite function parameter reagenttypes - try: - reagenttypes = [reagent.type.name for reagent in sub.reagents] - except AttributeError as e: - logger.error(f"Problem parsing reagents: {[f'{reagent.lot}, {reagent.type}' for reagent in sub.reagents]}") + for reagent in sub.reagents: + try: + # reagenttypes = [reagent.type.name for reagent in sub.reagents] + rt = list(set(reagent.type).intersection(sub.extraction_kit.reagent_types))[0].name + logger.debug(f"Got reagent type: {rt}") + reagenttypes.append(rt) + except AttributeError as e: + logger.error(f"Problem parsing reagents: {[f'{reagent.lot}, {reagent.type}' for reagent in sub.reagents]}") + reagenttypes.append(reagent.type[0].name) case models.KitType(): # ext_kit_rtypes = [reagenttype.name for reagenttype in sub.reagent_types if reagenttype.required == 1] ext_kit_rtypes = [item.name for item in sub.get_reagents(required=True)] @@ -1133,7 +1153,7 @@ def get_reagents_in_extkit(ctx:Settings, kit_name:str) -> List[str]: kit = lookup_kittype_by_name(ctx=ctx, name=kit_name) return kit.get_reagents(required=False) -def lookup_ww_association_by_plate_sample(ctx:Settings, rsl_plate_num:str, rsl_sample_num:str) -> models.SubmissionSampleAssociation: +def lookup_subsamp_association_by_plate_sample(ctx:Settings, rsl_plate_num:str, rsl_sample_num:str) -> models.SubmissionSampleAssociation: """ _summary_ @@ -1147,9 +1167,9 @@ def lookup_ww_association_by_plate_sample(ctx:Settings, rsl_plate_num:str, rsl_s """ return ctx.database_session.query(models.SubmissionSampleAssociation)\ .join(models.BasicSubmission)\ - .join(models.WastewaterSample)\ + .join(models.BasicSample)\ .filter(models.BasicSubmission.rsl_plate_num==rsl_plate_num)\ - .filter(models.WastewaterSample.rsl_number==rsl_sample_num)\ + .filter(models.BasicSample.submitter_id==rsl_sample_num)\ .first() def lookup_all_reagent_names_by_role(ctx:Settings, role_name:str) -> List[str]: @@ -1181,4 +1201,25 @@ def lookup_submissiontype_by_name(ctx:Settings, type_name:str) -> models.Submiss models.SubmissionType: _description_ """ - return ctx.database_session.query(models.SubmissionType).filter(models.SubmissionType.name==type_name).first() \ No newline at end of file + return ctx.database_session.query(models.SubmissionType).filter(models.SubmissionType.name==type_name).first() + +def add_reagenttype_to_kit(ctx:Settings, rt_name:str, kit_name:str, eol:int=0): + """ + Mostly commandline procedure to add missing reagenttypes to kits + + Args: + ctx (Settings): _description_ + rt_name (str): _description_ + kit_name (str): _description_ + eol (int, optional): _description_. Defaults to 0. + """ + kit = lookup_kittype_by_name(ctx=ctx, name=kit_name) + rt = lookup_reagenttype_by_name(ctx=ctx, rt_name=rt_name) + if rt == None: + rt = models.ReagentType(name=rt_name.strip(), eol_ext=timedelta(30*eol), last_used="") + ctx.database_session.add(rt) + assoc = models.KitTypeReagentTypeAssociation(kit_type=kit, reagent_type=rt, uses={}) + kit.kit_reagenttype_associations.append(assoc) + ctx.database_session.add(kit) + ctx.database_session.commit() + \ No newline at end of file diff --git a/src/submissions/backend/db/models/kits.py b/src/submissions/backend/db/models/kits.py index 30b9797..787c3c3 100644 --- a/src/submissions/backend/db/models/kits.py +++ b/src/submissions/backend/db/models/kits.py @@ -2,7 +2,7 @@ All kit and reagent related models ''' from . import Base -from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey, Interval, Table, FLOAT, CheckConstraint +from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey, Interval, Table, FLOAT from sqlalchemy.orm import relationship, validates from sqlalchemy.ext.associationproxy import association_proxy @@ -22,6 +22,8 @@ logger = logging.getLogger(f'submissions.{__name__}') # Column("required", INTEGER) # ) +reagenttypes_reagents = Table("_reagenttypes_reagents", Base.metadata, Column("reagent_id", INTEGER, ForeignKey("_reagents.id")), Column("reagenttype_id", INTEGER, ForeignKey("_reagent_types.id"))) + class KitType(Base): """ @@ -47,7 +49,7 @@ class KitType(Base): # association proxy of "user_keyword_associations" collection # to "keyword" attribute - reagent_types = association_proxy("kit_reagenttype_associations", "reagenttype") + reagent_types = association_proxy("kit_reagenttype_associations", "reagent_type") kit_submissiontype_associations = relationship( @@ -139,7 +141,7 @@ class ReagentType(Base): name = Column(String(64)) #: name of reagent type # kit_id = Column(INTEGER, ForeignKey("_kits.id", ondelete="SET NULL", use_alter=True, name="fk_RT_kits_id")) #: id of joined kit type # kits = relationship("KitType", back_populates="reagent_types", uselist=True, foreign_keys=[kit_id]) #: kits this reagent is used in - instances = relationship("Reagent", back_populates="type") #: concrete instances of this reagent type + instances = relationship("Reagent", back_populates="type", secondary=reagenttypes_reagents) #: concrete instances of this reagent type eol_ext = Column(Interval()) #: extension of life interval # required = Column(INTEGER, server_default="1") #: sqlite boolean to determine if reagent type is essential for the kit last_used = Column(String(32)) #: last used lot number of this type of reagent @@ -169,7 +171,7 @@ class Reagent(Base): __tablename__ = "_reagents" id = Column(INTEGER, primary_key=True) #: primary key - type = relationship("ReagentType", back_populates="instances") #: joined parent reagent type + type = relationship("ReagentType", back_populates="instances", secondary=reagenttypes_reagents) #: joined parent reagent type type_id = Column(INTEGER, ForeignKey("_reagent_types.id", ondelete='SET NULL', name="fk_reagent_type_id")) #: id of parent reagent type name = Column(String(64)) #: reagent name lot = Column(String(64)) #: lot number of reagent @@ -192,19 +194,26 @@ class Reagent(Base): """ return str(self.lot) - def to_sub_dict(self) -> dict: + def to_sub_dict(self, extraction_kit:KitType=None) -> dict: """ dictionary containing values necessary for gui Returns: dict: gui friendly dictionary """ + if extraction_kit != None: + try: + reagent_role = list(set(self.type).intersection(extraction_kit.reagent_types))[0] + except: + reagent_role = self.type[0] + else: + reagent_role = self.type[0] try: - type = self.type.name.replace("_", " ").title() + rtype = reagent_role.name.replace("_", " ").title() except AttributeError: - type = "Unknown" + rtype = "Unknown" try: - place_holder = self.expiry + self.type.eol_ext + place_holder = self.expiry + reagent_role.eol_ext # logger.debug(f"EOL_ext for {self.lot} -- {self.expiry} + {self.type.eol_ext} = {place_holder}") except TypeError as e: place_holder = date.today() @@ -213,14 +222,14 @@ class Reagent(Base): place_holder = date.today() logger.debug(f"We got an attribute error setting {self.lot} expiry: {e}. Setting to today for testing") return { - "type": type, + "type": rtype, "lot": self.lot, "expiry": place_holder.strftime("%Y-%m-%d") } def to_reagent_dict(self) -> dict: return { - "type": self.type.name, + "type": type, "lot": self.lot, "expiry": self.expiry.strftime("%Y-%m-%d") } @@ -279,4 +288,7 @@ class SubmissionTypeKitTypeAssociation(Base): self.submission_type = submission_type self.mutable_cost_column = 0.00 self.mutable_cost_sample = 0.00 - self.constant_cost = 0.00 \ No newline at end of file + self.constant_cost = 0.00 + + def __repr__(self) -> str: + return f" dict|None: @@ -511,7 +514,7 @@ class BacterialCultureSample(BasicSample): # rsl_plate_id = Column(INTEGER, ForeignKey("_submissions.id", ondelete="SET NULL", name="fk_BCS_sample_id")) #: id of parent plate # rsl_plate = relationship("BacterialCulture", back_populates="samples") #: relationship to parent plate - __mapper_args__ = {"polymorphic_identity": "bacterial_culture_sample", "polymorphic_load": "inline"} + __mapper_args__ = {"polymorphic_identity": "Bacterial Culture Sample", "polymorphic_load": "inline"} # def to_string(self) -> str: # """ @@ -543,10 +546,10 @@ class SubmissionSampleAssociation(Base): DOC: https://docs.sqlalchemy.org/en/14/orm/extensions/associationproxy.html """ __tablename__ = "_submission_sample" - sample_id = Column(INTEGER, ForeignKey("_samples.id"), primary_key=True) + sample_id = Column(INTEGER, ForeignKey("_samples.id"), nullable=False) submission_id = Column(INTEGER, ForeignKey("_submissions.id"), primary_key=True) - row = Column(INTEGER) - column = Column(INTEGER) + row = Column(INTEGER, primary_key=True) + column = Column(INTEGER, primary_key=True) submission = relationship(BasicSubmission, back_populates="submission_sample_associations") @@ -569,6 +572,9 @@ class SubmissionSampleAssociation(Base): self.row = row self.column = column + def __repr__(self) -> str: + return f" str: @@ -448,10 +449,10 @@ class InfoParser(object): logger.debug(f"Looking up submission type: {submission_type['value']}") submission_type = lookup_submissiontype_by_name(ctx=self.ctx, type_name=submission_type['value']) info_map = submission_type.info_map - try: - del info_map['samples'] - except KeyError: - pass + # try: + # del info_map['samples'] + # except KeyError: + # pass return info_map def parse_info(self) -> dict: @@ -472,14 +473,20 @@ class InfoParser(object): value = df.iat[relevant[item]['row']-1, relevant[item]['column']-1] logger.debug(f"Setting {item} on {sheet} to {value}") if check_not_nan(value): - try: - dicto[item] = dict(value=value, parsed=True) - except (KeyError, IndexError): - continue + if value != "None": + try: + dicto[item] = dict(value=value, parsed=True) + except (KeyError, IndexError): + continue + else: + try: + dicto[item] = dict(value=value, parsed=False) + except (KeyError, IndexError): + continue else: dicto[item] = dict(value=convert_nans_to_nones(value), parsed=False) - if "submitter_plate_num" not in dicto.keys(): - dicto['submitter_plate_num'] = dict(value=None, parsed=False) + # if "submitter_plate_num" not in dicto.keys(): + # dicto['submitter_plate_num'] = dict(value=None, parsed=False) return dicto class ReagentParser(object): @@ -554,6 +561,7 @@ class SampleParser(object): def fetch_sample_info_map(self, submission_type:dict) -> dict: logger.debug(f"Looking up submission type: {submission_type}") submission_type = lookup_submissiontype_by_name(ctx=self.ctx, type_name=submission_type) + logger.debug(f"info_map: {pprint.pformat(submission_type.info_map)}") sample_info_map = submission_type.info_map['samples'] return sample_info_map @@ -620,7 +628,13 @@ class SampleParser(object): def parse_samples(self) -> List[dict]: result = None new_samples = [] - for sample in self.samples: + for ii, sample in enumerate(self.samples): + # logger.debug(f"\n\n{new_samples}\n\n") + try: + if sample['submitter_id'] in [check_sample['sample'].submitter_id for check_sample in new_samples]: + sample['submitter_id'] = f"{sample['submitter_id']}-{ii}" + except KeyError as e: + logger.error(f"Sample obj: {sample}, error: {e}") translated_dict = {} for k, v in sample.items(): match v: @@ -647,11 +661,13 @@ class SampleParser(object): instance = lookup_sample_by_submitter_id(ctx=self.ctx, submitter_id=input_dict['submitter_id']) if instance == None: instance = database_obj() - for k,v in input_dict.items(): - try: - setattr(instance, k, v) - except Exception as e: - logger.error(f"Failed to set {k} due to {type(e).__name__}: {e}") + for k,v in input_dict.items(): + try: + setattr(instance, k, v) + except Exception as e: + logger.error(f"Failed to set {k} due to {type(e).__name__}: {e}") + else: + logger.debug(f"Sample already exists, will run update.") return dict(sample=instance, row=input_dict['row'], column=input_dict['column']) diff --git a/src/submissions/backend/pydant/__init__.py b/src/submissions/backend/pydant/__init__.py index 9f3a5cb..e831174 100644 --- a/src/submissions/backend/pydant/__init__.py +++ b/src/submissions/backend/pydant/__init__.py @@ -1,5 +1,5 @@ import uuid -from pydantic import BaseModel, field_validator, model_validator, Extra +from pydantic import BaseModel, field_validator, Extra from datetime import date, datetime from dateutil.parser import parse from dateutil.parser._parser import ParserError @@ -9,7 +9,6 @@ from pathlib import Path import re import logging from tools import check_not_nan, convert_nans_to_nones, Settings -import numpy as np from backend.db.functions import lookup_submission_by_rsl_num @@ -46,7 +45,11 @@ class PydReagent(BaseModel): # else: # return value if value != None: + if isinstance(value, int): + return datetime.fromordinal(datetime(1900, 1, 1).toordinal() + value - 2).date() return convert_nans_to_nones(str(value)) + if value == None: + value = date.today() return value @field_validator("name", mode="before") @@ -85,7 +88,7 @@ class PydSubmission(BaseModel, extra=Extra.allow): @classmethod def enforce_with_uuid(cls, value): logger.debug(f"submitter plate id: {value}") - if value['value'] == None: + if value['value'] == None or value['value'] == "None": return dict(value=uuid.uuid4().hex.upper(), parsed=False) else: return value diff --git a/src/submissions/frontend/custom_widgets/misc.py b/src/submissions/frontend/custom_widgets/misc.py index 47a1777..ad00683 100644 --- a/src/submissions/frontend/custom_widgets/misc.py +++ b/src/submissions/frontend/custom_widgets/misc.py @@ -89,7 +89,6 @@ class AddReagentForm(QDialog): self.name_input.clear() self.name_input.addItems(item for item in lookup_all_reagent_names_by_role(ctx=self.ctx, role_name=self.type_input.currentText().replace(" ", "_").lower())) - class ReportDatePicker(QDialog): """ custom dialog to ask for report start/stop dates @@ -118,7 +117,6 @@ class ReportDatePicker(QDialog): self.layout.addWidget(self.buttonBox) self.setLayout(self.layout) - class KitAdder(QWidget): """ dialog to get information to add kit @@ -195,7 +193,7 @@ class KitAdder(QWidget): yml_type['password'] = info['password'] except KeyError: pass - used = info['used_for'].replace(" ", "_").lower() + used = info['used_for'] yml_type[used] = {} yml_type[used]['kits'] = {} yml_type[used]['kits'][info['kit_name']] = {} @@ -210,7 +208,6 @@ class KitAdder(QWidget): msg.exec() self.__init__(self.ctx) - class ReagentTypeForm(QWidget): """ custom widget to add information about a new reagenttype @@ -234,7 +231,6 @@ class ReagentTypeForm(QWidget): eol.setMinimum(0) grid.addWidget(eol, 0,3) - class ControlsDatePicker(QWidget): """ custom widget to pick start and end dates for controls graphs @@ -259,7 +255,6 @@ class ControlsDatePicker(QWidget): def sizeHint(self) -> QSize: return QSize(80,20) - class ImportReagent(QComboBox): def __init__(self, ctx:dict, reagent:PydReagent): diff --git a/src/submissions/frontend/main_window_functions.py b/src/submissions/frontend/main_window_functions.py index b945370..f9a37be 100644 --- a/src/submissions/frontend/main_window_functions.py +++ b/src/submissions/frontend/main_window_functions.py @@ -26,13 +26,13 @@ from backend.db.functions import ( construct_submission_info, lookup_reagent, store_submission, lookup_submissions_by_date_range, create_kit_from_yaml, create_org_from_yaml, get_control_subtypes, get_all_controls_by_type, lookup_all_submissions_by_type, get_all_controls, lookup_submission_by_rsl_num, update_ww_sample, - check_kit_integrity, get_reagents_in_extkit + check_kit_integrity ) from backend.excel.parser import SheetParser, PCRParser from backend.excel.reports import make_report_html, make_report_xlsx, convert_data_list_to_df from backend.pydant import PydReagent from tools import check_not_nan -from .custom_widgets.pop_ups import AlertPop, KitSelector, QuestionAsker +from .custom_widgets.pop_ups import AlertPop, QuestionAsker from .custom_widgets import ReportDatePicker from .custom_widgets.misc import ImportReagent, ParsedQLabel from .visualizations.control_charts import create_charts, construct_html @@ -92,7 +92,7 @@ def import_submission_function(obj:QMainWindow) -> Tuple[QMainWindow, dict|None] for item in obj.table_widget.formlayout.parentWidget().findChildren(QWidget): item.setParent(None) # Get list of fields from pydantic model. - fields = list(pyd.model_fields.keys()) + fields = list(pyd.model_fields.keys()) + list(pyd.model_extra.keys()) fields.remove('filepath') logger.debug(f"pydantic fields: {fields}") for field in fields: @@ -175,7 +175,7 @@ def import_submission_function(obj:QMainWindow) -> Tuple[QMainWindow, dict|None] else: # try: # reg_label = QLabel(f"MISSING Lot: {reagent['value'].type}") - obj.missing_reagents.append(reagent['value'].type) + obj.missing_reagents.append(reagent['value']) continue # except AttributeError: # continue @@ -273,10 +273,10 @@ def kit_integrity_completion_function(obj:QMainWindow) -> Tuple[QMainWindow, dic # obj.missing_reagents = kit_integrity['missing'] # for item in kit_integrity['missing']: if len(obj.missing_reagents) > 0: - result = dict(message=f"The submission you are importing is missing some reagents expected by the kit.\n\nIt looks like you are missing: {[item.upper() for item in obj.missing_reagents]}\n\nAlternatively, you may have set the wrong extraction kit.\n\nThe program will populate lists using existing reagents.\n\nPlease make sure you check the lots carefully!", status="Warning") + result = dict(message=f"The submission you are importing is missing some reagents expected by the kit.\n\nIt looks like you are missing: {[item.type.upper() for item in obj.missing_reagents]}\n\nAlternatively, you may have set the wrong extraction kit.\n\nThe program will populate lists using existing reagents.\n\nPlease make sure you check the lots carefully!", status="Warning") for item in obj.missing_reagents: - obj.table_widget.formlayout.addWidget(ParsedQLabel({'parsed':False}, item, title=False)) - reagent = dict(type=item, lot=None, exp=None, name=None) + obj.table_widget.formlayout.addWidget(ParsedQLabel({'parsed':False}, item.type, title=False)) + reagent = dict(type=item.type, lot=None, exp=date.today(), name=None) add_widget = ImportReagent(ctx=obj.ctx, reagent=PydReagent(**reagent))#item=item) obj.table_widget.formlayout.addWidget(add_widget) submit_btn = QPushButton("Submit") @@ -316,8 +316,12 @@ def submit_new_sample_function(obj:QMainWindow) -> Tuple[QMainWindow, dict]: r_lot = reagents[reagent] dlg = QuestionAsker(title=f"Add {r_lot}?", message=f"Couldn't find reagent type {reagent.strip('Lot')}: {r_lot} in the database.\n\nWould you like to add it?") if dlg.exec(): - logger.debug(f"Looking through {obj.reagents} for reagent {reagent}") - picked_reagent = [item for item in obj.reagents if item.type == reagent][0] + logger.debug(f"Looking through {pprint.pformat(obj.reagents)} for reagent {reagent}") + try: + picked_reagent = [item for item in obj.reagents if item.type == reagent][0] + except IndexError: + logger.error(f"Couldn't find {reagent} in obj.reagents. Checking missing reagents {pprint.pformat(obj.missing_reagents)}") + picked_reagent = [item for item in obj.missing_reagents if item.type == reagent][0] logger.debug(f"checking reagent: {reagent} in obj.reagents. Result: {picked_reagent}") expiry_date = picked_reagent.exp wanted_reagent = obj.add_reagent(reagent_lot=r_lot, reagent_type=reagent.replace("lot_", ""), expiry=expiry_date, name=picked_reagent.name) @@ -369,14 +373,15 @@ def submit_new_sample_function(obj:QMainWindow) -> Tuple[QMainWindow, dict]: for item in obj.table_widget.formlayout.parentWidget().findChildren(QWidget): item.setParent(None) logger.debug(f"All attributes of obj: {pprint.pformat(obj.__dict__)}") - if len(obj.missing_reagents) > 0: + if len(obj.missing_reagents + obj.missing_info) > 0: logger.debug(f"We have blank reagents in the excel sheet.\n\tLet's try to fill them in.") extraction_kit = lookup_kittype_by_name(obj.ctx, name=obj.ext_kit) logger.debug(f"We have the extraction kit: {extraction_kit.name}") - logger.debug(f"Extraction kit map:\n\n{extraction_kit.used_for[obj.current_submission_type.replace('_', ' ')]}") + # TODO replace below with function in KitType object. Update Kittype associations. # excel_map = extraction_kit.used_for[obj.current_submission_type.replace('_', ' ')] excel_map = extraction_kit.construct_xl_map_for_use(obj.current_submission_type) + logger.debug(f"Extraction kit map:\n\n{pprint.pformat(excel_map)}") # excel_map.update(extraction_kit.used_for[obj.current_submission_type.replace('_', ' ').title()]) input_reagents = [item.to_reagent_dict() for item in parsed_reagents] autofill_excel(obj=obj, xl_map=excel_map, reagents=input_reagents, missing_reagents=obj.missing_reagents, info=info, missing_info=obj.missing_info) @@ -863,41 +868,47 @@ def autofill_excel(obj:QMainWindow, xl_map:dict, reagents:List[dict], missing_re logger.debug(f"Here is the info dict coming in:\n{pprint.pformat(info)}") logger.debug(f"Here are the missing reagents:\n{missing_reagents}") logger.debug(f"Here are the missing info:\n{missing_info}") + logger.debug(f"Here is the xl map: {pprint.pformat(xl_map)}") # pare down the xl map to only the missing data. - relevant_map = {k:v for k,v in xl_map.items() if k in missing_reagents} + relevant_reagent_map = {k:v for k,v in xl_map.items() if k in [reagent.type for reagent in missing_reagents]} # pare down reagents to only what's missing - relevant_reagents = [item for item in reagents if item['type'] in missing_reagents] + relevant_reagents = [item for item in reagents if item['type'] in [reagent.type for reagent in missing_reagents]] + logger.debug(f"Here are the relevant reagents: {pprint.pformat(relevant_reagents)}") # hacky manipulation of submission type so it looks better. # info['submission_type'] = info['submission_type'].replace("_", " ").title() # pare down info to just what's missing + relevant_info_map = {k:v for k,v in xl_map['info'].items() if k in missing_info and k != 'samples'} relevant_info = {k:v for k,v in info.items() if k in missing_info} logger.debug(f"Here is the relevant info: {pprint.pformat(relevant_info)}") # construct new objects to put into excel sheets: new_reagents = [] + logger.debug(f"Parsing from relevant reagent map: {pprint.pformat(relevant_reagent_map)}") for reagent in relevant_reagents: new_reagent = {} new_reagent['type'] = reagent['type'] - new_reagent['lot'] = relevant_map[new_reagent['type']]['lot'] + new_reagent['lot'] = relevant_reagent_map[new_reagent['type']]['lot'] new_reagent['lot']['value'] = reagent['lot'] - new_reagent['expiry'] = relevant_map[new_reagent['type']]['expiry'] + new_reagent['expiry'] = relevant_reagent_map[new_reagent['type']]['expiry'] new_reagent['expiry']['value'] = reagent['expiry'] - new_reagent['sheet'] = relevant_map[new_reagent['type']]['sheet'] + new_reagent['sheet'] = relevant_reagent_map[new_reagent['type']]['sheet'] # name is only present for Bacterial Culture try: - new_reagent['name'] = relevant_map[new_reagent['type']]['name'] + new_reagent['name'] = relevant_reagent_map[new_reagent['type']]['name'] new_reagent['name']['value'] = reagent['type'] except: pass new_reagents.append(new_reagent) # construct new info objects to put into excel sheets new_info = [] + logger.debug(f"Parsing from relevant info map: {pprint.pformat(relevant_info_map)}") for item in relevant_info: new_item = {} new_item['type'] = item - new_item['location'] = relevant_map[item] + new_item['location'] = relevant_info_map[item] new_item['value'] = relevant_info[item] new_info.append(new_item) logger.debug(f"New reagents: {new_reagents}") + logger.debug(f"New info: {new_info}") # open the workbook using openpyxl workbook = load_workbook(obj.xl) # get list of sheet names diff --git a/src/submissions/templates/submission_details.html b/src/submissions/templates/submission_details.html index 5cc664b..0d411fb 100644 --- a/src/submissions/templates/submission_details.html +++ b/src/submissions/templates/submission_details.html @@ -7,28 +7,16 @@

Submission Details for {{ sub['Plate Number'] }}

   

{% for key, value in sub.items() if key not in excluded %} - - - -     {{ key }}: {% if key=='Cost' %} {{ "${:,.2f}".format(value) }}{% else %}{{ value }}{% endif %}
- +     {{ key }}: {% if key=='Cost' %} {{ "${:,.2f}".format(value) }}{% else %}{{ value }}{% endif %}
{% endfor %}

Reagents:

{% for item in sub['reagents'] %} - - - -     {{ item['type'] }}: {{ item['lot'] }} (EXP: {{ item['expiry'] }})
- +     {{ item['type'] }}: {{ item['lot'] }} (EXP: {{ item['expiry'] }})
{% endfor %}

{% if sub['samples'] %}

Samples:

{% for item in sub['samples'] %} - - - -     {{ item['well'] }}: {{ item['name']|replace('\n\t', '
        ') }}
- +     {{ item['well'] }}: {{ item['name']|replace('\n\t', '
        ') }}
{% endfor %}

{% endif %} {% if sub['controls'] %} @@ -38,11 +26,7 @@ {% if item['kraken'] %}

   {{ item['name'] }} Top 5 Kraken Results:

{% for genera in item['kraken'] %} - - -         {{ genera['name'] }}: {{ genera['kraken_count'] }} ({{ genera['kraken_percent'] }})
- {% endfor %}

{% endif %} {% endfor %} @@ -51,15 +35,11 @@ {% for entry in sub['ext_info'] %}

Extraction Status:

{% for key, value in entry.items() %} - - - - {% if "column" in key %} -     {{ key|replace('_', ' ')|title() }}: {{ value }}uL
- {% else %} -     {{ key|replace('_', ' ')|title() }}: {{ value }}
- {% endif %} - + {% if "column" in key %} +     {{ key|replace('_', ' ')|title() }}: {{ value }}uL
+ {% else %} +     {{ key|replace('_', ' ')|title() }}: {{ value }}
+ {% endif %} {% endfor %}

{% endfor %} {% endif %} @@ -71,26 +51,18 @@

qPCR Status:

{% endif %}

{% for key, value in entry.items() if key != 'imported_by'%} - - - - {% if "column" in key %} -     {{ key|replace('_', ' ')|title() }}: {{ value }}uL
- {% else %} -     {{ key|replace('_', ' ')|title() }}: {{ value }}
- {% endif %} - + {% if "column" in key %} +     {{ key|replace('_', ' ')|title() }}: {{ value }}uL
+ {% else %} +     {{ key|replace('_', ' ')|title() }}: {{ value }}
+ {% endif %} {% endfor %}

{% endfor %} {% endif %} {% if sub['comments'] %}

Comments:

{% for entry in sub['comments'] %} - - - -      {{ entry['name'] }}:
{{ entry['text'] }}
- {{ entry['time'] }}
- +      {{ entry['name'] }}:
{{ entry['text'] }}
- {{ entry['time'] }}
{% endfor %}

{% endif %} {% if sub['platemap'] %} diff --git a/src/submissions/tools/__init__.py b/src/submissions/tools/__init__.py index f733cb9..8654dcd 100644 --- a/src/submissions/tools/__init__.py +++ b/src/submissions/tools/__init__.py @@ -53,6 +53,8 @@ def check_not_nan(cell_contents) -> bool: cell_contents = cell_contents.lower() except (TypeError, AttributeError): pass + if cell_contents == "nat": + cell_contents = np.nan if cell_contents == 'nan': cell_contents = np.nan if cell_contents == None: @@ -80,6 +82,7 @@ def convert_nans_to_nones(input_str) -> str|None: Returns: str: _description_ """ + # logger.debug(f"Input value of: {input_str}") if check_not_nan(input_str): return input_str return None