From 19448cc8f387e2b1967904ceb225f57ee9e9a955 Mon Sep 17 00:00:00 2001 From: Landon Wark Date: Thu, 4 Jan 2024 13:29:18 -0600 Subject: [PATCH] Improved form regeneration for artic. --- CHANGELOG.md | 4 + TODO.md | 3 +- alembic.ini | 4 +- .../30aab47d6f12_adding_role_tag_to_.py | 40 +++++ .../94289d4e63e6_updating_primary_key_for_.py | 36 +++++ .../bc7a74476609_adding_equipment_roles.py | 59 +++++++ src/submissions/__init__.py | 2 +- src/submissions/backend/db/models/kits.py | 144 +++++++++++++----- .../backend/db/models/submissions.py | 126 +++++++++++---- src/submissions/backend/excel/parser.py | 49 +++++- src/submissions/backend/validators/pydant.py | 83 +++++++--- .../frontend/widgets/equipment_usage.py | 54 +++++-- .../frontend/widgets/submission_table.py | 11 +- .../widgets/submission_type_creator.py | 2 +- .../frontend/widgets/submission_widget.py | 13 +- .../templates/submission_details.html | 4 +- src/submissions/templates/tooltip.html | 4 + src/submissions/tools.py | 4 +- 18 files changed, 519 insertions(+), 123 deletions(-) create mode 100644 alembic/versions/30aab47d6f12_adding_role_tag_to_.py create mode 100644 alembic/versions/94289d4e63e6_updating_primary_key_for_.py create mode 100644 alembic/versions/bc7a74476609_adding_equipment_roles.py create mode 100644 src/submissions/templates/tooltip.html diff --git a/CHANGELOG.md b/CHANGELOG.md index 932b130..dfcb8e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 202401.01 + +- Improved tooltips and form regeneration. + ## 202312.03 - Enabled creation of new submission types in gui. diff --git a/TODO.md b/TODO.md index 64b5414..0b7ff1e 100644 --- a/TODO.md +++ b/TODO.md @@ -1,5 +1,6 @@ +- [x] Finish Equipment Parser (add in regex to id asset_number) - [ ] Complete info_map in the SubmissionTypeCreator widget. -- [ ] Update Artic and add in equipment listings... *sigh*. +- [x] Update Artic and add in equipment listings... *sigh*. - [x] Fix WastewaterAssociations not in Session error. - Done... I think? - [x] Fix submitted date always being today. diff --git a/alembic.ini b/alembic.ini index f157eb0..309811a 100644 --- a/alembic.ini +++ b/alembic.ini @@ -55,8 +55,8 @@ version_path_separator = os # Use os.pathsep. Default configuration used for ne # are written from script.py.mako # output_encoding = utf-8 -sqlalchemy.url = sqlite:///L:\Robotics Laboratory Support\Submissions\submissions.db -; sqlalchemy.url = sqlite:///C:\Users\lwark\Documents\Archives\Submissions_app_backups\DB_backups\submissions-new.db +; sqlalchemy.url = sqlite:///L:\Robotics Laboratory Support\Submissions\submissions.db +sqlalchemy.url = sqlite:///C:\Users\lwark\Documents\Archives\Submissions_app_backups\DB_backups\submissions-new.db ; sqlalchemy.url = sqlite:///C:\Users\lwark\Documents\python\submissions\tests\test_assets\submissions-test.db diff --git a/alembic/versions/30aab47d6f12_adding_role_tag_to_.py b/alembic/versions/30aab47d6f12_adding_role_tag_to_.py new file mode 100644 index 0000000..02f0ae3 --- /dev/null +++ b/alembic/versions/30aab47d6f12_adding_role_tag_to_.py @@ -0,0 +1,40 @@ +"""Adding role tag to SubmissionEquipmentAssociation + +Revision ID: 30aab47d6f12 +Revises: bc7a74476609 +Create Date: 2023-12-27 09:00:26.262904 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '30aab47d6f12' +down_revision = 'bc7a74476609' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + # op.drop_table('_alembic_tmp__equipment') + with op.batch_alter_table('_equipment_submissions', schema=None) as batch_op: + batch_op.add_column(sa.Column('role', sa.String(length=64), nullable=True)) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('_equipment_submissions', schema=None) as batch_op: + batch_op.drop_column('role') + + # op.create_table('_alembic_tmp__equipment', + # sa.Column('id', sa.INTEGER(), nullable=False), + # sa.Column('name', sa.VARCHAR(length=64), nullable=True), + # sa.Column('nickname', sa.VARCHAR(length=64), nullable=True), + # sa.Column('asset_number', sa.VARCHAR(length=16), nullable=True), + # sa.PrimaryKeyConstraint('id') + # ) + # ### end Alembic commands ### diff --git a/alembic/versions/94289d4e63e6_updating_primary_key_for_.py b/alembic/versions/94289d4e63e6_updating_primary_key_for_.py new file mode 100644 index 0000000..7343ec2 --- /dev/null +++ b/alembic/versions/94289d4e63e6_updating_primary_key_for_.py @@ -0,0 +1,36 @@ +"""Updating primary key for SubmissionEquipmentAssociation + +Revision ID: 94289d4e63e6 +Revises: 30aab47d6f12 +Create Date: 2024-01-03 15:14:21.156127 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '94289d4e63e6' +down_revision = '30aab47d6f12' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('_equipment_submissions', schema=None) as batch_op: + batch_op.alter_column('role', + existing_type=sa.VARCHAR(length=64), + nullable=False) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('_equipment_submissions', schema=None) as batch_op: + batch_op.alter_column('role', + existing_type=sa.VARCHAR(length=64), + nullable=True) + + # ### end Alembic commands ### diff --git a/alembic/versions/bc7a74476609_adding_equipment_roles.py b/alembic/versions/bc7a74476609_adding_equipment_roles.py new file mode 100644 index 0000000..2a38e66 --- /dev/null +++ b/alembic/versions/bc7a74476609_adding_equipment_roles.py @@ -0,0 +1,59 @@ +"""Adding equipment roles + +Revision ID: bc7a74476609 +Revises: 761baf9d7842 +Create Date: 2023-12-21 10:44:23.520392 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import sqlite + +# revision identifiers, used by Alembic. +revision = 'bc7a74476609' +down_revision = '761baf9d7842' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('_equipment_roles', + sa.Column('id', sa.INTEGER(), nullable=False), + sa.Column('name', sa.String(length=32), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('_equipmentroles_equipment', + sa.Column('equipment_id', sa.INTEGER(), nullable=True), + sa.Column('equipmentroles_id', sa.INTEGER(), nullable=True), + sa.ForeignKeyConstraint(['equipment_id'], ['_equipment.id'], ), + sa.ForeignKeyConstraint(['equipmentroles_id'], ['_equipment_roles.id'], ) + ) + op.create_table('_submissiontype_equipmentrole', + sa.Column('equipmentrole_id', sa.INTEGER(), nullable=False), + sa.Column('submissiontype_id', sa.INTEGER(), nullable=False), + sa.Column('uses', sa.JSON(), nullable=True), + sa.Column('static', sa.INTEGER(), nullable=True), + sa.ForeignKeyConstraint(['equipmentrole_id'], ['_equipment_roles.id'], ), + sa.ForeignKeyConstraint(['submissiontype_id'], ['_submission_types.id'], ), + sa.PrimaryKeyConstraint('equipmentrole_id', 'submissiontype_id') + ) + op.drop_table('_submissiontype_equipment') + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('_submissiontype_equipment', + sa.Column('equipment_id', sa.INTEGER(), nullable=False), + sa.Column('submissiontype_id', sa.INTEGER(), nullable=False), + sa.Column('uses', sqlite.JSON(), nullable=True), + sa.Column('static', sa.INTEGER(), nullable=True), + sa.ForeignKeyConstraint(['equipment_id'], ['_equipment.id'], ), + sa.ForeignKeyConstraint(['submissiontype_id'], ['_submission_types.id'], ), + sa.PrimaryKeyConstraint('equipment_id', 'submissiontype_id') + ) + op.drop_table('_submissiontype_equipmentrole') + op.drop_table('_equipmentroles_equipment') + op.drop_table('_equipment_roles') + # ### end Alembic commands ### diff --git a/src/submissions/__init__.py b/src/submissions/__init__.py index aaf6e4f..c9bfda6 100644 --- a/src/submissions/__init__.py +++ b/src/submissions/__init__.py @@ -4,7 +4,7 @@ from pathlib import Path # Version of the realpython-reader package __project__ = "submissions" -__version__ = "202312.3b" +__version__ = "202312.4b" __author__ = {"name":"Landon Wark", "email":"Landon.Wark@phac-aspc.gc.ca"} __copyright__ = "2022-2023, Government of Canada" diff --git a/src/submissions/backend/db/models/kits.py b/src/submissions/backend/db/models/kits.py index b238892..351c19f 100644 --- a/src/submissions/backend/db/models/kits.py +++ b/src/submissions/backend/db/models/kits.py @@ -6,7 +6,7 @@ from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey, Int from sqlalchemy.orm import relationship, validates, Query from sqlalchemy.ext.associationproxy import association_proxy from datetime import date -import logging +import logging, re from tools import check_authorization, setup_lookup, query_return, Report, Result, Settings from typing import List from pandas import ExcelFile @@ -24,6 +24,14 @@ reagenttypes_reagents = Table( extend_existing = True ) +equipmentroles_equipment = Table( + "_equipmentroles_equipment", + Base.metadata, + Column("equipment_id", INTEGER, ForeignKey("_equipment.id")), + Column("equipmentroles_id", INTEGER, ForeignKey("_equipment_roles.id")), + extend_existing=True +) + class KitType(BaseClass): """ Base of kits used in submission processing @@ -589,13 +597,13 @@ class SubmissionType(BaseClass): kit_types = association_proxy("submissiontype_kit_associations", "kit_type") #: Proxy of kittype association - submissiontype_equipment_associations = relationship( - "SubmissionTypeEquipmentAssociation", + submissiontype_equipmentrole_associations = relationship( + "SubmissionTypeEquipmentRoleAssociation", back_populates="submission_type", cascade="all, delete-orphan" ) - equipment = association_proxy("submissiontype_equipment_associations", "equipment") + equipment = association_proxy("submissiontype_equipmentrole_associations", "equipment_role") def __repr__(self) -> str: return f"" @@ -609,34 +617,35 @@ class SubmissionType(BaseClass): """ return ExcelFile(self.template_file).sheet_names - def set_template_file(self, filepath:Path|str): + def set_template_file(self, ctx:Settings, filepath:Path|str): if isinstance(filepath, str): filepath = Path(filepath) with open (filepath, "rb") as f: data = f.read() self.template_file = data - self.save() + self.save(ctx=ctx) - def get_equipment(self) -> list: - from backend.validators.pydant import PydEquipmentPool - # if static: - # return [item.equipment.to_pydantic() for item in self.submissiontype_equipment_associations if item.static==1] - # else: - preliminary1 = [item.equipment.to_pydantic(static=item.static) for item in self.submissiontype_equipment_associations]# if item.static==0] - preliminary2 = [item.equipment.to_pydantic(static=item.static) for item in self.submissiontype_equipment_associations]# if item.static==0] + def construct_equipment_map(self): output = [] - pools = list(set([item.pool_name for item in preliminary1 if item.pool_name != None])) - for pool in pools: - c_ = [] - for item in preliminary1: - if item.pool_name == pool: - c_.append(item) - preliminary2.remove(item) - if len(c_) > 0: - output.append(PydEquipmentPool(name=pool, equipment=c_)) - for item in preliminary2: - output.append(item) + for item in self.submissiontype_equipmentrole_associations: + map = item.uses + map['role'] = item.equipment_role.name + output.append(map) return output + # return [item.uses for item in self.submissiontype_equipmentrole_associations] + + def get_equipment(self) -> List['PydEquipmentRole']: + return [item.to_pydantic(submission_type=self) for item in self.equipment] + + def get_processes_for_role(self, equipment_role:str|EquipmentRole): + match equipment_role: + case str(): + relevant = [item.get_all_processes() for item in self.submissiontype_equipmentrole_associations if item.equipment_role.name==equipment_role] + case EquipmentRole(): + relevant = [item.get_all_processes() for item in self.submissiontype_equipmentrole_associations if item.equipment_role==equipment_role] + case _: + raise TypeError(f"Type {type(equipment_role)} is not allowed") + return list(set([item for items in relevant for item in items if item != None ])) @classmethod @setup_lookup @@ -832,7 +841,7 @@ class Equipment(BaseClass): name = Column(String(64)) nickname = Column(String(64)) asset_number = Column(String(16)) - pool_name = Column(String(16)) + roles = relationship("EquipmentRole", back_populates="instances", secondary=equipmentroles_equipment) equipment_submission_associations = relationship( "SubmissionEquipmentAssociation", @@ -842,16 +851,11 @@ class Equipment(BaseClass): submissions = association_proxy("equipment_submission_associations", "submission") - equipment_submissiontype_associations = relationship( - "SubmissionTypeEquipmentAssociation", - back_populates="equipment", - cascade="all, delete-orphan", - ) - - submission_types = association_proxy("equipment_submission_associations", "submission_type") - def __repr__(self): return f"" + + def get_processes(self, submission_type:SubmissionType): + return [assoc.process for assoc in self.equipment_submission_associations if assoc.submission.submission_type_name==submission_type.name] @classmethod @setup_lookup @@ -882,14 +886,66 @@ class Equipment(BaseClass): pass return query_return(query=query, limit=limit) - def to_pydantic(self, static): + def to_pydantic(self, submission_type:SubmissionType): from backend.validators.pydant import PydEquipment - return PydEquipment(static=static, **self.__dict__) + return PydEquipment(processes=self.get_processes(submission_type=submission_type), role=None, **self.__dict__) def save(self): self.__database_session__.add(self) self.__database_session__.commit() + @classmethod + def get_regex(cls) -> re.Pattern: + return re.compile(r""" + (?P50\d{5}$)| + (?PHC-\d{6}$)| + (?P[^\d][A-Z0-9]{6}$)| + (?P[A-Z]{3}-\d{2}-[A-Z]-[A-Z]$)| + (?P\d{4}-\d{3}-\d{3}-\d$)""", + re.VERBOSE) + +class EquipmentRole(BaseClass): + + __tablename__ = "_equipment_roles" + + id = Column(INTEGER, primary_key=True) + name = Column(String(32)) + instances = relationship("Equipment", back_populates="roles", secondary=equipmentroles_equipment) + + equipmentrole_submissiontype_associations = relationship( + "SubmissionTypeEquipmentRoleAssociation", + back_populates="equipment_role", + cascade="all, delete-orphan", + ) + + submission_types = association_proxy("equipmentrole_submission_associations", "submission_type") + + def __repr__(self): + return f"" + + def to_pydantic(self, submission_type:SubmissionType): + from backend.validators.pydant import PydEquipmentRole + equipment = [item.to_pydantic(submission_type=submission_type) for item in self.instances] + return PydEquipmentRole(equipment=equipment, **self.__dict__) + + @classmethod + @setup_lookup + def query(cls, name:str|None=None, id:int|None=None, limit:int=0) -> EquipmentRole|List[EquipmentRole]: + query = cls.__database_session__.query(cls) + match id: + case int(): + query = query.filter(cls.id==id) + limit = 1 + case _: + pass + match name: + case str(): + query = query.filter(cls.name==name) + limit = 1 + case _: + pass + return query_return(query=query, limit=limit) + class SubmissionEquipmentAssociation(BaseClass): # Currently abstract until ready to implement @@ -899,6 +955,7 @@ class SubmissionEquipmentAssociation(BaseClass): equipment_id = Column(INTEGER, ForeignKey("_equipment.id"), primary_key=True) #: id of associated equipment submission_id = Column(INTEGER, ForeignKey("_submissions.id"), primary_key=True) #: id of associated submission + role = Column(String(64), primary_key=True) #: name of the role the equipment fills process = Column(String(64)) #: name of the process run on this equipment start_time = Column(TIMESTAMP) end_time = Column(TIMESTAMP) @@ -913,27 +970,27 @@ class SubmissionEquipmentAssociation(BaseClass): self.equipment = equipment def to_sub_dict(self) -> dict: - output = dict(name=self.equipment.name, asset_number=self.equipment.asset_number, comment=self.comments) + output = dict(name=self.equipment.name, asset_number=self.equipment.asset_number, comment=self.comments, process=[self.process], role=self.role, nickname=self.equipment.nickname) return output def save(self): self.__database_session__.add(self) self.__database_session__.commit() -class SubmissionTypeEquipmentAssociation(BaseClass): +class SubmissionTypeEquipmentRoleAssociation(BaseClass): # __abstract__ = True - __tablename__ = "_submissiontype_equipment" + __tablename__ = "_submissiontype_equipmentrole" - equipment_id = Column(INTEGER, ForeignKey("_equipment.id"), primary_key=True) #: id of associated equipment + equipmentrole_id = Column(INTEGER, ForeignKey("_equipment_roles.id"), primary_key=True) #: id of associated equipment submissiontype_id = Column(INTEGER, ForeignKey("_submission_types.id"), primary_key=True) #: id of associated submission uses = Column(JSON) #: locations of equipment on the submission type excel sheet. static = Column(INTEGER, default=1) #: if 1 this piece of equipment will always be used, otherwise it will need to be selected from list? - submission_type = relationship(SubmissionType, back_populates="submissiontype_equipment_associations") #: associated submission + submission_type = relationship(SubmissionType, back_populates="submissiontype_equipmentrole_associations") #: associated submission - equipment = relationship(Equipment, back_populates="equipment_submissiontype_associations") #: associated equipment + equipment_role = relationship(EquipmentRole, back_populates="equipmentrole_submissiontype_associations") #: associated equipment @validates('static') def validate_age(self, key, value): @@ -954,6 +1011,11 @@ class SubmissionTypeEquipmentAssociation(BaseClass): raise ValueError(f'Invalid required value {value}. Must be 0 or 1.') return value + def get_all_processes(self): + processes = [equipment.get_processes(self.submission_type) for equipment in self.equipment_role.instances] + processes = [item for items in processes for item in items if item != None ] + return processes + @check_authorization def save(self, ctx:Settings): self.__database_session__.add(self) diff --git a/src/submissions/backend/db/models/submissions.py b/src/submissions/backend/db/models/submissions.py index 58bfcfc..e390085 100644 --- a/src/submissions/backend/db/models/submissions.py +++ b/src/submissions/backend/db/models/submissions.py @@ -4,8 +4,9 @@ Models for the main submission types. from __future__ import annotations from getpass import getuser import math, json, logging, uuid, tempfile, re, yaml +from operator import attrgetter from pprint import pformat -from . import Reagent, SubmissionType, KitType, Organization, Equipment, SubmissionEquipmentAssociation +from . import Reagent, SubmissionType, KitType, Organization from sqlalchemy import Column, String, TIMESTAMP, INTEGER, ForeignKey, JSON, FLOAT, case from sqlalchemy.orm import relationship, validates, Query from json.decoder import JSONDecodeError @@ -13,7 +14,7 @@ from sqlalchemy.ext.associationproxy import association_proxy import pandas as pd from openpyxl import Workbook from . import BaseClass -from tools import check_not_nan, row_map, query_return, setup_lookup +from tools import check_not_nan, row_map, query_return, setup_lookup, jinja_template_loading from datetime import datetime, date from typing import List from dateutil.parser import parse @@ -137,21 +138,23 @@ class BasicSubmission(BaseClass): reagents = None # samples = [item.sample.to_sub_dict(submission_rsl=self.rsl_plate_num) for item in self.submission_sample_associations] samples = [item.to_sub_dict() for item in self.submission_sample_associations] + try: + equipment = [item.to_sub_dict() for item in self.submission_equipment_associations] + if len(equipment) == 0: + equipment = None + except Exception as e: + logger.error(f"Error setting equipment: {self.equipment}") + equipment = None else: reagents = None samples = None + equipment = None try: comments = self.comment except Exception as e: logger.error(f"Error setting comment: {self.comment}") comments = None - try: - equipment = [item.to_sub_dict() for item in self.submission_equipment_associations] - if len(equipment) == 0: - equipment = None - except Exception as e: - logger.error(f"Error setting equipment: {self.equipment}") - equipment = None + output = { "id": self.id, "Plate Number": self.rsl_plate_num, @@ -508,7 +511,7 @@ class BasicSubmission(BaseClass): field_value = len(self.samples) else: field_value = value - case "ctx" | "csv" | "filepath": + case "ctx" | "csv" | "filepath" | "equipment": return case "comment": if value == "" or value == None or value == 'null': @@ -552,8 +555,9 @@ class BasicSubmission(BaseClass): Returns: PydSubmission: converted object. """ - from backend.validators import PydSubmission, PydSample, PydReagent + from backend.validators import PydSubmission, PydSample, PydReagent, PydEquipment dicto = self.to_dict(full_data=True) + logger.debug(f"Backup dictionary: {pformat(dicto)}") # dicto['filepath'] = Path(tempfile.TemporaryFile().name) new_dict = {} for key, value in dicto.items(): @@ -562,6 +566,8 @@ class BasicSubmission(BaseClass): new_dict[key] = [PydReagent(**reagent) for reagent in value] case "samples": new_dict[key] = [PydSample(**sample) for sample in dicto['samples']] + case "equipment": + new_dict[key] = [PydEquipment(**equipment) for equipment in dicto['equipment']] case "Plate Number": new_dict['rsl_plate_num'] = dict(value=value, missing=True) case "Submitter Plate Number": @@ -576,19 +582,20 @@ class BasicSubmission(BaseClass): # sys.exit() return PydSubmission(**new_dict) - def backup(self, fname:Path): + def backup(self, fname:Path, full_backup:bool=True): """ Exports xlsx and yml info files for this instance. Args: fname (Path): Filename of xlsx file. """ - backup = self.to_dict(full_data=True) - try: - with open(self.__backup_path__.joinpath(fname.with_suffix(".yml")), "w") as f: - yaml.dump(backup, f) - except KeyError as e: - logger.error(f"Problem saving yml backup file: {e}") + if full_backup: + backup = self.to_dict(full_data=True) + try: + with open(self.__backup_path__.joinpath(fname.with_suffix(".yml")), "w") as f: + yaml.dump(backup, f) + except KeyError as e: + logger.error(f"Problem saving yml backup file: {e}") pyd = self.to_pydantic() wb = pyd.autofill_excel() wb = pyd.autofill_samples(wb) @@ -766,6 +773,8 @@ class BasicSubmission(BaseClass): msg = "This submission already exists.\nWould you like to overwrite?" return instance, code, msg + def get_used_equipment(self) -> List[str]: + return [item.role for item in self.submission_equipment_associations] # Below are the custom submission types @@ -882,7 +891,8 @@ class BacterialCulture(BasicSubmission): Returns: str: string for regex construction """ - return "(?PRSL-?\\d{2}-?\\d{4})" + # return "(?PRSL-?\\d{2}-?\\d{4})" + return "(?PRSL(?:-|_)?BC(?:-|_)?20\d{2}-?\d{2}-?\d{2}(?:(_|-)?\d?([^_0123456789\s]|$)?R?\d?)?)" @classmethod def filename_template(cls): @@ -1175,7 +1185,36 @@ class WastewaterArtic(BasicSubmission): logger.error(f"Couldn't construct df due to {e}") input_dict['csv'] = df return input_dict - + + @classmethod + def custom_autofill(cls, input_excel: Workbook, info: dict | None = None, backup: bool = False) -> Workbook: + input_excel = super().custom_autofill(input_excel, info, backup) + worksheet = input_excel["First Strand List"] + samples = cls.query(rsl_number=info['rsl_plate_num']['value']).submission_sample_associations + samples = sorted(samples, key=attrgetter('column', 'row')) + source_plates = [] + first_samples = [] + for sample in samples: + sample = sample.sample + try: + assoc = [item.submission.rsl_plate_num for item in sample.sample_submission_associations if item.submission.submission_type_name=="Wastewater"][-1] + except IndexError: + logger.error(f"Association not found for {sample}") + continue + if assoc not in source_plates: + source_plates.append(assoc) + first_samples.append(sample.ww_processing_num) + # Pad list to length of 3 + # source_plates = list(set(source_plates)) + source_plates += ['None'] * (3 - len(source_plates)) + first_samples += [''] * (3 - len(first_samples)) + source_plates = zip(source_plates, first_samples, strict=False) + for iii, plate in enumerate(source_plates, start=8): + logger.debug(f"Plate: {plate}") + for jjj, value in enumerate(plate, start=3): + worksheet.cell(row=iii, column=jjj, value=value) + return input_excel + # Sample Classes class BasicSample(BaseClass): @@ -1286,11 +1325,15 @@ class BasicSample(BaseClass): dict: dictionary of sample id, row and column in elution plate """ # Since there is no PCR, negliable result is necessary. - assoc = [item for item in self.sample_submission_associations if item.submission.rsl_plate_num==submission_rsl][0] - tooltip_text = f""" - Sample name: {self.submitter_id}
- Well: {row_map[assoc.row]}{assoc.column} - """ + # assoc = [item for item in self.sample_submission_associations if item.submission.rsl_plate_num==submission_rsl][0] + fields = self.to_sub_dict(submission_rsl=submission_rsl) + env = jinja_template_loading() + template = env.get_template("tooltip.html") + tooltip_text = template.render(fields=fields) + # tooltip_text = f""" + # Sample name: {self.submitter_id}
+ # Well: {row_map[assoc.row]}{assoc.column} + # """ return dict(name=self.submitter_id[:10], positive=False, tooltip=tooltip_text) @classmethod @@ -1436,6 +1479,7 @@ class BasicSample(BaseClass): used_class = cls.find_subclasses(attrs=kwargs, sample_type=sample_type) instance = used_class(**kwargs) instance.sample_type = sample_type + logger.debug(f"Creating instance: {instance}") return instance def save(self): @@ -1523,6 +1567,25 @@ class WastewaterSample(BasicSample): del output_dict['collection_date'] return output_dict + def to_sub_dict(self, submission_rsl: str | BasicSubmission) -> dict: + sample = super().to_sub_dict(submission_rsl) + if self.ww_processing_num != None: + sample['ww_processing_num'] = self.ww_processing_num + else: + sample['ww_processing_num'] = self.submitter_id + try: + assoc = [item for item in self.sample_submission_associations if item.submission.submission_type_name=="Wastewater"][-1] + except: + assoc = None + if assoc != None: + try: + sample['ct'] = f"{assoc.ct_n1:.2f}, {assoc.ct_n2:.2f}" + except TypeError: + sample['ct'] = "None, None" + sample['source_plate'] = assoc.submission.rsl_plate_num + sample['source_well'] = f"{row_map[assoc.row]}{assoc.column}" + return sample + class BacterialCultureSample(BasicSample): """ base of bacterial culture sample @@ -1541,7 +1604,9 @@ class BacterialCultureSample(BasicSample): dict: well location and name (sample id, organism) NOTE: keys must sync with WWSample to_sub_dict above """ sample = super().to_sub_dict(submission_rsl=submission_rsl) - sample['name'] = f"{self.submitter_id} - ({self.organism})" + sample['name'] = self.submitter_id + sample['organism'] = self.organism + sample['concentration'] = self.concentration return sample def to_hitpick(self, submission_rsl: str | None = None) -> dict | None: @@ -1622,13 +1687,15 @@ class SubmissionSampleAssociation(BaseClass): if isinstance(polymorphic_identity, dict): polymorphic_identity = polymorphic_identity['value'] if polymorphic_identity == None: - return cls + output = cls else: try: - return [item for item in cls.__subclasses__() if item.__mapper_args__['polymorphic_identity']==polymorphic_identity][0] + output = [item for item in cls.__subclasses__() if item.__mapper_args__['polymorphic_identity']==polymorphic_identity][0] except Exception as e: logger.error(f"Could not get polymorph {polymorphic_identity} of {cls} due to {e}") - return cls + output = cls + logger.debug(f"Using SubmissionSampleAssociation subclass: {output}") + return output @classmethod @setup_lookup @@ -1707,6 +1774,7 @@ class SubmissionSampleAssociation(BaseClass): Returns: SubmissionSampleAssociation: Queried or new association. """ + logger.debug(f"Attempting create or query with {kwargs}") match submission: case BasicSubmission(): pass diff --git a/src/submissions/backend/excel/parser.py b/src/submissions/backend/excel/parser.py index 6433680..3232f07 100644 --- a/src/submissions/backend/excel/parser.py +++ b/src/submissions/backend/excel/parser.py @@ -8,7 +8,7 @@ import pandas as pd import numpy as np from pathlib import Path from backend.db.models import * -from backend.validators import PydSubmission, PydReagent, RSLNamer, PydSample +from backend.validators import PydSubmission, PydReagent, RSLNamer, PydSample, PydEquipment import logging, re from collections import OrderedDict from datetime import date @@ -53,6 +53,7 @@ class SheetParser(object): self.parse_reagents() self.import_reagent_validation_check() self.parse_samples() + self.parse_equipment() self.finalize_parse() logger.debug(f"Parser.sub after info scrape: {pformat(self.sub)}") @@ -90,6 +91,10 @@ class SheetParser(object): self.sample_result, self.sub['samples'] = parser.parse_samples() self.plate_map = parser.plate_map + def parse_equipment(self): + parser = EquipmentParser(xl=self.xl, submission_type=self.sub['submission_type']['value']) + self.sub['equipment'] = parser.parse_equipment() + def import_kit_validation_check(self): """ Enforce that the parser has an extraction kit @@ -129,6 +134,9 @@ class SheetParser(object): PydSubmission: output pydantic model """ # logger.debug(f"Submission dictionary coming into 'to_pydantic':\n{pformat(self.sub)}") + logger.debug(f"Equipment: {self.sub['equipment']}") + if len(self.sub['equipment']) == 0: + self.sub['equipment'] = None psm = PydSubmission(filepath=self.filepath, **self.sub) return psm @@ -480,6 +488,45 @@ class SampleParser(object): plates.append(output) return plates +class EquipmentParser(object): + + def __init__(self, xl:pd.ExcelFile, submission_type:str) -> None: + self.submission_type = submission_type + self.xl = xl + self.map = self.fetch_equipment_map() + # self.equipment = self.parse_equipment() + + def fetch_equipment_map(self) -> List[dict]: + submission_type = SubmissionType.query(name=self.submission_type) + return submission_type.construct_equipment_map() + + def get_asset_number(self, input:str) -> str: + regex = Equipment.get_regex() + return regex.search(input).group().strip("-") + + def parse_equipment(self): + logger.debug(f"Equipment parser going into parsing: {pformat(self.__dict__)}") + output = [] + # sheets = list(set([item['sheet'] for item in self.map])) + # logger.debug(f"Sheets: {sheets}") + for sheet in self.xl.sheet_names: + df = self.xl.parse(sheet, header=None, dtype=object) + relevant = [item for item in self.map if item['sheet']==sheet] + # logger.debug(f"Relevant equipment: {pformat(relevant)}") + previous_asset = "" + for equipment in relevant: + asset = df.iat[equipment['name']['row']-1, equipment['name']['column']-1] + if not check_not_nan(asset): + asset = previous_asset + else: + previous_asset = asset + asset = self.get_asset_number(input=asset) + eq = Equipment.query(asset_number=asset) + process = df.iat[equipment['process']['row']-1, equipment['process']['column']-1] + output.append(PydEquipment(name=eq.name, process=[process], role=equipment['role'], asset_number=asset, nickname=eq.nickname)) + # logger.debug(f"Here is the output so far: {pformat(output)}") + return output + class PCRParser(object): """ Object to pull data from Design and Analysis PCR export file. diff --git a/src/submissions/backend/validators/pydant.py b/src/submissions/backend/validators/pydant.py index 93095e9..796d74c 100644 --- a/src/submissions/backend/validators/pydant.py +++ b/src/submissions/backend/validators/pydant.py @@ -1,6 +1,7 @@ ''' Contains pydantic models and accompanying validators ''' +from __future__ import annotations from operator import attrgetter import uuid, re, logging from pydantic import BaseModel, field_validator, Field @@ -189,6 +190,7 @@ class PydSample(BaseModel, extra='allow'): continue case _: instance.set_attribute(name=key, value=value) + out_associations = [] if submission != None: assoc_type = self.sample_type.replace("Sample", "").strip() for row, column in zip(self.row, self.column): @@ -198,12 +200,14 @@ class PydSample(BaseModel, extra='allow'): submission=submission, sample=instance, row=row, column=column) + logger.debug(f"Using submission_sample_association: {association}") try: instance.sample_submission_associations.append(association) + out_associations.append(association) except IntegrityError as e: logger.error(f"Could not attach submission sample association due to: {e}") instance.metadata.session.rollback() - return instance, report + return instance, out_associations, report class PydSubmission(BaseModel, extra='allow'): filepath: Path @@ -220,7 +224,16 @@ class PydSubmission(BaseModel, extra='allow'): submission_category: dict|None = Field(default=dict(value=None, missing=True), validate_default=True) comment: dict|None = Field(default=dict(value="", missing=True), validate_default=True) reagents: List[dict]|List[PydReagent] = [] - samples: List[Any] + samples: List[PydSample] + equipment: List[PydEquipment]|None + + @field_validator('equipment', mode='before') + @classmethod + def convert_equipment_dict(cls, value): + logger.debug(f"Equipment: {value}") + if isinstance(value, dict): + return value['value'] + return value @field_validator('comment', mode='before') @classmethod @@ -425,7 +438,17 @@ class PydSubmission(BaseModel, extra='allow'): match key: case "samples": for sample in self.samples: - sample, _ = sample.toSQL(submission=instance) + sample, associations, _ = sample.toSQL(submission=instance) + logger.debug(f"Sample SQL object to be added to submission: {sample.__dict__}") + for assoc in associations: + instance.submission_sample_associations.append(assoc) + case "equipment": + logger.debug(f"Equipment: {pformat(self.equipment)}") + for equip in self.equipment: + equip, association = equip.toSQL(submission=instance) + if association != None: + logger.debug(f"Equipment association SQL object to be added to submission: {association.__dict__}") + instance.submission_equipment_associations.append(association) case _: try: instance.set_attribute(key=key, value=value) @@ -559,6 +582,7 @@ class PydSubmission(BaseModel, extra='allow'): except Exception as e: logger.error(f"Could not write name {reagent['name']['value']} due to {e}") # Get relevant info for that sheet + new_info = [item for item in new_info if isinstance(item['location'], dict)] sheet_info = [item for item in new_info if sheet in item['location']['sheets']] for item in sheet_info: logger.debug(f"Attempting: {item['type']} in row {item['location']['row']}, column {item['location']['column']}") @@ -579,9 +603,11 @@ class PydSubmission(BaseModel, extra='allow'): Workbook: Updated excel workbook """ sample_info = SubmissionType.query(name=self.submission_type['value']).info_map['samples'] + logger.debug(f"Sample info: {pformat(sample_info)}") + logger.debug(f"Workbook sheets: {workbook.sheetnames}") worksheet = workbook[sample_info["lookup_table"]['sheet']] samples = sorted(self.samples, key=attrgetter('column', 'row')) - logger.debug(f"Samples: {samples}") + logger.debug(f"Samples: {pformat(samples)}") # Fail safe against multiple instances of the same sample for iii, sample in enumerate(samples, start=1): row = sample_info['lookup_table']['start_row'] + iii @@ -744,33 +770,46 @@ class PydKit(BaseModel): class PydEquipment(BaseModel, extra='ignore'): + asset_number: str name: str nickname: str|None - asset_number: str - pool_name: str|None - static: bool|int + process: List[str]|None + role: str|None - @field_validator("static") + @field_validator('process') @classmethod - def to_boolean(cls, value): - match value: - case int(): - if value == 0: - return False - else: - return True - case _: - return value + def remove_dupes(cls, value): + if isinstance(value, list): + return list(set(value)) + else: + return value def toForm(self, parent): from frontend.widgets.equipment_usage import EquipmentCheckBox return EquipmentCheckBox(parent=parent, equipment=self) + + def toSQL(self, submission:BasicSubmission|str=None): + if isinstance(submission, str): + submission = BasicSubmission.query(rsl_number=submission) + equipment = Equipment.query(asset_number=self.asset_number) + if equipment == None: + return + if submission != None: + assoc = SubmissionEquipmentAssociation(submission=submission, equipment=equipment) + assoc.process = self.process[0] + assoc.role = self.role + # equipment.equipment_submission_associations.append(assoc) + equipment.equipment_submission_associations.append(assoc) + else: + assoc = None + return equipment, assoc -class PydEquipmentPool(BaseModel): +class PydEquipmentRole(BaseModel): name: str equipment: List[PydEquipment] - - def toForm(self, parent): - from frontend.widgets.equipment_usage import PoolComboBox - return PoolComboBox(parent=parent, pool=self) \ No newline at end of file + + def toForm(self, parent, submission_type, used): + from frontend.widgets.equipment_usage import RoleComboBox + return RoleComboBox(parent=parent, role=self, submission_type=submission_type, used=used) + diff --git a/src/submissions/frontend/widgets/equipment_usage.py b/src/submissions/frontend/widgets/equipment_usage.py index fee48fe..1e940ae 100644 --- a/src/submissions/frontend/widgets/equipment_usage.py +++ b/src/submissions/frontend/widgets/equipment_usage.py @@ -2,18 +2,26 @@ from PyQt6.QtCore import Qt from PyQt6.QtWidgets import (QDialog, QComboBox, QCheckBox, QLabel, QWidget, QHBoxLayout, QVBoxLayout, QDialogButtonBox) -from backend.db.models import SubmissionType -from backend.validators.pydant import PydEquipment, PydEquipmentPool +from backend.db.models import SubmissionType, Equipment, BasicSubmission +from backend.validators.pydant import PydEquipment, PydEquipmentRole +import logging + +logger = logging.getLogger(f"submissions.{__name__}") class EquipmentUsage(QDialog): - def __init__(self, parent, submission_type:SubmissionType|str) -> QDialog: + def __init__(self, parent, submission_type:SubmissionType|str, submission:BasicSubmission) -> QDialog: super().__init__(parent) self.setWindowTitle("Equipment Checklist") + self.used_equipment = submission.get_used_equipment() + logger.debug(f"Existing equipment: {self.used_equipment}") if isinstance(submission_type, str): - submission_type = SubmissionType.query(name=submission_type) + self.submission_type = SubmissionType.query(name=submission_type) + else: + self.submission_type = submission_type # self.static_equipment = submission_type.get_equipment() - self.opt_equipment = submission_type.get_equipment() + self.opt_equipment = self.submission_type.get_equipment() + logger.debug(f"EquipmentRoles: {self.opt_equipment}") self.layout = QVBoxLayout() self.setLayout(self.layout) self.populate_form() @@ -24,14 +32,14 @@ class EquipmentUsage(QDialog): self.buttonBox.accepted.connect(self.accept) self.buttonBox.rejected.connect(self.reject) for eq in self.opt_equipment: - self.layout.addWidget(eq.toForm(parent=self)) + self.layout.addWidget(eq.toForm(parent=self, submission_type=self.submission_type, used=self.used_equipment)) self.layout.addWidget(self.buttonBox) def parse_form(self): output = [] for widget in self.findChildren(QWidget): match widget: - case (EquipmentCheckBox()|PoolComboBox()) : + case (EquipmentCheckBox()|RoleComboBox()) : output.append(widget.parse_form()) case _: pass @@ -65,25 +73,41 @@ class EquipmentCheckBox(QWidget): else: return None -class PoolComboBox(QWidget): +class RoleComboBox(QWidget): - def __init__(self, parent, pool:PydEquipmentPool) -> None: + def __init__(self, parent, role:PydEquipmentRole, submission_type:SubmissionType, used:list) -> None: super().__init__(parent) self.layout = QHBoxLayout() # label = QLabel() # label.setText(pool.name) + self.role = role + self.check = QCheckBox() + if role.name in used: + self.check.setChecked(False) + else: + self.check.setChecked(True) self.box = QComboBox() self.box.setMaximumWidth(125) self.box.setMinimumWidth(125) - self.box.addItems([item.name for item in pool.equipment]) - self.check = QCheckBox() + self.box.addItems([item.name for item in role.equipment]) + # self.check = QCheckBox() # self.layout.addWidget(label) - self.layout.addWidget(self.box) + self.process = QComboBox() + self.process.setMaximumWidth(125) + self.process.setMinimumWidth(125) + self.process.setEditable(True) + self.process.addItems(submission_type.get_processes_for_role(equipment_role=role.name)) self.layout.addWidget(self.check) + self.layout.addWidget(QLabel(f"{role.name}:")) + self.layout.addWidget(self.box) + self.layout.addWidget(self.process) + # self.layout.addWidget(self.check) self.setLayout(self.layout) - def parse_form(self) -> str: - if self.check.isChecked(): - return self.box.currentText() + def parse_form(self) -> str|None: + eq = Equipment.query(name=self.box.currentText()) + if self.check: + return PydEquipment(name=eq.name, processes=[self.process.currentText()], role=self.role.name, asset_number=eq.asset_number, nickname=eq.nickname) else: return None + \ No newline at end of file diff --git a/src/submissions/frontend/widgets/submission_table.py b/src/submissions/frontend/widgets/submission_table.py index 35e46b3..5298083 100644 --- a/src/submissions/frontend/widgets/submission_table.py +++ b/src/submissions/frontend/widgets/submission_table.py @@ -187,16 +187,19 @@ class SubmissionsSheet(QTableView): def add_equipment_function(self, rsl_plate_id): submission = BasicSubmission.query(id=rsl_plate_id) submission_type = submission.submission_type_name - dlg = EquipmentUsage(parent=self, submission_type=submission_type) + dlg = EquipmentUsage(parent=self, submission_type=submission_type, submission=submission) if dlg.exec(): equipment = dlg.parse_form() + logger.debug(f"We've got equipment: {equipment}") for equip in equipment: - e = Equipment.query(name=equip) + e = Equipment.query(name=equip.name) assoc = SubmissionEquipmentAssociation(submission=submission, equipment=e) + assoc.process = equip.processes[0] + assoc.role = equip.role # submission.submission_equipment_associations.append(assoc) logger.debug(f"Appending SubmissionEquipmentAssociation: {assoc}") # submission.save() - assoc.save() + # assoc.save() def delete_item(self, event): """ @@ -429,7 +432,7 @@ class SubmissionsSheet(QTableView): # delete_submission(id=value) sub = BasicSubmission.query(id=value) fname = select_save_file(self, default_name=sub.to_pydantic().construct_filename(), extension="xlsx") - sub.backup(fname=fname) + sub.backup(fname=fname, full_backup=False) class SubmissionDetails(QDialog): """ diff --git a/src/submissions/frontend/widgets/submission_type_creator.py b/src/submissions/frontend/widgets/submission_type_creator.py index 496494b..705aa2e 100644 --- a/src/submissions/frontend/widgets/submission_type_creator.py +++ b/src/submissions/frontend/widgets/submission_type_creator.py @@ -7,7 +7,7 @@ from PyQt6.QtWidgets import ( ) from sqlalchemy import FLOAT, INTEGER from sqlalchemy.orm.attributes import InstrumentedAttribute -from backend.db import SubmissionType, Equipment, SubmissionTypeEquipmentAssociation, BasicSubmission +from backend.db import SubmissionType, Equipment, SubmissionTypeEquipmentRoleAssociation, BasicSubmission from backend.validators import PydReagentType, PydKit import logging from pprint import pformat diff --git a/src/submissions/frontend/widgets/submission_widget.py b/src/submissions/frontend/widgets/submission_widget.py index a9ee97c..239aa36 100644 --- a/src/submissions/frontend/widgets/submission_widget.py +++ b/src/submissions/frontend/widgets/submission_widget.py @@ -315,6 +315,8 @@ class SubmissionFormContainer(QWidget): logger.debug(f"Here is the final submission: {pformat(base_submission.__dict__)}") logger.debug(f"Parsed reagents: {pformat(base_submission.reagents)}") logger.debug(f"Sending submission: {base_submission.rsl_plate_num} to database.") + logger.debug(f"Samples from pyd: {pformat(self.pyd.samples)}") + logger.debug(f"Samples SQL: {pformat([item.__dict__ for item in base_submission.samples])}") base_submission.save() # update summary sheet self.app.table_widget.sub_wid.setData() @@ -428,8 +430,8 @@ class SubmissionFormWidget(QWidget): # self.ignore = [None, "", "qt_spinbox_lineedit", "qt_scrollarea_viewport", "qt_scrollarea_hcontainer", # "qt_scrollarea_vcontainer", "submit_btn" # ] - self.ignore = ['filepath', 'samples', 'reagents', 'csv', 'ctx', 'comment'] - self.recover = ['filepath', 'samples', 'csv', 'comment'] + self.ignore = ['filepath', 'samples', 'reagents', 'csv', 'ctx', 'comment', 'equipment'] + self.recover = ['filepath', 'samples', 'csv', 'comment', 'equipment'] layout = QVBoxLayout() for k, v in kwargs.items(): if k not in self.ignore: @@ -475,8 +477,11 @@ class SubmissionFormWidget(QWidget): logger.debug(f"Reagents: {pformat(reagents)}") # logger.debug(f"Attrs not in info: {[k for k, v in self.__dict__.items() if k not in info.keys()]}") for item in self.recover: + logger.debug(f"Attempting to recover: {item}") if hasattr(self, item): - info[item] = getattr(self, item) + value = getattr(self, item) + logger.debug(f"Setting {item}") + info[item] = value # app = self.parent().parent().parent().parent().parent().parent().parent().parent # submission = PydSubmission(filepath=self.filepath, reagents=reagents, samples=self.samples, **info) submission = PydSubmission(reagents=reagents, **info) @@ -728,6 +733,8 @@ class ReagentFormWidget(QWidget): looked_up_reg = Reagent.query(lot_number=looked_up_rt.last_used) except AttributeError: looked_up_reg = None + if isinstance(looked_up_reg, list): + looked_up_reg = None logger.debug(f"Because there was no reagent listed for {reagent.lot}, we will insert the last lot used: {looked_up_reg}") if looked_up_reg != None: relevant_reagents.remove(str(looked_up_reg.lot)) diff --git a/src/submissions/templates/submission_details.html b/src/submissions/templates/submission_details.html index d1aee72..097ae53 100644 --- a/src/submissions/templates/submission_details.html +++ b/src/submissions/templates/submission_details.html @@ -48,13 +48,13 @@ {% if sub['equipment'] %}

Equipment:

{% for item in sub['equipment'] %} -     {{ item['name'] }}: {{ item['asset_number']|replace('\n\t', '
        ') }}
+     {{ item['role'] }}: {{ item['name'] }}({{ item['asset_number'] }}): {{ item['process']|replace('\n\t', '
        ') }}
{% endfor %}

{% endif %} {% if sub['samples'] %}

Samples:

{% for item in sub['samples'] %} -     {{ item['well'] }}: {{ item['name']|replace('\n\t', '
        ') }}
+     {{ item['well'] }}: {% if item['organism'] %} {{ item['name'] }} - ({{ item['organism']|replace('\n\t', '
        ') }}){% else %} {{ item['name']|replace('\n\t', '
        ') }}{% endif %}
{% endfor %}

{% endif %} {% if sub['controls'] %} diff --git a/src/submissions/templates/tooltip.html b/src/submissions/templates/tooltip.html new file mode 100644 index 0000000..0b473f2 --- /dev/null +++ b/src/submissions/templates/tooltip.html @@ -0,0 +1,4 @@ +Sample name: {{ fields['submitter_id'] }}
+{% if fields['organism'] %}Organism: {{ fields['organism'] }}
{% endif %} +{% if fields['concentration'] %}Concentration: {{ fields['concentration'] }}
{% endif %} +Well: {{ fields['row'] }}{{ fields['column'] }} \ No newline at end of file diff --git a/src/submissions/tools.py b/src/submissions/tools.py index 72cb71f..818369b 100644 --- a/src/submissions/tools.py +++ b/src/submissions/tools.py @@ -45,8 +45,10 @@ def check_not_nan(cell_contents) -> bool: bool: True if cell has value, else, false. """ # check for nan as a string first + exclude = ['unnamed:', 'blank', 'void'] try: - if "Unnamed:" in cell_contents or "blank" in cell_contents.lower(): + # if "Unnamed:" in cell_contents or "blank" in cell_contents.lower(): + if cell_contents.lower() in exclude: cell_contents = np.nan cell_contents = cell_contents.lower() except (TypeError, AttributeError):