From 06447f09380ea59dde0edddadbf41c0a33c293c4 Mon Sep 17 00:00:00 2001 From: Landon Wark Date: Tue, 2 May 2023 14:19:45 -0500 Subject: [PATCH] Upgrades to cost calculation methods --- CHANGELOG.md | 5 ++ TODO.md | 2 +- alembic.ini | 2 +- .../83b06f3f4869_updated_discount_table.py | 30 +++++++++++ .../cc9672a505f5_added_discount_table.py | 37 ++++++++++++++ ...8efd_split_mutable_costs_into_96_and_24.py | 51 +++++++++++++++++++ src/submissions/__init__.py | 2 +- src/submissions/backend/db/functions.py | 27 ++++++++-- src/submissions/backend/db/models/__init__.py | 2 +- src/submissions/backend/db/models/kits.py | 25 +++++---- .../backend/db/models/submissions.py | 29 ++++++++++- src/submissions/backend/excel/parser.py | 18 ++++--- 12 files changed, 203 insertions(+), 27 deletions(-) create mode 100644 alembic/versions/83b06f3f4869_updated_discount_table.py create mode 100644 alembic/versions/cc9672a505f5_added_discount_table.py create mode 100644 alembic/versions/dc780c868efd_split_mutable_costs_into_96_and_24.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a547838..851fc3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ +## 202305.01 + +- Improved kit cost calculation. + ## 202304.04 +- Added in discounts for kits based on kit used and submitting client. - Kraken controls graph now only pulls top 20 results to prevent crashing. - Improved cost calculations per column in a 96 well plate. diff --git a/TODO.md b/TODO.md index 51f2dff..c075321 100644 --- a/TODO.md +++ b/TODO.md @@ -1 +1 @@ -- [ ] Move bulk of functions from frontend.__init__ to frontend.functions as __init__ is getting bloated. \ No newline at end of file +- [x] Move bulk of functions from frontend.__init__ to frontend.functions as __init__ is getting bloated. \ No newline at end of file diff --git a/alembic.ini b/alembic.ini index f147f5e..20ebdc4 100644 --- a/alembic.ini +++ b/alembic.ini @@ -56,7 +56,7 @@ version_path_separator = os # Use os.pathsep. Default configuration used for ne # output_encoding = utf-8 ; sqlalchemy.url = sqlite:///L:\Robotics Laboratory Support\Submissions\submissions.db -sqlalchemy.url = sqlite:///C:\Users\lwark\Documents\Archives\DB_backups\submissions-20230302.db +sqlalchemy.url = sqlite:///C:\Users\lwark\Documents\Archives\DB_backups\submissions-20230427.db ; sqlalchemy.url = sqlite:///C:\Users\lwark\Documents\python\submissions\tests\test_assets\submissions_test.db diff --git a/alembic/versions/83b06f3f4869_updated_discount_table.py b/alembic/versions/83b06f3f4869_updated_discount_table.py new file mode 100644 index 0000000..f158e59 --- /dev/null +++ b/alembic/versions/83b06f3f4869_updated_discount_table.py @@ -0,0 +1,30 @@ +"""updated discount table + +Revision ID: 83b06f3f4869 +Revises: cc9672a505f5 +Create Date: 2023-04-27 13:04:35.886294 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import sqlite + +# revision identifiers, used by Alembic. +revision = '83b06f3f4869' +down_revision = 'cc9672a505f5' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('_discounts', schema=None) as batch_op: + batch_op.add_column(sa.Column('name', sa.String(length=128), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('_discounts', schema=None) as batch_op: + batch_op.drop_column('name') + # ### end Alembic commands ### diff --git a/alembic/versions/cc9672a505f5_added_discount_table.py b/alembic/versions/cc9672a505f5_added_discount_table.py new file mode 100644 index 0000000..fe187ae --- /dev/null +++ b/alembic/versions/cc9672a505f5_added_discount_table.py @@ -0,0 +1,37 @@ +"""added discount table + +Revision ID: cc9672a505f5 +Revises: 00de69ad6eab +Create Date: 2023-04-27 12:58:41.331563 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'cc9672a505f5' +down_revision = '00de69ad6eab' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('_discounts', + sa.Column('id', sa.INTEGER(), nullable=False), + sa.Column('kit_id', sa.INTEGER(), nullable=True), + sa.Column('client_id', sa.INTEGER(), nullable=True), + sa.Column('amount', sa.FLOAT(precision=2), nullable=True), + sa.ForeignKeyConstraint(['client_id'], ['_organizations.id'], name='fk_org_id', ondelete='SET NULL'), + sa.ForeignKeyConstraint(['kit_id'], ['_kits.id'], name='fk_kit_type_id', ondelete='SET NULL'), + sa.PrimaryKeyConstraint('id') + ) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('_discounts') + # ### end Alembic commands ### diff --git a/alembic/versions/dc780c868efd_split_mutable_costs_into_96_and_24.py b/alembic/versions/dc780c868efd_split_mutable_costs_into_96_and_24.py new file mode 100644 index 0000000..bf56000 --- /dev/null +++ b/alembic/versions/dc780c868efd_split_mutable_costs_into_96_and_24.py @@ -0,0 +1,51 @@ +"""split mutable costs into 96 and 24 + +Revision ID: dc780c868efd +Revises: 83b06f3f4869 +Create Date: 2023-05-01 14:05:47.762441 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import sqlite + +# revision identifiers, used by Alembic. +revision = 'dc780c868efd' +down_revision = '83b06f3f4869' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('_alembic_tmp__kits') + with op.batch_alter_table('_kits', schema=None) as batch_op: + batch_op.add_column(sa.Column('mutable_cost_96', sa.FLOAT(precision=2), nullable=True)) + batch_op.add_column(sa.Column('mutable_cost_24', sa.FLOAT(precision=2), nullable=True)) + batch_op.drop_column('mutable_cost') + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + + with op.batch_alter_table('_kits', schema=None) as batch_op: + batch_op.add_column(sa.Column('mutable_cost', sa.FLOAT(), nullable=True)) + batch_op.drop_column('mutable_cost_24') + batch_op.drop_column('mutable_cost_96') + + op.create_table('_alembic_tmp__kits', + sa.Column('id', sa.INTEGER(), nullable=False), + sa.Column('name', sa.VARCHAR(length=64), nullable=True), + sa.Column('used_for', sqlite.JSON(), nullable=True), + sa.Column('cost_per_run', sa.FLOAT(), nullable=True), + sa.Column('reagent_types_id', sa.INTEGER(), nullable=True), + sa.Column('constant_cost', sa.FLOAT(), nullable=True), + sa.Column('mutable_cost_96', sa.FLOAT(), nullable=True), + sa.Column('mutable_cost_24', sa.FLOAT(), nullable=True), + sa.ForeignKeyConstraint(['reagent_types_id'], ['_reagent_types.id'], ondelete='SET NULL'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name') + ) + # ### end Alembic commands ### diff --git a/src/submissions/__init__.py b/src/submissions/__init__.py index 4354579..5ff1ad3 100644 --- a/src/submissions/__init__.py +++ b/src/submissions/__init__.py @@ -4,7 +4,7 @@ from pathlib import Path # Version of the realpython-reader package __project__ = "submissions" -__version__ = "202304.4b" +__version__ = "202305.1b" __author__ = {"name":"Landon Wark", "email":"Landon.Wark@phac-aspc.gc.ca"} __copyright__ = "2022-2023, Government of Canada" diff --git a/src/submissions/backend/db/functions.py b/src/submissions/backend/db/functions.py index ac8b8c2..7c23ae5 100644 --- a/src/submissions/backend/db/functions.py +++ b/src/submissions/backend/db/functions.py @@ -164,13 +164,24 @@ def construct_submission_info(ctx:dict, info_dict:dict) -> models.BasicSubmissio try: # ceil(instance.sample_count / 8) will get number of columns # the cost of a full run multiplied by (that number / 12) is x twelfths the cost of a full run - logger.debug(f"Instance extraction kit details: {instance.extraction_kit.__dict__}") - cols_count = ceil(int(instance.sample_count) / 8) - instance.run_cost = instance.extraction_kit.constant_cost + (instance.extraction_kit.mutable_cost * (cols_count / 12)) + logger.debug(f"Calculating costs for procedure...") + # cols_count = ceil(int(instance.sample_count) / 8) + # instance.run_cost = instance.extraction_kit.constant_cost + (instance.extraction_kit.mutable_cost * (cols_count / 12)) + instance.calculate_base_cost() except (TypeError, AttributeError) as e: logger.debug(f"Looks like that kit doesn't have cost breakdown yet due to: {e}, using full plate cost.") instance.run_cost = instance.extraction_kit.cost_per_run + logger.debug(f"Calculated base run cost of: {instance.run_cost}") + try: + logger.debug("Checking and applying discounts...") + discounts = [item.amount for item in lookup_discounts_by_org_and_kit(ctx=ctx, kit_id=instance.extraction_kit.id, lab_id=instance.submitting_lab.id)] + logger.debug(f"We got discounts: {discounts}") + discounts = sum(discounts) + instance.run_cost = instance.run_cost - discounts + except Exception as e: + logger.error(f"An unknown exception occurred: {e}") # We need to make sure there's a proper rsl plate number + logger.debug(f"We've got a total cost of {instance.run_cost}") try: logger.debug(f"Constructed instance: {instance.to_string()}") except AttributeError as e: @@ -466,7 +477,7 @@ def create_kit_from_yaml(ctx:dict, exp:dict) -> dict: continue # A submission type may use multiple kits. for kt in exp[type]['kits']: - kit = models.KitType(name=kt, used_for=[type.replace("_", " ").title()], constant_cost=exp[type]["kits"][kt]["constant_cost"], mutable_cost=exp[type]["kits"][kt]["mutable_cost"]) + kit = models.KitType(name=kt, used_for=[type.replace("_", " ").title()], constant_cost=exp[type]["kits"][kt]["constant_cost"], mutable_cost_column=exp[type]["kits"][kt]["mutable_cost_column"]) # A kit contains multiple reagent types. for r in exp[type]['kits'][kt]['reagenttypes']: # check if reagent type already exists. @@ -736,4 +747,10 @@ def update_ww_sample(ctx:dict, sample_obj:dict): logger.error(f"Unable to find sample {sample_obj['sample']}") return ctx['database_session'].add(ww_samp) - ctx["database_session"].commit() \ No newline at end of file + ctx["database_session"].commit() + +def lookup_discounts_by_org_and_kit(ctx:dict, kit_id:int, lab_id:int): + return ctx['database_session'].query(models.Discount).join(models.KitType).join(models.Organization).filter(and_( + models.KitType.id==kit_id, + models.Organization.id==lab_id + )).all() \ No newline at end of file diff --git a/src/submissions/backend/db/models/__init__.py b/src/submissions/backend/db/models/__init__.py index c3ca906..7524ff3 100644 --- a/src/submissions/backend/db/models/__init__.py +++ b/src/submissions/backend/db/models/__init__.py @@ -7,7 +7,7 @@ Base = declarative_base() metadata = Base.metadata from .controls import Control, ControlType -from .kits import KitType, ReagentType, Reagent +from .kits import KitType, ReagentType, Reagent, Discount from .organizations import Organization, Contact from .samples import WWSample, BCSample from .submissions import BasicSubmission, BacterialCulture, Wastewater diff --git a/src/submissions/backend/db/models/kits.py b/src/submissions/backend/db/models/kits.py index 56bd1d6..43730d1 100644 --- a/src/submissions/backend/db/models/kits.py +++ b/src/submissions/backend/db/models/kits.py @@ -25,7 +25,9 @@ class KitType(Base): submissions = relationship("BasicSubmission", back_populates="extraction_kit") #: submissions this kit was used for used_for = Column(JSON) #: list of names of sample types this kit can process cost_per_run = Column(FLOAT(2)) #: dollar amount for each full run of this kit NOTE: depreciated, use the constant and mutable costs instead - mutable_cost = Column(FLOAT(2)) #: dollar amount per plate that can change with number of columns (reagents, tips, etc) + # TODO: Change below to 'mutable_cost_column' and 'mutable_cost_sample' before moving to production. + mutable_cost_column = Column(FLOAT(2)) #: dollar amount per 96 well plate that can change with number of columns (reagents, tips, etc) + mutable_cost_sample = Column(FLOAT(2)) #: dollar amount that can change with number of samples (reagents, tips, etc) constant_cost = Column(FLOAT(2)) #: dollar amount per plate that will remain constant (plates, man hours, etc) reagent_types = relationship("ReagentType", back_populates="kits", uselist=True, secondary=reagenttypes_kittypes) #: reagent types this kit contains reagent_types_id = Column(INTEGER, ForeignKey("_reagent_types.id", ondelete='SET NULL', use_alter=True, name="fk_KT_reagentstype_id")) #: joined reagent type id @@ -113,13 +115,16 @@ class Reagent(Base): } -# class Discounts(Base): -# """ -# Relationship table for client labs for certain kits. -# """ -# __tablename__ = "_discounts" +class Discount(Base): + """ + Relationship table for client labs for certain kits. + """ + __tablename__ = "_discounts" -# id = Column(INTEGER, primary_key=True) #: primary key -# kit = relationship("KitType") #: joined parent reagent type -# kit_id = Column(INTEGER, ForeignKey("_kits.id", ondelete='SET NULL', name="fk_kit_type_id")) -# client = relationship("Organization") \ No newline at end of file + id = Column(INTEGER, primary_key=True) #: primary key + kit = relationship("KitType") #: joined parent reagent type + kit_id = Column(INTEGER, ForeignKey("_kits.id", ondelete='SET NULL', name="fk_kit_type_id")) + client = relationship("Organization") #: joined client lab + client_id = Column(INTEGER, ForeignKey("_organizations.id", ondelete='SET NULL', name="fk_org_id")) + name = Column(String(128)) + amount = Column(FLOAT(2)) \ No newline at end of file diff --git a/src/submissions/backend/db/models/submissions.py b/src/submissions/backend/db/models/submissions.py index 15b2ad0..3be1144 100644 --- a/src/submissions/backend/db/models/submissions.py +++ b/src/submissions/backend/db/models/submissions.py @@ -8,6 +8,7 @@ from datetime import datetime as dt import logging import json from json.decoder import JSONDecodeError +from math import ceil logger = logging.getLogger(f"submissions.{__name__}") @@ -151,6 +152,8 @@ class BasicSubmission(Base): "Cost": self.run_cost } return output + + # Below are the custom submission types @@ -174,6 +177,18 @@ class BacterialCulture(BasicSubmission): return output + def calculate_base_cost(self): + try: + cols_count_96 = ceil(int(self.sample_count) / 8) + except Exception as e: + logger.error(f"Column count error: {e}") + # cols_count_24 = ceil(int(self.sample_count) / 3) + try: + self.run_cost = self.extraction_kit.constant_cost + (self.extraction_kit.mutable_cost_column * cols_count_96) + (self.extraction_kit.mutable_cost_sample * int(self.sample_count)) + except Exception as e: + logger.error(f"Calculation error: {e}") + + class Wastewater(BasicSubmission): """ derivative submission type from BasicSubmission @@ -195,4 +210,16 @@ class Wastewater(BasicSubmission): output['pcr_info'] = json.loads(self.pcr_info) except TypeError as e: pass - return output \ No newline at end of file + return output + + def calculate_base_cost(self): + try: + cols_count_96 = ceil(int(self.sample_count) / 8) + 1 #: Adding in one column to account for 24 samples + ext negatives + except Exception as e: + logger.error(f"Column count error: {e}") + # cols_count_24 = ceil(int(self.sample_count) / 3) + try: + self.run_cost = self.extraction_kit.constant_cost + (self.extraction_kit.mutable_cost_column * cols_count_96) + (self.extraction_kit.mutable_cost_sample * int(self.sample_count)) + except Exception as e: + logger.error(f"Calculation error: {e}") + \ No newline at end of file diff --git a/src/submissions/backend/excel/parser.py b/src/submissions/backend/excel/parser.py index 07c925d..69d0aa5 100644 --- a/src/submissions/backend/excel/parser.py +++ b/src/submissions/backend/excel/parser.py @@ -11,7 +11,7 @@ import logging from collections import OrderedDict import re import numpy as np -from datetime import date +from datetime import date, datetime import uuid from tools import check_not_nan, RSLNamer @@ -129,8 +129,12 @@ class SheetParser(object): logger.debug(f"Output variable is {output_var}") logger.debug(f"Expiry date for imported reagent: {row[3]}") if check_not_nan(row[3]): - expiry = row[3].date() + try: + expiry = row[3].date() + except AttributeError as e: + expiry = datetime.strptime(row[3], "%Y-%m-%d") else: + logger.debug(f"Date: {row[3]}") expiry = date.today() self.sub[f"lot_{reagent_type}"] = {'lot':output_var, 'exp':expiry} submission_info = self.parse_generic("Sample List") @@ -265,8 +269,8 @@ class SampleParser(object): new_list = [] for sample in self.samples: new = WWSample() - if check_not_nan(sample["Unnamed: 9"]): - new.rsl_number = sample['Unnamed: 9'] + if check_not_nan(sample["Unnamed: 7"]): + new.rsl_number = sample['Unnamed: 7'] # previously Unnamed: 9 else: logger.error(f"No RSL sample number found for this sample.") continue @@ -282,9 +286,9 @@ class SampleParser(object): new.collection_date = sample['Unnamed: 5'] else: new.collection_date = date.today() - new.testing_type = sample['Unnamed: 6'] - new.site_status = sample['Unnamed: 7'] - new.notes = str(sample['Unnamed: 8']) + # new.testing_type = sample['Unnamed: 6'] + # new.site_status = sample['Unnamed: 7'] + new.notes = str(sample['Unnamed: 6']) # previously Unnamed: 8 new.well_number = sample['Unnamed: 1'] new_list.append(new) return new_list