landon/Submissions-App
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added in ability to overwrite submissions.

Browse Source
This commit is contained in:
Landon Wark
2023-01-26 14:54:32 -06:00
parent 3c308f6559
commit bbb65d3fe6
9 changed files with 151 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
src/submissions/backend/db/__init__.py
View File

@@ -1,7 +1,9 @@
from . import models
import pandas as pd
# from sqlite3 import IntegrityError
from sqlalchemy.exc import IntegrityError
import sqlalchemy.exc
import sqlite3
# from sqlalchemy.exc import IntegrityError, OperationalError
# from sqlite3 import IntegrityError, OperationalError
import logging
from datetime import date, datetime
from sqlalchemy import and_
@@ -19,48 +21,69 @@ def get_kits_by_use( ctx:dict, kittype_str:str|None) -> list:
def store_submission(ctx:dict, base_submission:models.BasicSubmission) -> None:
logger.debug(f"Hello from store_submission")
for sample in base_submission.samples:
sample.rsl_plate = base_submission
logger.debug(f"Attempting to add sample: {sample.to_string()}")
try:
ctx['database_session'].add(sample)
except IntegrityError:
except (sqlite3.IntegrityError, sqlalchemy.exc.IntegrityError) as e:
logger.debug(f"Hit an integrity error : {e}")
continue
ctx['database_session'].add(base_submission)
logger.debug(f"Attempting to add submission: {base_submission.rsl_plate_num}")
try:
ctx['database_session'].commit()
except IntegrityError:
except (sqlite3.IntegrityError, sqlalchemy.exc.IntegrityError) as e:
logger.debug(f"Hit an integrity error : {e}")
ctx['database_session'].rollback()
return {"message":"This plate number already exists, so we can't add it."}
except (sqlite3.OperationalError, sqlalchemy.exc.IntegrityError) as e:
logger.debug(f"Hit an operational error: {e}")
ctx['database_session'].rollback()
return {"message":"The database is locked for editing."}
return None
def store_reagent(ctx:dict, reagent:models.Reagent) -> None:
logger.debug(reagent.__dict__)
ctx['database_session'].add(reagent)
ctx['database_session'].commit()
try:
ctx['database_session'].commit()
except OperationalError:
return {"message":"The database is locked for editing."}
def construct_submission_info(ctx:dict, info_dict:dict) -> models.BasicSubmission:
query = info_dict['submission_type'].replace(" ", "")
instance = ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.rsl_plate_num==info_dict['rsl_plate_num']).first()
msg = "This submission already exists.\nWould you like to overwrite?"
model = getattr(models, query)
info_dict['submission_type'] = info_dict['submission_type'].replace(" ", "_").lower()
instance = model()
if instance == None:
instance = model()
msg = None
for item in info_dict:
logger.debug(f"Setting {item} to {info_dict[item]}")
match item:
case "extraction_kit":
q_str = info_dict[item]
logger.debug(f"Looking up kit {q_str}")
field_value = lookup_kittype_by_name(ctx=ctx, name=q_str)
try:
field_value = lookup_kittype_by_name(ctx=ctx, name=q_str)
except (sqlite3.IntegrityError, sqlalchemy.exc.IntegrityError) as e:
logger.error(f"Hit an integrity error: {e}")
logger.debug(f"Got {field_value} for kit {q_str}")
case "submitting_lab":
q_str = info_dict[item].replace(" ", "_").lower()
logger.debug(f"looking up organization: {q_str}")
logger.debug(f"Looking up organization: {q_str}")
field_value = lookup_org_by_name(ctx=ctx, name=q_str)
logger.debug(f"Got {field_value} for organization {q_str}")
case "submitter_plate_num":
# Because of unique constraint, the submitter plate number cannot be None, so...
if info_dict[item] == None:
logger.debug(f"Submitter plate id: {info_dict[item]}")
if info_dict[item] == None or info_dict[item] == "None":
logger.debug(f"Got None as a submitter plate number, inserting random string to preserve database unique constraint.")
info_dict[item] = uuid.uuid4().hex.upper()
field_value = info_dict[item]
# case "samples":
@@ -75,7 +98,9 @@ def construct_submission_info(ctx:dict, info_dict:dict) -> models.BasicSubmissio
logger.debug(f"Could not set attribute: {item} to {info_dict[item]}")
continue
# logger.debug(instance.__dict__)
return instance
logger.debug(f"Constructed instance: {instance.to_string()}")
logger.debug(msg)
return instance, {'message':msg}
# looked_up = []
# for reagent in reagents:
# my_reagent = lookup_reagent(reagent)
@@ -129,7 +154,7 @@ def lookup_kittype_by_use(ctx:dict, used_by:str) -> list[models.KitType]:
def lookup_kittype_by_name(ctx:dict, name:str) -> models.KitType:
logger.debug(f"Querying kittype: {name}")
return ctx['database_session'].query(models.KitType).filter(models.KitType.name==name).first()
def lookup_regent_by_type_name(ctx:dict, type_name:str) -> list[models.ReagentType]:
# return [item for item in ctx['database_session'].query(models.Reagent).join(models.Reagent.type, aliased=True).filter(models.ReagentType.name==type_name).all()]
@@ -235,7 +260,11 @@ def lookup_all_sample_types(ctx:dict) -> list[str]:
def get_all_available_modes(ctx:dict) -> list[str]:
rel = ctx['database_session'].query(models.Control).first()
cols = [item.name for item in list(rel.__table__.columns) if isinstance(item.type, JSON)]
try:
cols = [item.name for item in list(rel.__table__.columns) if isinstance(item.type, JSON)]
except AttributeError as e:
logger.debug(f"Failed to get available modes from db: {e}")
cols = []
return cols

3
src/submissions/backend/db/models/controls.py
View File

@@ -32,5 +32,6 @@ class Control(Base):
matches = Column(JSON) #: unstructured hashes in matches.tsv for each organism
kraken = Column(JSON) #: unstructured output from kraken_report
# UniqueConstraint('name', name='uq_control_name')
submissions = relationship("BacterialCulture", back_populates="control")
submission_id = Column(INTEGER, ForeignKey("_submissions.id"))
submission = relationship("BacterialCulture", back_populates="controls", foreign_keys=[submission_id])

7
src/submissions/backend/db/models/submissions.py
View File

@@ -30,6 +30,9 @@ class BasicSubmission(Base):
"with_polymorphic": "*",
}
def to_string(self):
return f"{self.rsl_plate_num} - {self.submitter_plate_num}"
def to_dict(self):
try:
sub_lab = self.submitting_lab.name
@@ -90,8 +93,8 @@ class BasicSubmission(Base):
# Below are the custom submission
class BacterialCulture(BasicSubmission):
control = relationship("Control", back_populates="submissions") #: A control sample added to submission
control_id = Column(INTEGER, ForeignKey("_control_samples.id", ondelete="SET NULL", name="fk_BC_control_id"))
# control_id = Column(INTEGER, ForeignKey("_control_samples.id", ondelete="SET NULL", name="fk_BC_control_id"))
controls = relationship("Control", back_populates="submission", uselist=True) #: A control sample added to submission
samples = relationship("BCSample", back_populates="rsl_plate", uselist=True)
# bc_sample_id = Column(INTEGER, ForeignKey("_bc_samples.id", ondelete="SET NULL", name="fk_BC_sample_id"))
__mapper_args__ = {"polymorphic_identity": "bacterial_culture", "polymorphic_load": "inline"}

82
src/submissions/backend/excel/parser.py
View File

@@ -45,13 +45,14 @@ class SheetParser(object):
def _parse_generic(self, sheet_name:str):
submission_info = self.xl.parse(sheet_name=sheet_name)
self.sub['submitter_plate_num'] = submission_info.iloc[0][1]
self.sub['rsl_plate_num'] = str(submission_info.iloc[10][1])
self.sub['submitted_date'] = submission_info.iloc[1][1].date()#.strftime("%Y-%m-%d")
self.sub['submitting_lab'] = submission_info.iloc[0][3]
self.sub['sample_count'] = str(submission_info.iloc[2][3])
self.sub['extraction_kit'] = submission_info.iloc[3][3]
submission_info = self.xl.parse(sheet_name=sheet_name, dtype=object)
self.sub['submitter_plate_num'] = submission_info.iloc[0][1] #if pd.isnull(submission_info.iloc[0][1]) else string_formatter(submission_info.iloc[0][1])
self.sub['rsl_plate_num'] = submission_info.iloc[10][1] #if pd.isnull(submission_info.iloc[10][1]) else string_formatter(submission_info.iloc[10][1])
self.sub['submitted_date'] = submission_info.iloc[1][1] #if pd.isnull(submission_info.iloc[1][1]) else submission_info.iloc[1][1].date()#.strftime("%Y-%m-%d")
self.sub['submitting_lab'] = submission_info.iloc[0][3] #if pd.isnull(submission_info.iloc[0][3]) else string_formatter(submission_info.iloc[0][3])
self.sub['sample_count'] = submission_info.iloc[2][3] #if pd.isnull(submission_info.iloc[2][3]) else string_formatter(submission_info.iloc[2][3])
self.sub['extraction_kit'] = submission_info.iloc[3][3] #if #pd.isnull(submission_info.iloc[3][3]) else string_formatter(submission_info.iloc[3][3])
return submission_info
@@ -67,16 +68,17 @@ class SheetParser(object):
tech = ", ".join(tech_reg.findall(tech))
self.sub['technician'] = tech
# reagents
self.sub['lot_wash_1'] = submission_info.iloc[1][6]
self.sub['lot_wash_2'] = submission_info.iloc[2][6]
self.sub['lot_binding_buffer'] = submission_info.iloc[3][6]
self.sub['lot_magnetic_beads'] = submission_info.iloc[4][6]
self.sub['lot_lysis_buffer'] = submission_info.iloc[5][6]
self.sub['lot_elution_buffer'] = submission_info.iloc[6][6]
self.sub['lot_isopropanol'] = submission_info.iloc[9][6]
self.sub['lot_ethanol'] = submission_info.iloc[10][6]
self.sub['lot_positive_control'] = submission_info.iloc[103][1]
self.sub['lot_plate'] = submission_info.iloc[12][6]
self.sub['lot_wash_1'] = submission_info.iloc[1][6] #if pd.isnull(submission_info.iloc[1][6]) else string_formatter(submission_info.iloc[1][6])
self.sub['lot_wash_2'] = submission_info.iloc[2][6] #if pd.isnull(submission_info.iloc[2][6]) else string_formatter(submission_info.iloc[2][6])
self.sub['lot_binding_buffer'] = submission_info.iloc[3][6] #if pd.isnull(submission_info.iloc[3][6]) else string_formatter(submission_info.iloc[3][6])
self.sub['lot_magnetic_beads'] = submission_info.iloc[4][6] #if pd.isnull(submission_info.iloc[4][6]) else string_formatter(submission_info.iloc[4][6])
self.sub['lot_lysis_buffer'] = submission_info.iloc[5][6] #if np.nan(submission_info.iloc[5][6]) else string_formatter(submission_info.iloc[5][6])
self.sub['lot_elution_buffer'] = submission_info.iloc[6][6] #if pd.isnull(submission_info.iloc[6][6]) else string_formatter(submission_info.iloc[6][6])
self.sub['lot_isopropanol'] = submission_info.iloc[9][6] #if pd.isnull(submission_info.iloc[9][6]) else string_formatter(submission_info.iloc[9][6])
self.sub['lot_ethanol'] = submission_info.iloc[10][6] #if pd.isnull(submission_info.iloc[10][6]) else string_formatter(submission_info.iloc[10][6])
self.sub['lot_positive_control'] = submission_info.iloc[103][1] #if pd.isnull(submission_info.iloc[103][1]) else string_formatter(submission_info.iloc[103][1])
self.sub['lot_plate'] = submission_info.iloc[12][6] #if pd.isnull(submission_info.iloc[12][6]) else string_formatter(submission_info.iloc[12][6])
sample_parser = SampleParser(submission_info.iloc[15:111])
sample_parse = getattr(sample_parser, f"parse_{self.sub['submission_type'].lower()}_samples")
logger.debug(f"Parser result: {self.sub}")
@@ -86,25 +88,26 @@ class SheetParser(object):
def _parse_wastewater(self):
# submission_info = self.xl.parse("WW Submissions (ENTER HERE)")
submission_info = self._parse_generic("WW Submissions (ENTER HERE)")
enrichment_info = self.xl.parse("Enrichment Worksheet")
extraction_info = self.xl.parse("Extraction Worksheet")
qprc_info = self.xl.parse("qPCR Worksheet")
enrichment_info = self.xl.parse("Enrichment Worksheet", dtype=object)
extraction_info = self.xl.parse("Extraction Worksheet", dtype=object)
qprc_info = self.xl.parse("qPCR Worksheet", dtype=object)
self.sub['technician'] = f"Enr: {enrichment_info.columns[2]}, Ext: {extraction_info.columns[2]}, PCR: {qprc_info.columns[2]}"
# reagents
self.sub['lot_lysis_buffer'] = enrichment_info.iloc[0][14]
self.sub['lot_proteinase_K'] = enrichment_info.iloc[1][14]
self.sub['lot_magnetic_virus_particles'] = enrichment_info.iloc[2][14]
self.sub['lot_enrichment_reagent_1'] = enrichment_info.iloc[3][14]
self.sub['lot_binding_buffer'] = extraction_info.iloc[0][14]
self.sub['lot_magnetic_beads'] = extraction_info.iloc[1][14]
self.sub['lot_wash'] = extraction_info.iloc[2][14]
self.sub['lot_ethanol'] = extraction_info.iloc[3][14]
self.sub['lot_elution_buffer'] = extraction_info.iloc[4][14]
self.sub['lot_master_mix'] = qprc_info.iloc[0][14]
self.sub['lot_pre_mix_1'] = qprc_info.iloc[1][14]
self.sub['lot_pre_mix_2'] = qprc_info.iloc[2][14]
self.sub['lot_positive_control'] = qprc_info.iloc[3][14]
self.sub['lot_ddh2o'] = qprc_info.iloc[4][14]
logger.debug(qprc_info)
self.sub['lot_lysis_buffer'] = enrichment_info.iloc[0][14] #if pd.isnull(enrichment_info.iloc[0][14]) else string_formatter(enrichment_info.iloc[0][14])
self.sub['lot_proteinase_K'] = enrichment_info.iloc[1][14] #if pd.isnull(enrichment_info.iloc[1][14]) else string_formatter(enrichment_info.iloc[1][14])
self.sub['lot_magnetic_virus_particles'] = enrichment_info.iloc[2][14] #if pd.isnull(enrichment_info.iloc[2][14]) else string_formatter(enrichment_info.iloc[2][14])
self.sub['lot_enrichment_reagent_1'] = enrichment_info.iloc[3][14] #if pd.isnull(enrichment_info.iloc[3][14]) else string_formatter(enrichment_info.iloc[3][14])
self.sub['lot_binding_buffer'] = extraction_info.iloc[0][14] #if pd.isnull(extraction_info.iloc[0][14]) else string_formatter(extraction_info.iloc[0][14])
self.sub['lot_magnetic_beads'] = extraction_info.iloc[1][14] #if pd.isnull(extraction_info.iloc[1][14]) else string_formatter(extraction_info.iloc[1][14])
self.sub['lot_wash'] = extraction_info.iloc[2][14] #if pd.isnull(extraction_info.iloc[2][14]) else string_formatter(extraction_info.iloc[2][14])
self.sub['lot_ethanol'] = extraction_info.iloc[3][14] #if pd.isnull(extraction_info.iloc[3][14]) else string_formatter(extraction_info.iloc[3][14])
self.sub['lot_elution_buffer'] = extraction_info.iloc[4][14] #if pd.isnull(extraction_info.iloc[4][14]) else string_formatter(extraction_info.iloc[4][14])
self.sub['lot_master_mix'] = qprc_info.iloc[0][14] #if pd.isnull(qprc_info.iloc[0][14]) else string_formatter(qprc_info.iloc[0][14])
self.sub['lot_pre_mix_1'] = qprc_info.iloc[1][14] #if pd.isnull(qprc_info.iloc[1][14]) else string_formatter(qprc_info.iloc[1][14])
self.sub['lot_pre_mix_2'] = qprc_info.iloc[2][14] #if pd.isnull(qprc_info.iloc[2][14]) else string_formatter(qprc_info.iloc[2][14])
self.sub['lot_positive_control'] = qprc_info.iloc[3][14] #if pd.isnull(qprc_info.iloc[3][14]) else string_formatter(qprc_info.iloc[3][14])
self.sub['lot_ddh2o'] = qprc_info.iloc[4][14] #if pd.isnull(qprc_info.iloc[4][14]) else string_formatter(qprc_info.iloc[4][14])
sample_parser = SampleParser(submission_info.iloc[16:40])
sample_parse = getattr(sample_parser, f"parse_{self.sub['submission_type'].lower()}_samples")
self.sub['samples'] = sample_parse()
@@ -164,4 +167,13 @@ class SampleParser(object):
new.well_number = sample['Unnamed: 1']
new_list.append(new)
return new_list
def string_formatter(input):
logger.debug(f"{input} : {type(input)}")
match input:
case int() | float() | np.float64:
return "{:0.0f}".format(input)
case _:
return input