diff --git a/src/submissions/backend/db/__init__.py b/src/submissions/backend/db/__init__.py
index cfe0fc7..22424e9 100644
--- a/src/submissions/backend/db/__init__.py
+++ b/src/submissions/backend/db/__init__.py
@@ -1,4 +1,6 @@
from . import models
+from .models.kits import reagenttypes_kittypes
+from .models.submissions import reagents_submissions
import pandas as pd
import sqlalchemy.exc
import sqlite3
@@ -7,13 +9,23 @@ from datetime import date, datetime, timedelta
from sqlalchemy import and_
import uuid
# import base64
-from sqlalchemy import JSON
+from sqlalchemy import JSON, event
+from sqlalchemy.engine import Engine
import json
# from dateutil.relativedelta import relativedelta
from getpass import getuser
+import numpy as np
logger = logging.getLogger(f"submissions.{__name__}")
+# The below should allow automatic creation of foreign keys in the database
+@event.listens_for(Engine, "connect")
+def set_sqlite_pragma(dbapi_connection, connection_record):
+ cursor = dbapi_connection.cursor()
+ cursor.execute("PRAGMA foreign_keys=ON")
+ cursor.close()
+
+
def get_kits_by_use( ctx:dict, kittype_str:str|None) -> list:
pass
# ctx dict should contain the database session
@@ -266,7 +278,7 @@ def lookup_regent_by_type_name(ctx:dict, type_name:str) -> list[models.Reagent]:
def lookup_regent_by_type_name_and_kit_name(ctx:dict, type_name:str, kit_name:str) -> list[models.Reagent]:
"""
- Lookup reagents by their type name and kits they belong to
+ Lookup reagents by their type name and kits they belong to (Broken)
Args:
ctx (dict): settings pass by gui
@@ -276,11 +288,31 @@ def lookup_regent_by_type_name_and_kit_name(ctx:dict, type_name:str, kit_name:st
Returns:
list[models.Reagent]: list of retrieved reagents
"""
+ # What I want to do is get the reagent type by name
# Hang on, this is going to be a long one.
- by_type = ctx['database_session'].query(models.Reagent).join(models.Reagent.type, aliased=True).filter(models.ReagentType.name.endswith(type_name))
- # add filter for kit name
- add_in = by_type.join(models.ReagentType.kits).filter(models.KitType.name==kit_name)
- return add_in
+ # by_type = ctx['database_session'].query(models.Reagent).join(models.Reagent.type, aliased=True).filter(models.ReagentType.name.endswith(type_name)).all()
+ rt_types = ctx['database_session'].query(models.ReagentType).filter(models.ReagentType.name.endswith(type_name))
+
+
+
+
+ # add filter for kit name... which I can not get to work.
+ # add_in = by_type.join(models.ReagentType.kits).filter(models.KitType.name==kit_name)
+ try:
+ check = not np.isnan(kit_name)
+ except TypeError:
+ check = True
+ if check:
+ kit_type = lookup_kittype_by_name(ctx=ctx, name=kit_name)
+ logger.debug(f"reagenttypes: {[item.name for item in rt_types.all()]}, kit: {kit_type.name}")
+ rt_types = rt_types.join(reagenttypes_kittypes).filter(reagenttypes_kittypes.c.kits_id==kit_type.id).first()
+
+ # for item in by_type:
+ # logger.debug([thing.name for thing in item.type.kits])
+ # output = [item for item in by_type if kit_name in [thing.name for thing in item.type.kits]]
+ # else:
+ output = rt_types.instances
+ return output
def lookup_all_submissions_by_type(ctx:dict, type:str|None=None) -> list[models.BasicSubmission]:
@@ -346,6 +378,10 @@ def submissions_to_df(ctx:dict, type:str|None=None) -> pd.DataFrame:
df = df.drop("controls", axis=1)
except:
logger.warning(f"Couldn't drop 'controls' column from submissionsheet df.")
+ try:
+ df = df.drop("ext_info", axis=1)
+ except:
+ logger.warning(f"Couldn't drop 'controls' column from submissionsheet df.")
# logger.debug(f"Post: {df['Technician']}")
return df
@@ -428,6 +464,9 @@ def create_kit_from_yaml(ctx:dict, exp:dict) -> None:
else:
rt = look_up
rt.kits.append(kit)
+ # add this because I think it's necessary to get proper back population
+ # rt.kit_id.append(kit.id)
+ kit.reagent_types_id.append(rt.id)
ctx['database_session'].add(rt)
logger.debug(rt.__dict__)
logger.debug(kit.__dict__)
@@ -521,3 +560,15 @@ def get_control_subtypes(ctx:dict, type:str, mode:str) -> list[str]:
return []
subtypes = [item for item in jsoner[genera] if "_hashes" not in item and "_ratio" not in item]
return subtypes
+
+
+def get_all_controls(ctx:dict):
+ return ctx['database_session'].query(models.Control).all()
+
+
+def lookup_submission_by_rsl_num(ctx:dict, rsl_num:str):
+ return ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.rsl_plate_num.startswith(rsl_num)).first()
+
+
+def lookup_submissions_using_reagent(ctx:dict, reagent:models.Reagent) -> list[models.BasicSubmission]:
+ return ctx['database_session'].query(models.BasicSubmission).join(reagents_submissions).filter(reagents_submissions.c.reagent_id==reagent.id)
\ No newline at end of file
diff --git a/src/submissions/backend/db/models/__init__.py b/src/submissions/backend/db/models/__init__.py
index 1a99d0b..f8c9d7f 100644
--- a/src/submissions/backend/db/models/__init__.py
+++ b/src/submissions/backend/db/models/__init__.py
@@ -6,6 +6,6 @@ metadata = Base.metadata
from .controls import Control, ControlType
from .kits import KitType, ReagentType, Reagent
-from .submissions import BasicSubmission, BacterialCulture, Wastewater
from .organizations import Organization, Contact
-from .samples import WWSample, BCSample
\ No newline at end of file
+from .samples import WWSample, BCSample
+from .submissions import BasicSubmission, BacterialCulture, Wastewater
diff --git a/src/submissions/backend/db/models/submissions.py b/src/submissions/backend/db/models/submissions.py
index 445da54..4da816e 100644
--- a/src/submissions/backend/db/models/submissions.py
+++ b/src/submissions/backend/db/models/submissions.py
@@ -3,6 +3,7 @@ from sqlalchemy import Column, String, TIMESTAMP, INTEGER, ForeignKey, Table, JS
from sqlalchemy.orm import relationship
from datetime import datetime as dt
import logging
+import json
logger = logging.getLogger(f"submissions.{__name__}")
@@ -69,6 +70,10 @@ class BasicSubmission(Base):
ext_kit = self.extraction_kit.name
except AttributeError:
ext_kit = None
+ try:
+ ext_info = json.loads(self.extraction_info)
+ except TypeError:
+ ext_info = None
output = {
"id": self.id,
"Plate Number": self.rsl_plate_num,
@@ -80,8 +85,9 @@ class BasicSubmission(Base):
"Extraction Kit": ext_kit,
"Technician": self.technician,
"Cost": self.run_cost,
+ "ext_info": ext_info
}
- logger.debug(f"{self.rsl_plate_num} technician: {output['Technician']}")
+ # logger.debug(f"{self.rsl_plate_num} extraction: {output['Extraction Status']}")
return output
diff --git a/src/submissions/backend/excel/parser.py b/src/submissions/backend/excel/parser.py
index e8ad0bb..83e14b3 100644
--- a/src/submissions/backend/excel/parser.py
+++ b/src/submissions/backend/excel/parser.py
@@ -92,6 +92,9 @@ class SheetParser(object):
def _parse_reagents(df:pd.DataFrame) -> None:
for ii, row in df.iterrows():
+ # skip positive control
+ if ii == 11:
+ continue
logger.debug(f"Running reagent parse for {row[1]} with type {type(row[1])} and value: {row[2]} with type {type(row[2])}")
try:
check = not np.isnan(row[1])
@@ -162,7 +165,7 @@ class SheetParser(object):
check = True
if not isinstance(row[5], float) and check:
# must be prefixed with 'lot_' to be recognized by gui
- output_key = re.sub(r"\d{1,3}%", "", row[0].replace(' ', '_').lower())
+ output_key = re.sub(r"\d{1,3}%", "", row[0].lower().strip().replace(' ', '_'))
try:
output_var = row[5].upper()
except AttributeError:
diff --git a/src/submissions/configure/__init__.py b/src/submissions/configure/__init__.py
index cccce2e..ebe3df5 100644
--- a/src/submissions/configure/__init__.py
+++ b/src/submissions/configure/__init__.py
@@ -130,16 +130,18 @@ def get_config(settings_path: str|None=None) -> dict:
return settings
-def create_database_session(database_path: Path|None) -> Session:
+def create_database_session(database_path: Path|str|None=None) -> Session:
"""
Get database settings from path or default if blank.
Args:
- database_path (str, optional): _description_. Defaults to "".
-
+ database_path (Path | str | None, optional): path to sqlite database. Defaults to None.
+
Returns:
- database_path: string of database path
- """
+ Session: database session
+ """
+ if isinstance(database_path, str):
+ database_path = Path(database_path)
if database_path == None:
if Path.home().joinpath(".submissions", "submissions.db").exists():
database_path = Path.home().joinpath(".submissions", "submissions.db")
diff --git a/src/submissions/frontend/__init__.py b/src/submissions/frontend/__init__.py
index 4f19a43..59387f9 100644
--- a/src/submissions/frontend/__init__.py
+++ b/src/submissions/frontend/__init__.py
@@ -1,3 +1,4 @@
+import json
import re
from PyQt6.QtWidgets import (
QMainWindow, QLabel, QToolBar,
@@ -16,7 +17,6 @@ from pathlib import Path
import plotly
import pandas as pd
from openpyxl.utils import get_column_letter
-from openpyxl.styles import NamedStyle
from xhtml2pdf import pisa
# import plotly.express as px
import yaml
@@ -26,9 +26,9 @@ from backend.excel.parser import SheetParser
from backend.excel.reports import convert_control_by_mode, convert_data_list_to_df
from backend.db import (construct_submission_info, lookup_reagent,
construct_reagent, store_reagent, store_submission, lookup_kittype_by_use,
- lookup_regent_by_type_name_and_kit_name, lookup_all_orgs, lookup_submissions_by_date_range,
+ lookup_regent_by_type_name, lookup_all_orgs, lookup_submissions_by_date_range,
get_all_Control_Types_names, create_kit_from_yaml, get_all_available_modes, get_all_controls_by_type,
- get_control_subtypes
+ get_control_subtypes, lookup_all_submissions_by_type, get_all_controls, lookup_submission_by_rsl_num
)
from backend.excel.reports import make_report_xlsx, make_report_html
import numpy
@@ -79,9 +79,12 @@ class App(QMainWindow):
# Creating menus using a title
editMenu = menuBar.addMenu("&Edit")
reportMenu = menuBar.addMenu("&Reports")
+ maintenanceMenu = menuBar.addMenu("&Monthly")
helpMenu = menuBar.addMenu("&Help")
fileMenu.addAction(self.importAction)
reportMenu.addAction(self.generateReportAction)
+ maintenanceMenu.addAction(self.joinControlsAction)
+ maintenanceMenu.addAction(self.joinExtractionAction)
def _createToolBar(self):
"""
@@ -100,6 +103,8 @@ class App(QMainWindow):
self.addReagentAction = QAction("Add Reagent", self)
self.generateReportAction = QAction("Make Report", self)
self.addKitAction = QAction("Add Kit", self)
+ self.joinControlsAction = QAction("Link Controls")
+ self.joinExtractionAction = QAction("Link Ext Logs")
def _connectActions(self):
@@ -114,6 +119,8 @@ class App(QMainWindow):
self.table_widget.mode_typer.currentIndexChanged.connect(self._controls_getter)
self.table_widget.datepicker.start_date.dateChanged.connect(self._controls_getter)
self.table_widget.datepicker.end_date.dateChanged.connect(self._controls_getter)
+ self.joinControlsAction.triggered.connect(self.linkControls)
+ self.joinExtractionAction.triggered.connect(self.linkExtractions)
def importSubmission(self):
@@ -215,7 +222,17 @@ class App(QMainWindow):
except ValueError:
pass
# query for reagents using type name from sheet and kit from sheet
- relevant_reagents = [item.__str__() for item in lookup_regent_by_type_name_and_kit_name(ctx=self.ctx, type_name=query_var, kit_name=prsr.sub['extraction_kit'])]
+ logger.debug(f"Attempting lookup of reagents by type: {query_var} and kit: {prsr.sub['extraction_kit']}")
+ # below was lookup_reagent_by_type_name_and_kit_name, but I couldn't get it to work.
+ relevant_reagents = [item.__str__() for item in lookup_regent_by_type_name(ctx=self.ctx, type_name=query_var)]#, kit_name=prsr.sub['extraction_kit'])]
+ output_reg = []
+ for reagent in relevant_reagents:
+ if isinstance(reagent, set):
+ for thing in reagent:
+ output_reg.append(thing)
+ elif isinstance(reagent, str):
+ output_reg.append(reagent)
+ relevant_reagents = output_reg
logger.debug(f"Relevant reagents: {relevant_reagents}")
# if reagent in sheet is not found insert it into items
if prsr.sub[item] not in relevant_reagents and prsr.sub[item] != 'nan':
@@ -283,6 +300,7 @@ class App(QMainWindow):
# add reagents to submission object
for reagent in parsed_reagents:
base_submission.reagents.append(reagent)
+ # base_submission.reagents_id = reagent.id
logger.debug(f"Sending submission: {base_submission.rsl_plate_num} to database.")
result = store_submission(ctx=self.ctx, base_submission=base_submission)
# check result of storing for issues
@@ -463,7 +481,7 @@ class App(QMainWindow):
# block signal that will rerun controls getter and set start date
with QSignalBlocker(self.table_widget.datepicker.start_date) as blocker:
self.table_widget.datepicker.start_date.setDate(threemonthsago)
- self.controls_getter()
+ self._controls_getter()
return
# convert to python useable date object
self.start_date = self.table_widget.datepicker.start_date.date().toPyDate()
@@ -534,6 +552,102 @@ class App(QMainWindow):
logger.debug("Figure updated... I hope.")
+ def linkControls(self):
+ # all_bcs = self.ctx['database_session'].query(models.BacterialCulture).all()
+ all_bcs = lookup_all_submissions_by_type(self.ctx, "Bacterial Culture")
+ logger.debug(all_bcs)
+ all_controls = get_all_controls(self.ctx)
+ ac_list = [control.name for control in all_controls]
+ count = 0
+ for bcs in all_bcs:
+ logger.debug(f"Running for {bcs.rsl_plate_num}")
+ logger.debug(f"Here is the current control: {[control.name for control in bcs.controls]}")
+ samples = [sample.sample_id for sample in bcs.samples]
+ logger.debug(bcs.controls)
+ for sample in samples:
+ # if "Jan" in sample and "EN" in sample:
+ # sample = sample.replace("EN-", "EN1-")
+ # logger.debug(f"Checking for {sample}")
+ # replace below is a stopgap method because some dingus decided to add spaces in some of the ATCC49... so it looks like "ATCC 49"...
+ if " " in sample:
+ logger.warning(f"There is not supposed to be a space in the sample name!!!")
+ sample = sample.replace(" ", "")
+ if sample not in ac_list:
+ continue
+ else:
+ for control in all_controls:
+ diff = difflib.SequenceMatcher(a=sample, b=control.name).ratio()
+ if diff > 0.955:
+ logger.debug(f"Checking {sample} against {control.name}... {diff}")
+ # if sample == control.name:
+ logger.debug(f"Found match:\n\tSample: {sample}\n\tControl: {control.name}\n\tDifference: {diff}")
+ if control in bcs.controls:
+ logger.debug(f"{control.name} already in {bcs.rsl_plate_num}, skipping")
+ continue
+ else:
+ logger.debug(f"Adding {control.name} to {bcs.rsl_plate_num} as control")
+ bcs.controls.append(control)
+ # bcs.control_id.append(control.id)
+ control.submission = bcs
+ control.submission_id = bcs.id
+ self.ctx["database_session"].add(control)
+ count += 1
+ self.ctx["database_session"].add(bcs)
+ # logger.debug(f"To be added: {ctx['database_session'].new}")
+ logger.debug(f"Here is the new control: {[control.name for control in bcs.controls]}")
+ # p = ctx["database_session"].query(models.BacterialCulture).filter(models.BacterialCulture.rsl_plate_num==bcs.rsl_plate_num).first()
+ result = f"We added {count} controls to bacterial cultures."
+ logger.debug(result)
+ # logger.debug(ctx["database_session"].new)
+ self.ctx['database_session'].commit()
+ msg = QMessageBox()
+ # msg.setIcon(QMessageBox.critical)
+ msg.setText("Controls added")
+ msg.setInformativeText(result)
+ msg.setWindowTitle("Controls added")
+ msg.exec()
+
+
+
+ def linkExtractions(self):
+ home_dir = str(Path(self.ctx["directory_path"]))
+ fname = Path(QFileDialog.getOpenFileName(self, 'Open file', home_dir, filter = "csv(*.csv)")[0])
+ with open(fname.__str__(), 'r') as f:
+ runs = [col.strip().split(",") for col in f.readlines()]
+ # check = []
+ for run in runs:
+ obj = dict(
+ start_time=run[0].strip(),
+ rsl_plate_num=run[1].strip(),
+ sample_count=run[2].strip(),
+ status=run[3].strip(),
+ experiment_name=run[4].strip(),
+ end_time=run[5].strip()
+ )
+ for ii in range(6, len(run)):
+ obj[f"column{str(ii-5)}_vol"] = run[ii]
+ # check.append(json.dumps(obj))
+ # sub = self.ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.rsl_plate_num.startswith(obj["rsl_plate_num"])).first()
+ sub = lookup_submission_by_rsl_num(ctx=self.ctx, rsl_num=obj['rsl_plate_num'])
+ try:
+ logger.debug(f"Found submission: {sub.rsl_plate_num}")
+ except AttributeError:
+ continue
+ output = json.dumps(obj)
+ try:
+ if output in sub.extraction_info:
+ logger.debug(f"Looks like we already have that info.")
+ continue
+ except TypeError:
+ pass
+ try:
+ sub.extraction_info += output
+ except TypeError:
+ sub.extraction_info = output
+ self.ctx['database_session'].add(sub)
+ self.ctx["database_session"].commit()
+
+
class AddSubForm(QWidget):
def __init__(self, parent):
diff --git a/src/submissions/templates/submission_details.html b/src/submissions/templates/submission_details.html
index 5f801c0..185f925 100644
--- a/src/submissions/templates/submission_details.html
+++ b/src/submissions/templates/submission_details.html
@@ -5,28 +5,58 @@
Submission Details for {{ sub['Plate Number'] }}
- {% for key, value in sub.items() if key != 'reagents' and key != 'samples' and key != 'controls' %}
- {% if key=='Cost' %} {{ key }}: {{ "${:,.2f}".format(value) }}
{% else %} {{ key }}: {{ value }}
{% endif %}
- {% endfor %}
+ {% for key, value in sub.items() if key != 'reagents' and key != 'samples' and key != 'controls' and key != 'ext_info' %}
+ {% if loop.index == 1 %}
+ {% if key=='Cost' %}{{ key }}: {{ "${:,.2f}".format(value) }}{% else %}{{ key }}: {{ value }}{% endif %}
+ {% else %}
+ {% if key=='Cost' %}{{ key }}: {{ "${:,.2f}".format(value) }}{% else %}{{ key }}: {{ value }}{% endif %}
+ {% endif %}
+ {% endfor %}
Reagents:
- {% for item in sub['reagents'] %}
- {{ item['type'] }}: {{ item['lot'] }} (EXP: {{ item['expiry'] }})
- {% endfor %}
+ {% for item in sub['reagents'] %}
+ {% if loop.index == 1%}
+ {{ item['type'] }}: {{ item['lot'] }} (EXP: {{ item['expiry'] }})
+ {% else %}
+ {{ item['type'] }}: {{ item['lot'] }} (EXP: {{ item['expiry'] }})
+ {% endif %}
+ {% endfor %}
Samples:
- {% for item in sub['samples'] %}
- {{ item['well'] }}: {{ item['name'] }}
- {% endfor %}
+ {% for item in sub['samples'] %}
+ {% if loop.index == 1 %}
+ {{ item['well'] }}: {{ item['name'] }}
+ {% else %}
+ {{ item['well'] }}: {{ item['name'] }}
+ {% endif %}
+ {% endfor %}
{% if sub['controls'] %}
Attached Controls:
{% for item in sub['controls'] %}
- {{ item['name'] }}: {{ item['type'] }} (Targets: {{ item['targets'] }})
+ {{ item['name'] }}: {{ item['type'] }} (Targets: {{ item['targets'] }})
{% if item['kraken'] %}
- {{ item['name'] }} Top 5 Kraken Results
- {% for genera in item['kraken'] %}
- {{ genera['name'] }}: {{ genera['kraken_count'] }} ({{ genera['kraken_percent'] }})
- {% endfor %}
+ {{ item['name'] }} Top 5 Kraken Results:
+ {% for genera in item['kraken'] %}
+ {% if loop.index == 1 %}
+ {{ genera['name'] }}: {{ genera['kraken_count'] }} ({{ genera['kraken_percent'] }})
+ {% else %}
+ {{ genera['name'] }}: {{ genera['kraken_count'] }} ({{ genera['kraken_percent'] }})
+ {% endif %}
+ {% endfor %}
{% endif %}
{% endfor %}
{% endif %}
+ {% if sub['ext_info'] %}
+ Extraction Status:
+ {% for key, value in sub['ext_info'].items() %}
+ {% if loop.index == 1%}
+ {{ key|replace('_', ' ')|title() }}: {{ value }}
+ {% else %}
+ {% if "column" in key %}
+ {{ key|replace('_', ' ')|title() }}: {{ value }}uL
+ {% else %}
+ {{ key|replace('_', ' ')|title() }}: {{ value }}
+ {% endif %}
+ {% endif %}
+ {% endfor %}
+ {% endif %}