Improved retrieval of existing reagents.

src/submissions/backend/db/__init__.py

@@ -1,4 +1,6 @@
 from . import models
+from .models.kits import reagenttypes_kittypes
+from .models.submissions import reagents_submissions
 import pandas as pd
 import sqlalchemy.exc
 import sqlite3
@@ -7,13 +9,23 @@ from datetime import date, datetime, timedelta
 from sqlalchemy import and_
 import uuid
 # import base64
-from sqlalchemy import JSON
+from sqlalchemy import JSON, event
+from sqlalchemy.engine import Engine
 import json
 # from dateutil.relativedelta import relativedelta
 from getpass import getuser
+import numpy as np
 
 logger = logging.getLogger(f"submissions.{__name__}")
 
+# The below should allow automatic creation of foreign keys in the database
+@event.listens_for(Engine, "connect")
+def set_sqlite_pragma(dbapi_connection, connection_record):
+    cursor = dbapi_connection.cursor()
+    cursor.execute("PRAGMA foreign_keys=ON")
+    cursor.close()
+
+
 def get_kits_by_use( ctx:dict, kittype_str:str|None) -> list:
     pass
     # ctx dict should contain the database session
@@ -266,7 +278,7 @@ def lookup_regent_by_type_name(ctx:dict, type_name:str) -> list[models.Reagent]:
 
 def lookup_regent_by_type_name_and_kit_name(ctx:dict, type_name:str, kit_name:str) -> list[models.Reagent]:
     """
-    Lookup reagents by their type name and kits they belong to
+    Lookup reagents by their type name and kits they belong to (Broken)
 
     Args:
         ctx (dict): settings pass by gui
@@ -276,11 +288,31 @@ def lookup_regent_by_type_name_and_kit_name(ctx:dict, type_name:str, kit_name:st
     Returns:
         list[models.Reagent]: list of retrieved reagents
     """    
+    # What I want to do is get the reagent type by name 
     # Hang on, this is going to be a long one.
-    by_type = ctx['database_session'].query(models.Reagent).join(models.Reagent.type, aliased=True).filter(models.ReagentType.name.endswith(type_name))
+    # by_type = ctx['database_session'].query(models.Reagent).join(models.Reagent.type, aliased=True).filter(models.ReagentType.name.endswith(type_name)).all()
-    # add filter for kit name
+    rt_types = ctx['database_session'].query(models.ReagentType).filter(models.ReagentType.name.endswith(type_name))
-    add_in = by_type.join(models.ReagentType.kits).filter(models.KitType.name==kit_name)
+    
-    return add_in
+    
+    
+
+    # add filter for kit name... which I can not get to work.
+    # add_in = by_type.join(models.ReagentType.kits).filter(models.KitType.name==kit_name)
+    try:
+        check = not np.isnan(kit_name)
+    except TypeError:
+        check = True
+    if check:
+        kit_type = lookup_kittype_by_name(ctx=ctx, name=kit_name)
+        logger.debug(f"reagenttypes: {[item.name for item in rt_types.all()]}, kit: {kit_type.name}")
+        rt_types = rt_types.join(reagenttypes_kittypes).filter(reagenttypes_kittypes.c.kits_id==kit_type.id).first()
+
+        # for item in by_type:
+        #     logger.debug([thing.name for thing in item.type.kits])
+        # output = [item for item in by_type if kit_name in [thing.name for thing in item.type.kits]]
+    # else:
+    output = rt_types.instances
+    return output
 
 
 def lookup_all_submissions_by_type(ctx:dict, type:str|None=None) -> list[models.BasicSubmission]:
@@ -346,6 +378,10 @@ def submissions_to_df(ctx:dict, type:str|None=None) -> pd.DataFrame:
         df = df.drop("controls", axis=1)
     except:
         logger.warning(f"Couldn't drop 'controls' column from submissionsheet df.")
+    try:
+        df = df.drop("ext_info", axis=1)
+    except:
+        logger.warning(f"Couldn't drop 'controls' column from submissionsheet df.")
     # logger.debug(f"Post: {df['Technician']}")
     return df
      
@@ -428,6 +464,9 @@ def create_kit_from_yaml(ctx:dict, exp:dict) -> None:
                 else:
                     rt = look_up
                     rt.kits.append(kit)
+                    # add this because I think it's necessary to get proper back population
+                    # rt.kit_id.append(kit.id)
+                    kit.reagent_types_id.append(rt.id)
                 ctx['database_session'].add(rt)
                 logger.debug(rt.__dict__)
             logger.debug(kit.__dict__)
@@ -521,3 +560,15 @@ def get_control_subtypes(ctx:dict, type:str, mode:str) -> list[str]:
         return []
     subtypes = [item for item in jsoner[genera] if "_hashes" not in item and "_ratio" not in item]
     return subtypes
+
+
+def get_all_controls(ctx:dict):
+    return ctx['database_session'].query(models.Control).all()
+
+
+def lookup_submission_by_rsl_num(ctx:dict, rsl_num:str):
+    return ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.rsl_plate_num.startswith(rsl_num)).first()
+
+
+def lookup_submissions_using_reagent(ctx:dict, reagent:models.Reagent) -> list[models.BasicSubmission]:
+    return ctx['database_session'].query(models.BasicSubmission).join(reagents_submissions).filter(reagents_submissions.c.reagent_id==reagent.id)

src/submissions/backend/db/models/__init__.py

@@ -6,6 +6,6 @@ metadata = Base.metadata
 
 from .controls import Control, ControlType
 from .kits import KitType, ReagentType, Reagent
-from .submissions import BasicSubmission, BacterialCulture, Wastewater
 from .organizations import Organization, Contact
-from .samples import WWSample, BCSample
+from .samples import WWSample, BCSample
+from .submissions import BasicSubmission, BacterialCulture, Wastewater

src/submissions/backend/db/models/submissions.py

@@ -3,6 +3,7 @@ from sqlalchemy import Column, String, TIMESTAMP, INTEGER, ForeignKey, Table, JS
 from sqlalchemy.orm import relationship
 from datetime import datetime as dt
 import logging
+import json
 
 logger = logging.getLogger(f"submissions.{__name__}")
 
@@ -69,6 +70,10 @@ class BasicSubmission(Base):
             ext_kit = self.extraction_kit.name
         except AttributeError:
             ext_kit = None
+        try:
+            ext_info = json.loads(self.extraction_info)
+        except TypeError:
+            ext_info = None
         output = {
             "id": self.id,
             "Plate Number": self.rsl_plate_num,
@@ -80,8 +85,9 @@ class BasicSubmission(Base):
             "Extraction Kit": ext_kit,
             "Technician": self.technician,
             "Cost": self.run_cost,
+            "ext_info": ext_info
         }
-        logger.debug(f"{self.rsl_plate_num} technician: {output['Technician']}")
+        # logger.debug(f"{self.rsl_plate_num} extraction: {output['Extraction Status']}")
         return output
 
 

src/submissions/backend/excel/parser.py

@@ -92,6 +92,9 @@ class SheetParser(object):
 
         def _parse_reagents(df:pd.DataFrame) -> None:
             for ii, row in df.iterrows():
+                # skip positive control
+                if ii == 11:
+                    continue
                 logger.debug(f"Running reagent parse for {row[1]} with type {type(row[1])} and value: {row[2]} with type {type(row[2])}")
                 try:
                     check = not np.isnan(row[1])
@@ -162,7 +165,7 @@ class SheetParser(object):
                     check = True
                 if not isinstance(row[5], float) and check:
                     # must be prefixed with 'lot_' to be recognized by gui
-                    output_key = re.sub(r"\d{1,3}%", "", row[0].replace(' ', '_').lower())
+                    output_key = re.sub(r"\d{1,3}%", "", row[0].lower().strip().replace(' ', '_'))
                     try:
                         output_var = row[5].upper()
                     except AttributeError:

src/submissions/configure/__init__.py

@@ -130,16 +130,18 @@ def get_config(settings_path: str|None=None) -> dict:
     return settings
 
 
-def create_database_session(database_path: Path|None) -> Session:
+def create_database_session(database_path: Path|str|None=None) -> Session:
     """
     Get database settings from path or default if blank.
 
     Args:
-        database_path (str, optional): _description_. Defaults to "".
+        database_path (Path | str | None, optional): path to sqlite database. Defaults to None.
-    
+
     Returns:
-        database_path: string of database path
+        Session: database session
-    """
+    """    
+    if isinstance(database_path, str):
+        database_path = Path(database_path)
     if database_path == None:
         if Path.home().joinpath(".submissions", "submissions.db").exists():
             database_path = Path.home().joinpath(".submissions", "submissions.db")

src/submissions/frontend/__init__.py

@@ -1,3 +1,4 @@
+import json
 import re
 from PyQt6.QtWidgets import (
     QMainWindow, QLabel, QToolBar, 
@@ -16,7 +17,6 @@ from pathlib import Path
 import plotly
 import pandas as pd
 from openpyxl.utils import get_column_letter
-from openpyxl.styles import NamedStyle
 from xhtml2pdf import pisa
 # import plotly.express as px
 import yaml
@@ -26,9 +26,9 @@ from backend.excel.parser import SheetParser
 from backend.excel.reports import convert_control_by_mode, convert_data_list_to_df
 from backend.db import (construct_submission_info, lookup_reagent, 
     construct_reagent, store_reagent, store_submission, lookup_kittype_by_use,
-    lookup_regent_by_type_name_and_kit_name, lookup_all_orgs, lookup_submissions_by_date_range,
+    lookup_regent_by_type_name, lookup_all_orgs, lookup_submissions_by_date_range,
     get_all_Control_Types_names, create_kit_from_yaml, get_all_available_modes, get_all_controls_by_type,
-    get_control_subtypes
+    get_control_subtypes, lookup_all_submissions_by_type, get_all_controls, lookup_submission_by_rsl_num
 )
 from backend.excel.reports import make_report_xlsx, make_report_html
 import numpy
@@ -79,9 +79,12 @@ class App(QMainWindow):
         # Creating menus using a title
         editMenu = menuBar.addMenu("&Edit")
         reportMenu = menuBar.addMenu("&Reports")
+        maintenanceMenu = menuBar.addMenu("&Monthly")
         helpMenu = menuBar.addMenu("&Help")
         fileMenu.addAction(self.importAction)
         reportMenu.addAction(self.generateReportAction)
+        maintenanceMenu.addAction(self.joinControlsAction)
+        maintenanceMenu.addAction(self.joinExtractionAction)
         
     def _createToolBar(self):
         """
@@ -100,6 +103,8 @@ class App(QMainWindow):
         self.addReagentAction = QAction("Add Reagent", self)
         self.generateReportAction = QAction("Make Report", self)
         self.addKitAction = QAction("Add Kit", self)
+        self.joinControlsAction = QAction("Link Controls")
+        self.joinExtractionAction = QAction("Link Ext Logs")
 
 
     def _connectActions(self):
@@ -114,6 +119,8 @@ class App(QMainWindow):
         self.table_widget.mode_typer.currentIndexChanged.connect(self._controls_getter)
         self.table_widget.datepicker.start_date.dateChanged.connect(self._controls_getter)
         self.table_widget.datepicker.end_date.dateChanged.connect(self._controls_getter)
+        self.joinControlsAction.triggered.connect(self.linkControls)
+        self.joinExtractionAction.triggered.connect(self.linkExtractions)
 
 
     def importSubmission(self):
@@ -215,7 +222,17 @@ class App(QMainWindow):
                         except ValueError:
                             pass
                     # query for reagents using type name from sheet and kit from sheet
-                    relevant_reagents = [item.__str__() for item in lookup_regent_by_type_name_and_kit_name(ctx=self.ctx, type_name=query_var, kit_name=prsr.sub['extraction_kit'])]
+                    logger.debug(f"Attempting lookup of reagents by type: {query_var} and kit: {prsr.sub['extraction_kit']}")
+                    # below was lookup_reagent_by_type_name_and_kit_name, but I couldn't get it to work.
+                    relevant_reagents = [item.__str__() for item in lookup_regent_by_type_name(ctx=self.ctx, type_name=query_var)]#, kit_name=prsr.sub['extraction_kit'])]
+                    output_reg = []
+                    for reagent in relevant_reagents:
+                        if isinstance(reagent, set):
+                            for thing in reagent:
+                                output_reg.append(thing)
+                        elif isinstance(reagent, str):
+                            output_reg.append(reagent)
+                    relevant_reagents = output_reg
                     logger.debug(f"Relevant reagents: {relevant_reagents}")
                     # if reagent in sheet is not found insert it into items
                     if prsr.sub[item] not in relevant_reagents and prsr.sub[item] != 'nan':
@@ -283,6 +300,7 @@ class App(QMainWindow):
         # add reagents to submission object
         for reagent in parsed_reagents:
             base_submission.reagents.append(reagent)
+            # base_submission.reagents_id = reagent.id
         logger.debug(f"Sending submission: {base_submission.rsl_plate_num} to database.")
         result = store_submission(ctx=self.ctx, base_submission=base_submission)
         # check result of storing for issues
@@ -463,7 +481,7 @@ class App(QMainWindow):
             # block signal that will rerun controls getter and set start date
             with QSignalBlocker(self.table_widget.datepicker.start_date) as blocker:
                 self.table_widget.datepicker.start_date.setDate(threemonthsago)
-            self.controls_getter()
+            self._controls_getter()
             return
             # convert to python useable date object
         self.start_date = self.table_widget.datepicker.start_date.date().toPyDate()
@@ -534,6 +552,102 @@ class App(QMainWindow):
         logger.debug("Figure updated... I hope.")
 
 
+    def linkControls(self):
+        # all_bcs = self.ctx['database_session'].query(models.BacterialCulture).all()
+        all_bcs = lookup_all_submissions_by_type(self.ctx, "Bacterial Culture")
+        logger.debug(all_bcs)
+        all_controls = get_all_controls(self.ctx)
+        ac_list = [control.name for control in all_controls]
+        count = 0
+        for bcs in all_bcs:
+            logger.debug(f"Running for {bcs.rsl_plate_num}")
+            logger.debug(f"Here is the current control: {[control.name for control in bcs.controls]}")
+            samples = [sample.sample_id for sample in bcs.samples]
+            logger.debug(bcs.controls)
+            for sample in samples:
+                # if "Jan" in sample and "EN" in sample:
+                #     sample = sample.replace("EN-", "EN1-")
+                #     logger.debug(f"Checking for {sample}")
+                # replace below is a stopgap method because some dingus decided to add spaces in some of the ATCC49... so it looks like "ATCC 49"...
+                if " " in sample:
+                    logger.warning(f"There is not supposed to be a space in the sample name!!!")
+                    sample = sample.replace(" ", "")
+                if sample not in ac_list:
+                    continue
+                else:
+                    for control in all_controls:
+                        diff = difflib.SequenceMatcher(a=sample, b=control.name).ratio()
+                        if diff > 0.955:
+                            logger.debug(f"Checking {sample} against {control.name}... {diff}")
+                        # if sample == control.name:
+                            logger.debug(f"Found match:\n\tSample: {sample}\n\tControl: {control.name}\n\tDifference: {diff}")
+                            if control in bcs.controls:
+                                logger.debug(f"{control.name} already in {bcs.rsl_plate_num}, skipping")
+                                continue
+                            else:
+                                logger.debug(f"Adding {control.name} to {bcs.rsl_plate_num} as control")
+                                bcs.controls.append(control)
+                                # bcs.control_id.append(control.id)
+                                control.submission = bcs
+                                control.submission_id = bcs.id
+                                self.ctx["database_session"].add(control)
+                                count += 1
+            self.ctx["database_session"].add(bcs)
+            # logger.debug(f"To be added: {ctx['database_session'].new}")
+            logger.debug(f"Here is the new control: {[control.name for control in bcs.controls]}")
+            # p = ctx["database_session"].query(models.BacterialCulture).filter(models.BacterialCulture.rsl_plate_num==bcs.rsl_plate_num).first()
+        result = f"We added {count} controls to bacterial cultures."
+        logger.debug(result)
+        # logger.debug(ctx["database_session"].new)
+        self.ctx['database_session'].commit()
+        msg = QMessageBox()
+        # msg.setIcon(QMessageBox.critical)
+        msg.setText("Controls added")
+        msg.setInformativeText(result)
+        msg.setWindowTitle("Controls added")
+        msg.exec()
+
+
+
+    def linkExtractions(self):
+        home_dir = str(Path(self.ctx["directory_path"]))
+        fname = Path(QFileDialog.getOpenFileName(self, 'Open file', home_dir, filter = "csv(*.csv)")[0])
+        with open(fname.__str__(), 'r') as f:
+            runs = [col.strip().split(",") for col in f.readlines()]
+        # check = []
+        for run in runs:
+            obj = dict(
+                    start_time=run[0].strip(), 
+                    rsl_plate_num=run[1].strip(), 
+                    sample_count=run[2].strip(), 
+                    status=run[3].strip(),
+                    experiment_name=run[4].strip(),
+                    end_time=run[5].strip()
+                )
+            for ii in range(6, len(run)):
+                obj[f"column{str(ii-5)}_vol"] = run[ii]
+            # check.append(json.dumps(obj))
+            # sub = self.ctx['database_session'].query(models.BasicSubmission).filter(models.BasicSubmission.rsl_plate_num.startswith(obj["rsl_plate_num"])).first()
+            sub = lookup_submission_by_rsl_num(ctx=self.ctx, rsl_num=obj['rsl_plate_num'])
+            try:
+                logger.debug(f"Found submission: {sub.rsl_plate_num}")
+            except AttributeError:
+                continue
+            output = json.dumps(obj)
+            try:
+                if output in sub.extraction_info:
+                    logger.debug(f"Looks like we already have that info.")
+                    continue
+            except TypeError:
+                pass
+            try:
+                sub.extraction_info += output
+            except TypeError:
+                sub.extraction_info = output
+            self.ctx['database_session'].add(sub)
+            self.ctx["database_session"].commit()
+
+
 class AddSubForm(QWidget):
     
     def __init__(self, parent):

src/submissions/templates/submission_details.html

@@ -5,28 +5,58 @@
     </head>
     <body>
         <h2><u>Submission Details for {{ sub['Plate Number'] }}</u></h2>
-        {% for key, value in sub.items() if key != 'reagents' and key != 'samples' and key != 'controls' %}
+        <p>{% for key, value in sub.items() if key != 'reagents' and key != 'samples' and key != 'controls' and key != 'ext_info' %}
-        {% if key=='Cost' %} <p>{{ key }}: {{ "${:,.2f}".format(value) }}</p> {% else %} <p>{{ key }}: {{ value }}</p> {% endif %}
+            {% if loop.index == 1 %}
-        {% endfor %}
+                &nbsp;&nbsp;&nbsp;{% if key=='Cost' %}{{ key }}: {{ "${:,.2f}".format(value) }}{% else %}{{ key }}: {{ value }}{% endif %}<br>
+            {% else %}
+                &nbsp;&nbsp;&nbsp;&nbsp;{% if key=='Cost' %}{{ key }}: {{ "${:,.2f}".format(value) }}{% else %}{{ key }}: {{ value }}{% endif %}<br>
+            {% endif %}
+        {% endfor %}</p>
         <h3><u>Reagents:</u></h3>
-        {% for item in sub['reagents'] %}
+        <p>{% for item in sub['reagents'] %}
-            <p>{{ item['type'] }}: {{ item['lot'] }} (EXP: {{ item['expiry'] }})</p>
+            {% if loop.index == 1%}
-        {% endfor %}
+                &nbsp;&nbsp;&nbsp;{{ item['type'] }}: {{ item['lot'] }} (EXP: {{ item['expiry'] }})<br>
+            {% else %}
+                &nbsp;&nbsp;&nbsp;&nbsp;{{ item['type'] }}: {{ item['lot'] }} (EXP: {{ item['expiry'] }})<br>
+            {% endif %}
+        {% endfor %}</p>
         <h3><u>Samples:</u></h3>
-        {% for item in sub['samples'] %}
+        <p>{% for item in sub['samples'] %}
-            <p>{{ item['well'] }}: {{ item['name'] }}</p>
+            {% if loop.index == 1 %}
-        {% endfor %}
+                &nbsp;&nbsp;&nbsp;{{ item['well'] }}: {{ item['name'] }}<br>
+            {% else %}
+                &nbsp;&nbsp;&nbsp;&nbsp;{{ item['well'] }}: {{ item['name'] }}<br>
+            {% endif %}
+        {% endfor %}</p>
         {% if sub['controls'] %}
         <h3><u>Attached Controls:</u></h3>
         {% for item in sub['controls'] %}
-            <p><b>{{ item['name'] }}:</b> {{ item['type'] }} (Targets: {{ item['targets'] }})</p>
+            <p>&nbsp;&nbsp;&nbsp;<b>{{ item['name'] }}:</b> {{ item['type'] }} (Targets: {{ item['targets'] }})</p>
             {% if item['kraken'] %}
-            <p>{{ item['name'] }} Top 5 Kraken Results</p>
+            <p>&nbsp;&nbsp;&nbsp;{{ item['name'] }} Top 5 Kraken Results:</p>
-            {% for genera in item['kraken'] %}
+            <p>{% for genera in item['kraken'] %}
-                <p>{{ genera['name'] }}: {{ genera['kraken_count'] }} ({{ genera['kraken_percent'] }})</p>
+                {% if loop.index == 1 %}
-            {% endfor %}
+                &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;{{ genera['name'] }}: {{ genera['kraken_count'] }} ({{ genera['kraken_percent'] }})<br>
+                {% else %}
+                &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;{{ genera['name'] }}: {{ genera['kraken_count'] }} ({{ genera['kraken_percent'] }})<br>
+                {% endif %}
+            {% endfor %}</p>
             {% endif %}
         {% endfor %}
         {% endif %}
+        {% if sub['ext_info'] %}
+        <h3><u>Extraction Status:</u></h3>
+        <p>{% for key, value in sub['ext_info'].items() %}
+            {% if loop.index == 1%}
+                &nbsp;&nbsp;&nbsp;{{ key|replace('_', ' ')|title() }}: {{ value }}<br>
+            {% else %}
+                {% if "column" in key %}
+                    &nbsp;&nbsp;&nbsp;&nbsp;{{ key|replace('_', ' ')|title() }}: {{ value }}uL<br>
+                {% else %}
+                    &nbsp;&nbsp;&nbsp;&nbsp;{{ key|replace('_', ' ')|title() }}: {{ value }}<br>
+                {% endif %}
+            {% endif %}
+        {% endfor %}</p>
+        {% endif %}
     </body>
 </html>

src/submissions/templates/submission_details.txt

@@ -1,16 +1,15 @@
 {# template for constructing submission details #}
 
-{% for key, value in sub.items() if key != 'reagents' and key != 'samples' and key != 'controls' %}
+{% for key, value in sub.items() if key != 'reagents' and key != 'samples' and key != 'controls' and key != 'ext_info' %}
 {% if key=='Cost' %} {{ key }}: {{ "${:,.2f}".format(value) }} {% else %} {{ key }}: {{ value }} {% endif %}
 {% endfor %}
 Reagents:
 {% for item in sub['reagents'] %}
-    {{ item['type'] }}: {{ item['lot'] }} (EXP: {{ item['expiry'] }})
+    {{ item['type'] }}: {{ item['lot'] }} (EXP: {{ item['expiry'] }}){% endfor %}
-{% endfor %}
+    
 Samples:
 {% for item in sub['samples'] %}
-    {{ item['well'] }}: {{ item['name'] }}
+    {{ item['well'] }}: {{ item['name'] }}{% endfor %}
-{% endfor %}
 {% if sub['controls'] %}
 Attached Controls:
 {% for item in sub['controls'] %}
@@ -19,5 +18,9 @@ Attached Controls:
         {{ item['name'] }} Top 5 Kraken Results
         {% for genera in item['kraken'] %}
             {{ genera['name'] }}: {{ genera['kraken_count'] }} ({{ genera['kraken_percent'] }}){% endfor %}{% endif %}
-{% endfor %}
+{% endfor %}{% endif %}
+{% if sub['ext_info'] %}
+Extraction Status:
+{% for key, value in sub['ext_info'].items() %}
+    {{ key|replace('_', ' ')|title() }}: {{ value }}{% endfor %}
 {% endif %}