landon/Submissions-App
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Qubit results parsing complete.

Browse Source
This commit is contained in:
lwark
2025-09-23 08:57:40 -05:00
parent 6f1202d3ba
commit 39d20bbc22
15 changed files with 113 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
CHANGELOG.md
View File

@@ -1,10 +1,11 @@
# 202509.03
- Sortable headers in treeview.
- Added gitea remote.
# 202509.02
- First Useable updated version.
- First usable updated version.
# 202504.04
@@ -12,7 +13,7 @@
# 202504.03
- Split Concentration controls on the chart so they are individually selectable.
- Split Concentration controls on the chart, so they are individually selectable.
# 202504.02
@@ -315,7 +316,7 @@
## 202307.03
- Auto-filling of some empty cells in Excel file.
- Autofilling of some empty cells in Excel file.
- Better pydantic validations of missing data.
## 202307.02

4
TODO.md
View File

@@ -1,5 +1,5 @@
- [ ] Add in database objects for rsl_run (submission -> run), procedure (run -> procedure), many more things will likely be associated with procedure.
- [ ] Add in database object for client submission.
- [x] Add in database objects for rsl_run (submission -> run), procedure (run -> procedure), many more things will likely be associated with procedure.
- [x] Add in database object for client submission.
- [ ] Add arbitrary pipette addition to equipment UI.
- [ ] transfer details template rendering fully into sql objects
- [x] Add in connecting links for tips.

4
src/submissions/backend/db/models/__init__.py
View File

@@ -18,7 +18,7 @@ from sqlalchemy.exc import ArgumentError
from typing import Any, List, ClassVar
from pathlib import Path
from sqlalchemy.orm.relationships import _RelationshipDeclared
from tools import report_result, list_sort_dict, jinja_template_loading, Report, Result, ctx
from tools import report_result, list_sort_dict, jinja_template_loading, Report, Alert, ctx
# NOTE: Load testing environment
if 'pytest' in sys.modules:
@@ -364,7 +364,7 @@ class BaseClass(Base):
logger.error(f"Error message: {type(e)}")
logger.error(pformat(self.__dict__))
self.__database_session__.rollback()
report.add_result(Result(msg=e, status="Critical"))
report.add_result(Alert(msg=e, status="Critical"))
return report
@property

10
src/submissions/backend/db/models/procedures.py
View File

@@ -4,13 +4,14 @@ All kittype and reagent related models
from __future__ import annotations
import zipfile, logging, re, numpy as np
from operator import itemgetter
from pathlib import Path
from pprint import pformat
from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey, Interval, Table, FLOAT, func
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.orm import relationship, validates, Query, declared_attr
from sqlalchemy.ext.associationproxy import association_proxy
from datetime import date, datetime, timedelta
from tools import check_authorization, setup_lookup, Report, Result, check_regex_match, timezone, \
from tools import check_authorization, setup_lookup, Report, Alert, check_regex_match, timezone, \
jinja_template_loading, flatten_list
from typing import List, Literal, Generator, Any, Tuple, TYPE_CHECKING
from . import BaseClass, ClientLab, LogMixin
@@ -926,10 +927,13 @@ class Procedure(BaseClass):
logger.info(f"Add Results! {resultstype_name}")
from backend.managers import results
results_manager = getattr(results, f"{resultstype_name}Manager")
rs = results_manager(procedure=self, parent=obj)
rs = results_manager(procedure=self, parent=obj, fname=Path("C:\\Users\lwark\Documents\Submission_Forms\QubitData_18-09-2025_13-43-53.csv"))
procedure = rs.procedure_to_pydantic()
samples = rs.samples_to_pydantic()
procedure_sql = procedure.to_sql()
if procedure:
procedure_sql = procedure.to_sql()
else:
return
procedure_sql.save()
for sample in samples:
sample_sql = sample.to_sql()

13
src/submissions/backend/db/models/submissions.py
View File

@@ -18,7 +18,8 @@ from sqlalchemy.orm.attributes import flag_modified
from sqlalchemy.ext.associationproxy import association_proxy
from sqlalchemy.exc import OperationalError as AlcOperationalError, IntegrityError as AlcIntegrityError, StatementError
from sqlite3 import OperationalError as SQLOperationalError, IntegrityError as SQLIntegrityError
from tools import setup_lookup, jinja_template_loading, create_holidays_for_year, check_dictionary_inclusion_equality, is_power_user
from tools import (setup_lookup, jinja_template_loading, create_holidays_for_year,
check_dictionary_inclusion_equality, is_power_user, row_map)
from datetime import datetime, date
from typing import List, Literal, Generator, TYPE_CHECKING
from pathlib import Path
@@ -1865,6 +1866,16 @@ class ProcedureSampleAssociation(BaseClass):
results = relationship("Results", back_populates="sampleprocedureassociation") #: associated results
@property
def well(self):
if self.row > 0:
if self.column > 0:
return f"{row_map[self.row]}{self.column}"
else:
return self.row
else:
return None
@classmethod
def query(cls, sample: Sample | str | None = None, procedure: Procedure | str | None = None, limit: int = 0,
**kwargs):

26
src/submissions/backend/excel/parsers/__init__.py
View File

@@ -2,11 +2,13 @@
Default Parser archetypes.
"""
from __future__ import annotations
import logging, re
import logging, re, csv
from pathlib import Path
from pprint import pformat
from typing import Generator, TYPE_CHECKING
from openpyxl.cell import MergedCell
from openpyxl.reader.excel import load_workbook
from openpyxl.workbook import Workbook
from pandas import DataFrame
from backend.validators import pydant
if TYPE_CHECKING:
@@ -44,6 +46,8 @@ class DefaultParser(object):
**kwargs ():
"""
logger.info(f"\n\nHello from {self.__class__.__name__}\n\n")
if isinstance(filepath, str):
filepath = Path(filepath)
self.filepath = filepath
self.proceduretype = proceduretype
try:
@@ -58,13 +62,27 @@ class DefaultParser(object):
self.sheet = sheet
if not start_row:
start_row = self.__class__.start_row
self.workbook = load_workbook(self.filepath, data_only=True)
self.worksheet = self.workbook[self.sheet]
if self.filepath.suffix == ".xslx":
self.workbook = load_workbook(self.filepath, data_only=True)
self.worksheet = self.workbook[self.sheet]
elif self.filepath.suffix == ".csv":
self.workbook, self.worksheet = self.csv2xlsx(self.filepath)
self.start_row = self.delineate_start_row(start_row=start_row)
self.end_row = self.delineate_end_row(start_row=self.start_row)
@classmethod
def csv2xlsx(cls, filepath):
wb = Workbook()
ws = wb.active
with open(filepath, "r") as f:
reader = csv.reader(f, delimiter=",")
for row in reader:
ws.append(row)
return wb, ws
def to_pydantic(self):
data = self.parsed_info
logger.debug(f"Data for {self.__class__.__name__}: {pformat(data)}")
data['filepath'] = self.filepath
return self._pyd_object(**data)
@@ -85,7 +103,7 @@ class DefaultParser(object):
for iii, row in enumerate(self.worksheet.iter_rows(min_row=start_row), start=start_row):
if all([item.value is None for item in row]):
return iii
return self.worksheet.max_row
return self.worksheet.max_row + 1
class DefaultKEYVALUEParser(DefaultParser):

36
src/submissions/backend/excel/parsers/results_parsers/__init__.py
View File

@@ -12,12 +12,22 @@ logger = logging.getLogger(f"submissions.{__name__}")
class DefaultResultsInfoParser(DefaultKEYVALUEParser):
pyd_name = "PydResults"
def __init__(self, filepath: Path | str, proceduretype: "ProcedureType" | None = None,
results_type: str | None = "PCR", *args, **kwargs):
def __init__(self, filepath: Path | str, results_type: str, proceduretype: "ProcedureType" | None = None,
*args, **kwargs):
if results_type:
self.results_type = results_type
sheet = proceduretype.allowed_result_methods[results_type]['info']['sheet']
start_row = proceduretype.allowed_result_methods[results_type]['info']['start_row']
try:
sheet = proceduretype.allowed_result_methods[results_type]['info']['sheet']
except KeyError:
sheet = 1
if "start_row" not in kwargs:
try:
start_row = proceduretype.allowed_result_methods[results_type]['info']['start_row']
except KeyError:
start_row = 1
else:
start_row = kwargs.pop('start_row')
# start_row = proceduretype.allowed_result_methods[results_type]['info']['start_row']
super().__init__(filepath=filepath, proceduretype=proceduretype, sheet=sheet, start_row=start_row, *args,
**kwargs)
@@ -25,14 +35,24 @@ class DefaultResultsInfoParser(DefaultKEYVALUEParser):
class DefaultResultsSampleParser(DefaultTABLEParser):
pyd_name = "PydResults"
def __init__(self, filepath: Path | str, proceduretype: "ProcedureType" | None = None,
results_type: str | None = "PCR", *args, **kwargs):
def __init__(self, filepath: Path | str, results_type: str, proceduretype: "ProcedureType" | None = None,
*args, **kwargs):
if results_type:
self.results_type = results_type
sheet = proceduretype.allowed_result_methods[results_type]['sample']['sheet']
start_row = proceduretype.allowed_result_methods[results_type]['sample']['start_row']
try:
sheet = proceduretype.allowed_result_methods[results_type]['sample']['sheet']
except KeyError:
sheet = 1
if "start_row" not in kwargs:
try:
start_row = proceduretype.allowed_result_methods[results_type]['sample']['start_row']
except KeyError:
start_row = 1
else:
start_row = kwargs.pop('start_row')
super().__init__(filepath=filepath, proceduretype=proceduretype, sheet=sheet, start_row=start_row, *args,
**kwargs)
from .pcr_results_parser import PCRInfoParser, PCRSampleParser
from .qubit_results_parser import QubitInfoParser, QubitSampleParser

13
src/submissions/backend/managers/results/__init__.py
View File

@@ -17,16 +17,20 @@ logger = logging.getLogger(f"submission.{__name__}")
class DefaultResultsManager(DefaultManager):
def __init__(self, procedure: Procedure, parent, fname: Path | str | None = None):
def __init__(self, procedure: Procedure, parent, fname: Path | str | None = None, extension: str|None="xlsx"):
self.procedure = procedure
if not fname:
self.fname = select_open_file(file_extension="xlsx", obj=get_application_from_parent(parent))
fname = select_open_file(file_extension=extension, obj=get_application_from_parent(parent))
elif isinstance(fname, str):
self.fname = Path(fname)
fname = Path(fname)
self.fname = fname
def procedure_to_pydantic(self) -> PydResults:
logger.debug(f"Info parser: {self.info_parser}")
info = self.info_parser.to_pydantic()
info.parent = self.procedure
if info:
info.parent = self.procedure
return info
def samples_to_pydantic(self) -> List[PydResults]:
@@ -34,3 +38,4 @@ class DefaultResultsManager(DefaultManager):
return sample
from .pcr_results_manager import PCRManager
from .qubit_results_manager import QubitManager

8
src/submissions/backend/validators/pydant.py
View File

@@ -12,7 +12,7 @@ from typing import List, Tuple, Literal, Generator
from types import GeneratorType
from . import RSLNamer
from pathlib import Path
from tools import check_not_nan, convert_nans_to_nones, Report, Result, timezone, sort_dict_by_list, row_keys, flatten_list
from tools import check_not_nan, convert_nans_to_nones, Report, Alert, timezone, sort_dict_by_list, row_keys, flatten_list
from backend.db import models
from backend.db.models import *
from sqlalchemy.orm.properties import ColumnProperty
@@ -1397,14 +1397,14 @@ class PydRun(PydBaseClass): #, extra='allow'):
Converts this instance into a backend.db.models.procedure.BasicRun instance
Returns:
Tuple[BasicRun, Result]: BasicRun instance, result object
Tuple[BasicRun, Alert]: BasicRun instance, result object
"""
report = Report()
dicto = self.improved_dict()
instance, result = Run.query_or_create(submissiontype=self.submission_type['value'],
rsl_plate_number=self.rsl_plate_number['value'])
if instance is None:
report.add_result(Result(msg="Overwrite Cancelled."))
report.add_result(Alert(msg="Overwrite Cancelled."))
return None, report
report.add_result(result)
self.handle_duplicate_samples()
@@ -1585,7 +1585,7 @@ class PydRun(PydBaseClass): #, extra='allow'):
expired.append(f"{reagent.role}, {reagent.lot}: {reagent.expiry.date()} + {role_eol.days}")
if expired:
output = '\n'.join(expired)
result = Result(status="Warning",
result = Alert(status="Warning",
msg=f"The following reagents are expired:\n\n{output}"
)
report.add_result(result)

4
src/submissions/frontend/widgets/info_tab.py
View File

@@ -5,7 +5,7 @@ from datetime import date
from PyQt6.QtCore import QSignalBlocker
from PyQt6.QtWebEngineWidgets import QWebEngineView
from PyQt6.QtWidgets import QWidget, QGridLayout
from tools import Report, report_result, Result
from tools import Report, report_result, Alert
from .misc import StartEndDatePicker
from .functions import select_save_file, save_pdf
import logging
@@ -42,7 +42,7 @@ class InfoPane(QWidget):
with QSignalBlocker(self.datepicker.start_date) as blocker:
self.datepicker.start_date.setDate(lastmonth)
self.update_data()
report.add_result(Result(owner=self.__str__(), msg=msg, status="Warning"))
report.add_result(Alert(owner=self.__str__(), msg=msg, status="Warning"))
return report
@classmethod

4
src/submissions/frontend/widgets/submission_table.py
View File

@@ -109,9 +109,7 @@ class SubmissionsTree(QTreeView):
sets data in model
"""
self.clear()
self.data = [item.to_dict(full_data=True) for item in
# self.data = [item.details_dict() for item in
ClientSubmission.query(chronologic=True, page=page, page_size=page_size)]
self.data = [item.to_dict(full_data=True) for item in ClientSubmission.query(chronologic=True, page=page, page_size=page_size)]
root = self.model.invisibleRootItem()
for submission in self.data:
group_str = f"{submission['submissiontype']}-{submission['submitter_plate_id']}-{submission['submitted_date']}"

12
src/submissions/frontend/widgets/submission_widget.py
View File

@@ -9,7 +9,7 @@ from PyQt6.QtWidgets import (
from PyQt6.QtCore import pyqtSignal, Qt, QSignalBlocker
from .functions import select_open_file, select_save_file
from pathlib import Path
from tools import Report, Result, check_not_nan, main_form_style, report_result, get_application_from_parent
from tools import Report, Alert, check_not_nan, main_form_style, report_result, get_application_from_parent
from backend.validators import PydReagent, PydClientSubmission, PydSample
from backend.db.models import (
ClientLab, SubmissionType, Reagent, ReagentLot,
@@ -116,7 +116,7 @@ class SubmissionFormContainer(QWidget):
if isinstance(fname, bool) or fname is None:
fname = select_open_file(self, file_extension="xlsx")
if not fname:
report.add_result(Result(msg=f"File {fname.__str__()} not found.", status="critical"))
report.add_result(Alert(msg=f"File {fname.__str__()} not found.", status="critical"))
return report
# NOTE: create sheetparser using excel sheet and context from gui
self.clientsubmission_manager = DefaultClientSubmissionManager(parent=self, input_object=fname)
@@ -133,7 +133,7 @@ class SubmissionFormContainer(QWidget):
else:
message = "Submission cancelled."
logger.warning(message)
report.add_result(Result(msg=message, owner=self.__class__.__name__, status="Warning"))
report.add_result(Alert(msg=message, owner=self.__class__.__name__, status="Warning"))
return report
@report_result
@@ -157,7 +157,7 @@ class SubmissionFormContainer(QWidget):
# NOTE: send reagent to db
sqlobj = reagent.to_sql()
sqlobj.save()
report.add_result(Result(owner=__name__, code=0, msg="New reagent created.", status="Information"))
report.add_result(Alert(owner=__name__, code=0, msg="New reagent created.", status="Information"))
return reagent, report
@@ -386,7 +386,7 @@ class SubmissionFormWidget(QWidget):
if reagent is not None:
reagents.append(reagent)
else:
report.add_result(Result(msg="Failed integrity check", status="Critical"))
report.add_result(Alert(msg="Failed integrity check", status="Critical"))
return report
case self.InfoItem():
field, value = widget.parse_form()
@@ -779,7 +779,7 @@ class ClientSubmissionFormWidget(SubmissionFormWidget):
if reagent is not None:
reagents.append(reagent)
else:
report.add_result(Result(msg="Failed integrity check", status="Critical"))
report.add_result(Alert(msg="Failed integrity check", status="Critical"))
return report
case self.InfoItem():
field, value = widget.parse_form()

8
src/submissions/templates/css/styles.css
View File

@@ -104,8 +104,6 @@ div.gallery {
padding: 5px;
}
.plate {
display: inline-grid;
grid-auto-flow: column;
@@ -189,3 +187,9 @@ ul.no-bullets {
display: grid;
grid-auto-flow: column;
}
.disable_section {
pointer-events: none;
opacity: 0.4;
}

2
src/submissions/templates/details.html
View File

@@ -25,7 +25,7 @@
{% block script %}
{% if not child %}
{% for j in js%}
{% for j in js %}
<script>
{{ j }}

13
src/submissions/tools/__init__.py
View File

@@ -458,7 +458,6 @@ def render_details_template(template_name: str, css_in: List[str] | str = [], js
js_in = ["details"] + js_in
js_in = [html_folder.joinpath("js", f"{j}.js") for j in js_in]
template = env.get_template(f"{template_name}.html")
# template_path = Path(template.environment.loader.__getattribute__("searchpath")[0])
css_out = []
for css in css_in:
with open(css, "r") as f:
@@ -645,7 +644,7 @@ def get_application_from_parent(widget):
return widget
class Result(BaseModel, arbitrary_types_allowed=True):
class Alert(BaseModel, arbitrary_types_allowed=True):
owner: str = Field(default="", validate_default=True)
code: int = Field(default=0)
msg: str | Exception
@@ -704,7 +703,7 @@ class Result(BaseModel, arbitrary_types_allowed=True):
class Report(BaseModel):
results: List[Result] = Field(default=[])
results: List[Alert] = Field(default=[])
def __repr__(self):
return f"<Report(result_count:{len(self.results)})>"
@@ -717,10 +716,10 @@ class Report(BaseModel):
Takes a result object or all results in another report and adds them to this one.
Args:
result (Result | Report | None): Results to be added.
result (Alert | Report | None): Results to be added.
"""
match result:
case Result():
case Alert():
logger.info(f"Adding {result} to results.")
try:
self.results.append(result)
@@ -853,7 +852,7 @@ def check_authorization(func):
logger.error(error_msg)
report = Report()
report.add_result(
Result(owner=func.__str__(), code=1, msg=error_msg, status="warning"))
Alert(owner=func.__str__(), code=1, msg=error_msg, status="warning"))
return report, kwargs
return wrapper
@@ -877,7 +876,7 @@ def under_development(func):
logger.error(error_msg)
report = Report()
report.add_result(
Result(owner=func.__str__(), code=1, msg=error_msg,
Alert(owner=func.__str__(), code=1, msg=error_msg,
status="warning"))
return report
return wrapper