From 39d20bbc226cac32443f60399d3f8338b5ccfbe7 Mon Sep 17 00:00:00 2001 From: lwark Date: Tue, 23 Sep 2025 08:57:40 -0500 Subject: [PATCH] Qubit results parsing complete. --- CHANGELOG.md | 7 ++-- TODO.md | 4 +-- src/submissions/backend/db/models/__init__.py | 4 +-- .../backend/db/models/procedures.py | 10 ++++-- .../backend/db/models/submissions.py | 13 ++++++- .../backend/excel/parsers/__init__.py | 26 +++++++++++--- .../excel/parsers/results_parsers/__init__.py | 36 ++++++++++++++----- .../backend/managers/results/__init__.py | 13 ++++--- src/submissions/backend/validators/pydant.py | 8 ++--- src/submissions/frontend/widgets/info_tab.py | 4 +-- .../frontend/widgets/submission_table.py | 4 +-- .../frontend/widgets/submission_widget.py | 12 +++---- src/submissions/templates/css/styles.css | 10 ++++-- src/submissions/templates/details.html | 2 +- src/submissions/tools/__init__.py | 13 ++++--- 15 files changed, 113 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40fe4c7..50f6ef0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,11 @@ # 202509.03 - Sortable headers in treeview. +- Added gitea remote. # 202509.02 -- First Useable updated version. +- First usable updated version. # 202504.04 @@ -12,7 +13,7 @@ # 202504.03 -- Split Concentration controls on the chart so they are individually selectable. +- Split Concentration controls on the chart, so they are individually selectable. # 202504.02 @@ -315,7 +316,7 @@ ## 202307.03 -- Auto-filling of some empty cells in Excel file. +- Autofilling of some empty cells in Excel file. - Better pydantic validations of missing data. ## 202307.02 diff --git a/TODO.md b/TODO.md index 817b4f3..88a673b 100644 --- a/TODO.md +++ b/TODO.md @@ -1,5 +1,5 @@ -- [ ] Add in database objects for rsl_run (submission -> run), procedure (run -> procedure), many more things will likely be associated with procedure. -- [ ] Add in database object for client submission. +- [x] Add in database objects for rsl_run (submission -> run), procedure (run -> procedure), many more things will likely be associated with procedure. +- [x] Add in database object for client submission. - [ ] Add arbitrary pipette addition to equipment UI. - [ ] transfer details template rendering fully into sql objects - [x] Add in connecting links for tips. diff --git a/src/submissions/backend/db/models/__init__.py b/src/submissions/backend/db/models/__init__.py index d669a67..2c61ed4 100644 --- a/src/submissions/backend/db/models/__init__.py +++ b/src/submissions/backend/db/models/__init__.py @@ -18,7 +18,7 @@ from sqlalchemy.exc import ArgumentError from typing import Any, List, ClassVar from pathlib import Path from sqlalchemy.orm.relationships import _RelationshipDeclared -from tools import report_result, list_sort_dict, jinja_template_loading, Report, Result, ctx +from tools import report_result, list_sort_dict, jinja_template_loading, Report, Alert, ctx # NOTE: Load testing environment if 'pytest' in sys.modules: @@ -364,7 +364,7 @@ class BaseClass(Base): logger.error(f"Error message: {type(e)}") logger.error(pformat(self.__dict__)) self.__database_session__.rollback() - report.add_result(Result(msg=e, status="Critical")) + report.add_result(Alert(msg=e, status="Critical")) return report @property diff --git a/src/submissions/backend/db/models/procedures.py b/src/submissions/backend/db/models/procedures.py index be1f7c6..a28ed44 100644 --- a/src/submissions/backend/db/models/procedures.py +++ b/src/submissions/backend/db/models/procedures.py @@ -4,13 +4,14 @@ All kittype and reagent related models from __future__ import annotations import zipfile, logging, re, numpy as np from operator import itemgetter +from pathlib import Path from pprint import pformat from sqlalchemy import Column, String, TIMESTAMP, JSON, INTEGER, ForeignKey, Interval, Table, FLOAT, func from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import relationship, validates, Query, declared_attr from sqlalchemy.ext.associationproxy import association_proxy from datetime import date, datetime, timedelta -from tools import check_authorization, setup_lookup, Report, Result, check_regex_match, timezone, \ +from tools import check_authorization, setup_lookup, Report, Alert, check_regex_match, timezone, \ jinja_template_loading, flatten_list from typing import List, Literal, Generator, Any, Tuple, TYPE_CHECKING from . import BaseClass, ClientLab, LogMixin @@ -926,10 +927,13 @@ class Procedure(BaseClass): logger.info(f"Add Results! {resultstype_name}") from backend.managers import results results_manager = getattr(results, f"{resultstype_name}Manager") - rs = results_manager(procedure=self, parent=obj) + rs = results_manager(procedure=self, parent=obj, fname=Path("C:\\Users\lwark\Documents\Submission_Forms\QubitData_18-09-2025_13-43-53.csv")) procedure = rs.procedure_to_pydantic() samples = rs.samples_to_pydantic() - procedure_sql = procedure.to_sql() + if procedure: + procedure_sql = procedure.to_sql() + else: + return procedure_sql.save() for sample in samples: sample_sql = sample.to_sql() diff --git a/src/submissions/backend/db/models/submissions.py b/src/submissions/backend/db/models/submissions.py index fca3713..a491b23 100644 --- a/src/submissions/backend/db/models/submissions.py +++ b/src/submissions/backend/db/models/submissions.py @@ -18,7 +18,8 @@ from sqlalchemy.orm.attributes import flag_modified from sqlalchemy.ext.associationproxy import association_proxy from sqlalchemy.exc import OperationalError as AlcOperationalError, IntegrityError as AlcIntegrityError, StatementError from sqlite3 import OperationalError as SQLOperationalError, IntegrityError as SQLIntegrityError -from tools import setup_lookup, jinja_template_loading, create_holidays_for_year, check_dictionary_inclusion_equality, is_power_user +from tools import (setup_lookup, jinja_template_loading, create_holidays_for_year, + check_dictionary_inclusion_equality, is_power_user, row_map) from datetime import datetime, date from typing import List, Literal, Generator, TYPE_CHECKING from pathlib import Path @@ -1865,6 +1866,16 @@ class ProcedureSampleAssociation(BaseClass): results = relationship("Results", back_populates="sampleprocedureassociation") #: associated results + @property + def well(self): + if self.row > 0: + if self.column > 0: + return f"{row_map[self.row]}{self.column}" + else: + return self.row + else: + return None + @classmethod def query(cls, sample: Sample | str | None = None, procedure: Procedure | str | None = None, limit: int = 0, **kwargs): diff --git a/src/submissions/backend/excel/parsers/__init__.py b/src/submissions/backend/excel/parsers/__init__.py index 48817a8..15eac59 100644 --- a/src/submissions/backend/excel/parsers/__init__.py +++ b/src/submissions/backend/excel/parsers/__init__.py @@ -2,11 +2,13 @@ Default Parser archetypes. """ from __future__ import annotations -import logging, re +import logging, re, csv from pathlib import Path +from pprint import pformat from typing import Generator, TYPE_CHECKING from openpyxl.cell import MergedCell from openpyxl.reader.excel import load_workbook +from openpyxl.workbook import Workbook from pandas import DataFrame from backend.validators import pydant if TYPE_CHECKING: @@ -44,6 +46,8 @@ class DefaultParser(object): **kwargs (): """ logger.info(f"\n\nHello from {self.__class__.__name__}\n\n") + if isinstance(filepath, str): + filepath = Path(filepath) self.filepath = filepath self.proceduretype = proceduretype try: @@ -58,13 +62,27 @@ class DefaultParser(object): self.sheet = sheet if not start_row: start_row = self.__class__.start_row - self.workbook = load_workbook(self.filepath, data_only=True) - self.worksheet = self.workbook[self.sheet] + if self.filepath.suffix == ".xslx": + self.workbook = load_workbook(self.filepath, data_only=True) + self.worksheet = self.workbook[self.sheet] + elif self.filepath.suffix == ".csv": + self.workbook, self.worksheet = self.csv2xlsx(self.filepath) self.start_row = self.delineate_start_row(start_row=start_row) self.end_row = self.delineate_end_row(start_row=self.start_row) + @classmethod + def csv2xlsx(cls, filepath): + wb = Workbook() + ws = wb.active + with open(filepath, "r") as f: + reader = csv.reader(f, delimiter=",") + for row in reader: + ws.append(row) + return wb, ws + def to_pydantic(self): data = self.parsed_info + logger.debug(f"Data for {self.__class__.__name__}: {pformat(data)}") data['filepath'] = self.filepath return self._pyd_object(**data) @@ -85,7 +103,7 @@ class DefaultParser(object): for iii, row in enumerate(self.worksheet.iter_rows(min_row=start_row), start=start_row): if all([item.value is None for item in row]): return iii - return self.worksheet.max_row + return self.worksheet.max_row + 1 class DefaultKEYVALUEParser(DefaultParser): diff --git a/src/submissions/backend/excel/parsers/results_parsers/__init__.py b/src/submissions/backend/excel/parsers/results_parsers/__init__.py index 3f839d5..38a3a34 100644 --- a/src/submissions/backend/excel/parsers/results_parsers/__init__.py +++ b/src/submissions/backend/excel/parsers/results_parsers/__init__.py @@ -12,12 +12,22 @@ logger = logging.getLogger(f"submissions.{__name__}") class DefaultResultsInfoParser(DefaultKEYVALUEParser): pyd_name = "PydResults" - def __init__(self, filepath: Path | str, proceduretype: "ProcedureType" | None = None, - results_type: str | None = "PCR", *args, **kwargs): + def __init__(self, filepath: Path | str, results_type: str, proceduretype: "ProcedureType" | None = None, + *args, **kwargs): if results_type: self.results_type = results_type - sheet = proceduretype.allowed_result_methods[results_type]['info']['sheet'] - start_row = proceduretype.allowed_result_methods[results_type]['info']['start_row'] + try: + sheet = proceduretype.allowed_result_methods[results_type]['info']['sheet'] + except KeyError: + sheet = 1 + if "start_row" not in kwargs: + try: + start_row = proceduretype.allowed_result_methods[results_type]['info']['start_row'] + except KeyError: + start_row = 1 + else: + start_row = kwargs.pop('start_row') + # start_row = proceduretype.allowed_result_methods[results_type]['info']['start_row'] super().__init__(filepath=filepath, proceduretype=proceduretype, sheet=sheet, start_row=start_row, *args, **kwargs) @@ -25,14 +35,24 @@ class DefaultResultsInfoParser(DefaultKEYVALUEParser): class DefaultResultsSampleParser(DefaultTABLEParser): pyd_name = "PydResults" - def __init__(self, filepath: Path | str, proceduretype: "ProcedureType" | None = None, - results_type: str | None = "PCR", *args, **kwargs): + def __init__(self, filepath: Path | str, results_type: str, proceduretype: "ProcedureType" | None = None, + *args, **kwargs): if results_type: self.results_type = results_type - sheet = proceduretype.allowed_result_methods[results_type]['sample']['sheet'] - start_row = proceduretype.allowed_result_methods[results_type]['sample']['start_row'] + try: + sheet = proceduretype.allowed_result_methods[results_type]['sample']['sheet'] + except KeyError: + sheet = 1 + if "start_row" not in kwargs: + try: + start_row = proceduretype.allowed_result_methods[results_type]['sample']['start_row'] + except KeyError: + start_row = 1 + else: + start_row = kwargs.pop('start_row') super().__init__(filepath=filepath, proceduretype=proceduretype, sheet=sheet, start_row=start_row, *args, **kwargs) from .pcr_results_parser import PCRInfoParser, PCRSampleParser +from .qubit_results_parser import QubitInfoParser, QubitSampleParser diff --git a/src/submissions/backend/managers/results/__init__.py b/src/submissions/backend/managers/results/__init__.py index 46a7ca6..9524649 100644 --- a/src/submissions/backend/managers/results/__init__.py +++ b/src/submissions/backend/managers/results/__init__.py @@ -17,16 +17,20 @@ logger = logging.getLogger(f"submission.{__name__}") class DefaultResultsManager(DefaultManager): - def __init__(self, procedure: Procedure, parent, fname: Path | str | None = None): + def __init__(self, procedure: Procedure, parent, fname: Path | str | None = None, extension: str|None="xlsx"): self.procedure = procedure if not fname: - self.fname = select_open_file(file_extension="xlsx", obj=get_application_from_parent(parent)) + fname = select_open_file(file_extension=extension, obj=get_application_from_parent(parent)) elif isinstance(fname, str): - self.fname = Path(fname) + fname = Path(fname) + self.fname = fname + def procedure_to_pydantic(self) -> PydResults: + logger.debug(f"Info parser: {self.info_parser}") info = self.info_parser.to_pydantic() - info.parent = self.procedure + if info: + info.parent = self.procedure return info def samples_to_pydantic(self) -> List[PydResults]: @@ -34,3 +38,4 @@ class DefaultResultsManager(DefaultManager): return sample from .pcr_results_manager import PCRManager +from .qubit_results_manager import QubitManager diff --git a/src/submissions/backend/validators/pydant.py b/src/submissions/backend/validators/pydant.py index 8dc6d93..5473add 100644 --- a/src/submissions/backend/validators/pydant.py +++ b/src/submissions/backend/validators/pydant.py @@ -12,7 +12,7 @@ from typing import List, Tuple, Literal, Generator from types import GeneratorType from . import RSLNamer from pathlib import Path -from tools import check_not_nan, convert_nans_to_nones, Report, Result, timezone, sort_dict_by_list, row_keys, flatten_list +from tools import check_not_nan, convert_nans_to_nones, Report, Alert, timezone, sort_dict_by_list, row_keys, flatten_list from backend.db import models from backend.db.models import * from sqlalchemy.orm.properties import ColumnProperty @@ -1397,14 +1397,14 @@ class PydRun(PydBaseClass): #, extra='allow'): Converts this instance into a backend.db.models.procedure.BasicRun instance Returns: - Tuple[BasicRun, Result]: BasicRun instance, result object + Tuple[BasicRun, Alert]: BasicRun instance, result object """ report = Report() dicto = self.improved_dict() instance, result = Run.query_or_create(submissiontype=self.submission_type['value'], rsl_plate_number=self.rsl_plate_number['value']) if instance is None: - report.add_result(Result(msg="Overwrite Cancelled.")) + report.add_result(Alert(msg="Overwrite Cancelled.")) return None, report report.add_result(result) self.handle_duplicate_samples() @@ -1585,7 +1585,7 @@ class PydRun(PydBaseClass): #, extra='allow'): expired.append(f"{reagent.role}, {reagent.lot}: {reagent.expiry.date()} + {role_eol.days}") if expired: output = '\n'.join(expired) - result = Result(status="Warning", + result = Alert(status="Warning", msg=f"The following reagents are expired:\n\n{output}" ) report.add_result(result) diff --git a/src/submissions/frontend/widgets/info_tab.py b/src/submissions/frontend/widgets/info_tab.py index 49765ca..8e1dd1b 100644 --- a/src/submissions/frontend/widgets/info_tab.py +++ b/src/submissions/frontend/widgets/info_tab.py @@ -5,7 +5,7 @@ from datetime import date from PyQt6.QtCore import QSignalBlocker from PyQt6.QtWebEngineWidgets import QWebEngineView from PyQt6.QtWidgets import QWidget, QGridLayout -from tools import Report, report_result, Result +from tools import Report, report_result, Alert from .misc import StartEndDatePicker from .functions import select_save_file, save_pdf import logging @@ -42,7 +42,7 @@ class InfoPane(QWidget): with QSignalBlocker(self.datepicker.start_date) as blocker: self.datepicker.start_date.setDate(lastmonth) self.update_data() - report.add_result(Result(owner=self.__str__(), msg=msg, status="Warning")) + report.add_result(Alert(owner=self.__str__(), msg=msg, status="Warning")) return report @classmethod diff --git a/src/submissions/frontend/widgets/submission_table.py b/src/submissions/frontend/widgets/submission_table.py index 8ff7f08..89dd66d 100644 --- a/src/submissions/frontend/widgets/submission_table.py +++ b/src/submissions/frontend/widgets/submission_table.py @@ -109,9 +109,7 @@ class SubmissionsTree(QTreeView): sets data in model """ self.clear() - self.data = [item.to_dict(full_data=True) for item in - # self.data = [item.details_dict() for item in - ClientSubmission.query(chronologic=True, page=page, page_size=page_size)] + self.data = [item.to_dict(full_data=True) for item in ClientSubmission.query(chronologic=True, page=page, page_size=page_size)] root = self.model.invisibleRootItem() for submission in self.data: group_str = f"{submission['submissiontype']}-{submission['submitter_plate_id']}-{submission['submitted_date']}" diff --git a/src/submissions/frontend/widgets/submission_widget.py b/src/submissions/frontend/widgets/submission_widget.py index 5478f74..d9a12ee 100644 --- a/src/submissions/frontend/widgets/submission_widget.py +++ b/src/submissions/frontend/widgets/submission_widget.py @@ -9,7 +9,7 @@ from PyQt6.QtWidgets import ( from PyQt6.QtCore import pyqtSignal, Qt, QSignalBlocker from .functions import select_open_file, select_save_file from pathlib import Path -from tools import Report, Result, check_not_nan, main_form_style, report_result, get_application_from_parent +from tools import Report, Alert, check_not_nan, main_form_style, report_result, get_application_from_parent from backend.validators import PydReagent, PydClientSubmission, PydSample from backend.db.models import ( ClientLab, SubmissionType, Reagent, ReagentLot, @@ -116,7 +116,7 @@ class SubmissionFormContainer(QWidget): if isinstance(fname, bool) or fname is None: fname = select_open_file(self, file_extension="xlsx") if not fname: - report.add_result(Result(msg=f"File {fname.__str__()} not found.", status="critical")) + report.add_result(Alert(msg=f"File {fname.__str__()} not found.", status="critical")) return report # NOTE: create sheetparser using excel sheet and context from gui self.clientsubmission_manager = DefaultClientSubmissionManager(parent=self, input_object=fname) @@ -133,7 +133,7 @@ class SubmissionFormContainer(QWidget): else: message = "Submission cancelled." logger.warning(message) - report.add_result(Result(msg=message, owner=self.__class__.__name__, status="Warning")) + report.add_result(Alert(msg=message, owner=self.__class__.__name__, status="Warning")) return report @report_result @@ -157,7 +157,7 @@ class SubmissionFormContainer(QWidget): # NOTE: send reagent to db sqlobj = reagent.to_sql() sqlobj.save() - report.add_result(Result(owner=__name__, code=0, msg="New reagent created.", status="Information")) + report.add_result(Alert(owner=__name__, code=0, msg="New reagent created.", status="Information")) return reagent, report @@ -386,7 +386,7 @@ class SubmissionFormWidget(QWidget): if reagent is not None: reagents.append(reagent) else: - report.add_result(Result(msg="Failed integrity check", status="Critical")) + report.add_result(Alert(msg="Failed integrity check", status="Critical")) return report case self.InfoItem(): field, value = widget.parse_form() @@ -779,7 +779,7 @@ class ClientSubmissionFormWidget(SubmissionFormWidget): if reagent is not None: reagents.append(reagent) else: - report.add_result(Result(msg="Failed integrity check", status="Critical")) + report.add_result(Alert(msg="Failed integrity check", status="Critical")) return report case self.InfoItem(): field, value = widget.parse_form() diff --git a/src/submissions/templates/css/styles.css b/src/submissions/templates/css/styles.css index bfcb3b1..c17e50c 100644 --- a/src/submissions/templates/css/styles.css +++ b/src/submissions/templates/css/styles.css @@ -104,8 +104,6 @@ div.gallery { padding: 5px; } - - .plate { display: inline-grid; grid-auto-flow: column; @@ -188,4 +186,10 @@ ul.no-bullets { .grid-container { display: grid; grid-auto-flow: column; - } \ No newline at end of file + } + +.disable_section { + pointer-events: none; + opacity: 0.4; +} + diff --git a/src/submissions/templates/details.html b/src/submissions/templates/details.html index 795ef9e..e32a188 100644 --- a/src/submissions/templates/details.html +++ b/src/submissions/templates/details.html @@ -25,7 +25,7 @@ {% block script %} {% if not child %} -{% for j in js%} +{% for j in js %}