Sanity checking.

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 202405.03
+
+- Settings can now pull values from the db.
+- Improved generic and WW specific PCR parsers.
+- Various bug fixes.
+
 ## 202405.01
 
 - New Excel writers

alembic.ini

@@ -55,10 +55,10 @@ version_path_separator = os  # Use os.pathsep. Default configuration used for ne
 # are written from script.py.mako
 # output_encoding = utf-8
 
-sqlalchemy.url = sqlite:///L:\Robotics Laboratory Support\Submissions\submissions.db
+; sqlalchemy.url = sqlite:///L:\Robotics Laboratory Support\Submissions\submissions.db
-;sqlalchemy.url = sqlite:///C:\Users\lwark\Documents\Archives\Submissions_app_backups\DB_backups\submissions-demo.db
+; sqlalchemy.url = sqlite:///C:\Users\lwark\Documents\Archives\Submissions_app_backups\DB_backups\submissions-demo.db
 ;sqlalchemy.url = sqlite:///C:\Users\lwark\Documents\Archives\Submissions_app_backups\DB_backups\submissions-prototypes.db
-;sqlalchemy.url = sqlite:///C:\Users\lwark\Documents\python\submissions\mytests\test_assets\submissions-test.db
+sqlalchemy.url = sqlite:///C:\Users\lwark\Documents\python\submissions\mytests\test_assets\submissions-test.db
 
 
 [post_write_hooks]

src/submissions/backend/db/models/__init__.py

@@ -3,6 +3,8 @@ Contains all models for sqlalchemy
 '''
 from __future__ import annotations
 import sys, logging
+
+from sqlalchemy import Column, INTEGER, String, JSON
 from sqlalchemy.orm import DeclarativeMeta, declarative_base, Query, Session
 from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.exc import ArgumentError
@@ -11,8 +13,6 @@ from pathlib import Path
 
 # Load testing environment
 if 'pytest' in sys.modules:
-    from pathlib import Path
-
     sys.path.append(Path(__file__).parents[4].absolute().joinpath("tests").__str__())
 
 Base: DeclarativeMeta = declarative_base()
@@ -46,7 +46,7 @@ class BaseClass(Base):
         Returns:
             Session: DB session from ctx settings.
         """
-        if not 'pytest' in sys.modules:
+        if 'pytest' not in sys.modules:
             from tools import ctx
         else:
             from test_settings import ctx
@@ -60,7 +60,7 @@ class BaseClass(Base):
         Returns:
             Path: Location of the Submissions directory in Settings object
         """
-        if not 'pytest' in sys.modules:
+        if 'pytest' not in sys.modules:
             from tools import ctx
         else:
             from test_settings import ctx
@@ -74,7 +74,7 @@ class BaseClass(Base):
         Returns:
             Path: Location of the Submissions backup directory in Settings object
         """
-        if not 'pytest' in sys.modules:
+        if 'pytest' not in sys.modules:
             from tools import ctx
         else:
             from test_settings import ctx
@@ -104,8 +104,9 @@ class BaseClass(Base):
         Execute sqlalchemy query.
 
         Args:
-            query (Query): input query object
+            model (Any, optional): model to be queried. Defaults to None
-            limit (int): Maximum number of results. (0 = all)
+            query (Query, optional): input query object. Defaults to None
+            limit (int): Maximum number of results. (0 = all). Defaults to 0
 
         Returns:
             Any | List[Any]: Single result if limit = 1 or List if other.
@@ -153,6 +154,19 @@ class BaseClass(Base):
             self.__database_session__.rollback()
 
 
+class ConfigItem(BaseClass):
+    id = Column(INTEGER, primary_key=True)
+    key = Column(String(32))
+    value = Column(JSON)
+
+    def __repr__(self):
+        return f"ConfigItem({self.key} : {self.value})"
+
+    @classmethod
+    def get_config_items(cls):
+        return cls.__database_session__.query(cls).all()
+
+
 from .controls import *
 # import order must go: orgs, kit, subs due to circular import issues
 from .organizations import *

src/submissions/backend/db/models/controls.py

@@ -175,19 +175,19 @@ class Control(BaseClass):
         output = []
         # logger.debug("load json string for mode (i.e. contains, matches, kraken2)")
         try:
-            # data = json.loads(getattr(self, mode))
             data = self.__getattribute__(mode)
         except TypeError:
             data = {}
         logger.debug(f"Length of data: {len(data)}")
         # logger.debug("dict keys are genera of bacteria, e.g. 'Streptococcus'")
         for genus in data:
-            _dict = {}
+            _dict = dict(
-            _dict['name'] = self.name
+                name=self.name,
-            _dict['submitted_date'] = self.submitted_date
+                submitted_date=self.submitted_date,
-            _dict['genus'] = genus
+                genus=genus,
+                target='Target' if genus.strip("*") in self.controltype.targets else "Off-target"
+            )
             # logger.debug("get Target or Off-target of genus")
-            _dict['target'] = 'Target' if genus.strip("*") in self.controltype.targets else "Off-target"
             # logger.debug("set 'contains_hashes', etc for genus")
             for key in data[genus]:
                 _dict[key] = data[genus][key]
@@ -247,13 +247,13 @@ class Control(BaseClass):
             case _:
                 pass
         # by date range
-        if start_date != None and end_date == None:
+        if start_date is not None and end_date is None:
             logger.warning(f"Start date with no end date, using today.")
             end_date = date.today()
-        if end_date != None and start_date == None:
+        if end_date is not None and start_date is None:
             logger.warning(f"End date with no start date, using Jan 1, 2023")
             start_date = date(2023, 1, 1)
-        if start_date != None:
+        if start_date is not None:
             match start_date:
                 case date():
                     # logger.debug(f"Lookup control by start date({start_date})")

src/submissions/tools.py

@@ -2,6 +2,8 @@
 Contains miscellaenous functions used by both frontend and backend.
 '''
 from __future__ import annotations
+
+import json
 from pathlib import Path
 import numpy as np
 import logging, re, yaml, sys, os, stat, platform, getpass, inspect, csv
@@ -10,7 +12,7 @@ from jinja2 import Environment, FileSystemLoader
 from logging import handlers
 from pathlib import Path
 from sqlalchemy.orm import Session
-from sqlalchemy import create_engine
+from sqlalchemy import create_engine, text
 from pydantic import field_validator, BaseModel, Field
 from pydantic_settings import BaseSettings, SettingsConfigDict
 from typing import Any, Tuple, Literal, List
@@ -137,7 +139,7 @@ def get_first_blank_df_row(df:pd.DataFrame) -> int:
 
 # Settings
 
-class Settings(BaseSettings):
+class Settings(BaseSettings, extra="allow"):
     """
     Pydantic model to hold settings
 
@@ -147,21 +149,26 @@ class Settings(BaseSettings):
     """    
     directory_path: Path
     database_path: Path|str|None = None
-    backup_path: Path
+    backup_path: Path|str|None = None
-    super_users: list|None = None
+    # super_users: list|None = None
-    power_users: list|None = None
+    # power_users: list|None = None
-    rerun_regex: str
+    # rerun_regex: str
     submission_types: dict|None = None
     database_session: Session|None = None
     package: Any|None = None
 
     model_config = SettingsConfigDict(env_file_encoding='utf-8')
 
-    @field_validator('backup_path')
+    @field_validator('backup_path', mode="before")
     @classmethod
-    def set_backup_path(cls, value):
+    def set_backup_path(cls, value, values):
-        if isinstance(value, str):
+        match value:
+            case str():
                 value = Path(value)
+            case None:
+                value = values.data['directory_path'].joinpath("Database backups")
+        if not value.exists():
+            value.mkdir(parents=True)
         # metadata.backup_path = value
         return value
 
@@ -177,11 +184,14 @@ class Settings(BaseSettings):
         
     @field_validator('database_path', mode="before")
     @classmethod
-    def ensure_database_exists(cls, value):
+    def ensure_database_exists(cls, value, values):
         if value == ":memory:":
             return value
-        if isinstance(value, str):
+        match value:
+            case str():
                 value = Path(value)
+            case None:
+                value = values.data['directory_path'].joinpath("submissions.db")
         if value.exists():
             return value
         else:
@@ -225,6 +235,20 @@ class Settings(BaseSettings):
         if value == None:
             return package
 
+    def __init__(self, *args, **kwargs):
+
+        super().__init__(*args, **kwargs)
+        # self.set_from_db(db_path=kwargs['database_path'])
+
+    def set_from_db(self, db_path:Path):
+        session = Session(create_engine(f"sqlite:///{db_path}"))
+        config_items = session.execute(text("SELECT * FROM _configitem")).all()
+        session.close()
+        config_items = {item[1]:json.loads(item[2]) for item in config_items}
+        for k, v in config_items.items():
+            if not hasattr(self, k):
+                self.__setattr__(k, v)
+
 def get_config(settings_path: Path|str|None=None) -> Settings:
     """
     Get configuration settings from path or default if blank.