t
This commit is contained in:
Remy Moll
@@ -1,12 +1,12 @@
|
||||
## What is happening here?
|
||||
|
||||
This "persistence"-module aims to standardize 2 things:
|
||||
* the creation of a common set of variables that survives a potential (let's face it, likely) crash
|
||||
* advanced logging and analytics
|
||||
|
||||
### Common variables
|
||||
These are saved as a json file and are handled internally as a dict. Each change in the dict triggers a write to the file.
|
||||
|
||||
|
||||
### Logging
|
||||
## What is happening here?
|
||||
|
||||
This "persistence"-module aims to standardize 2 things:
|
||||
* the creation of a common set of variables that survives a potential (let's face it, likely) crash
|
||||
* advanced logging and analytics
|
||||
|
||||
### Common variables
|
||||
These are saved as a json file and are handled internally as a dict. Each change in the dict triggers a write to the file.
|
||||
|
||||
|
||||
### Logging
|
||||
A chunky sqlite-db which periodically gets new entries. From all modules. Ideally this db is then visualized through grafana. WIP
|
||||
@@ -1,71 +1,55 @@
|
||||
from peewee import *
|
||||
# from playhouse.pool import PooledMySQLDatabase
|
||||
from playhouse.shortcuts import ReconnectMixin
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
||||
db = DatabaseProxy()
|
||||
# set the nature of the db at runtime
|
||||
|
||||
class ReconnectDataBase(ReconnectMixin, MySQLDatabase):
|
||||
pass
|
||||
|
||||
|
||||
class DBModel(Model):
|
||||
# specific to the above DB
|
||||
class Meta:
|
||||
database = db
|
||||
|
||||
def save(self):
|
||||
# fail-safe writing of the db-object. Usually threaded because the caller is threaded
|
||||
try:
|
||||
# db.connect()
|
||||
super().save()
|
||||
# db.close()
|
||||
except Exception as e:
|
||||
logger.error("Could not write to db. Dropping content of {}".format(self.__class__.__name__))
|
||||
logger.error(e)
|
||||
# db.atomic().rollback()
|
||||
|
||||
# def get(self, *query, **filters):
|
||||
# try:
|
||||
# return super().get(*query, **filters)
|
||||
# except Exception as e:
|
||||
# logger.error("Error while executing get: {}".format(e))
|
||||
# print(query, filters)
|
||||
|
||||
|
||||
class Metric(DBModel):
|
||||
time = DateTimeField()
|
||||
|
||||
|
||||
### Actual metrics:
|
||||
|
||||
class SensorMetric(Metric):
|
||||
# this is a continuous metric
|
||||
temperature = IntegerField()
|
||||
humidity = IntegerField()
|
||||
luminosity = IntegerField()
|
||||
default = {"temperature": 100, "humidity": 100, "luminosity": 100}
|
||||
|
||||
|
||||
|
||||
class ChatMetric(Metric):
|
||||
read = BooleanField()
|
||||
send = BooleanField()
|
||||
execute = BooleanField()
|
||||
default = {"read": False, "send": False, "execute": False}
|
||||
|
||||
|
||||
class ErrorMetric(Metric):
|
||||
# same as above
|
||||
error = TextField()
|
||||
default = {"error": "SQL connection broke off"}
|
||||
|
||||
|
||||
class List(DBModel):
|
||||
name = CharField(unique=True)
|
||||
content = TextField() # unlimited length, use to serialise list into
|
||||
default = {"content": "SQL connection broke off"}
|
||||
from datetime import datetime
|
||||
from peewee import *
|
||||
import logging
|
||||
import json
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def create_tables(db):
|
||||
db.create_tables([SensorMetric, ChatMetric, ErrorMetric, List])
|
||||
|
||||
db = DatabaseProxy()
|
||||
# set the nature of the db at runtime
|
||||
|
||||
|
||||
class DBModel(Model):
|
||||
# specific to the above DB
|
||||
class Meta:
|
||||
database = db
|
||||
|
||||
class Metric(DBModel):
|
||||
time = DateTimeField(default = datetime.now())
|
||||
|
||||
|
||||
|
||||
### Actual metrics:
|
||||
|
||||
class SensorMetric(Metric):
|
||||
# this is a continuous metric
|
||||
temperature = IntegerField()
|
||||
humidity = IntegerField()
|
||||
luminosity = IntegerField()
|
||||
|
||||
|
||||
class ChatMetric(Metric):
|
||||
read = BooleanField()
|
||||
send = BooleanField()
|
||||
execute = BooleanField()
|
||||
|
||||
|
||||
class ErrorMetric(Metric):
|
||||
# same as above
|
||||
error = TextField()
|
||||
|
||||
|
||||
class List(DBModel):
|
||||
name = CharField(unique=True)
|
||||
content = TextField() # unlimited length, use to serialise list into
|
||||
|
||||
@property
|
||||
def content_as_list(self):
|
||||
return json.loads(self.content)
|
||||
|
||||
def set_content(self, list_content):
|
||||
self.content = json.dumps(list_content)
|
||||
@@ -1,87 +1,87 @@
|
||||
import json
|
||||
import os
|
||||
|
||||
|
||||
class PersistentDict(dict):
|
||||
"""Extended dict that writes its content to a file every time a value is changed"""
|
||||
|
||||
def __init__(self, file_name, *args, **kwargs):
|
||||
"""initialization of the dict and of the required files"""
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
self.path = file_name
|
||||
self.last_action = ""
|
||||
try:
|
||||
self.read_dict()
|
||||
except:
|
||||
with open(self.path, "a") as f:
|
||||
f.write("{}")
|
||||
|
||||
|
||||
## helper - functions
|
||||
def write_dict(self):
|
||||
with open(self.path, "w") as f:
|
||||
json.dump(self, f)
|
||||
self.last_action = "w"
|
||||
|
||||
def read_dict(self):
|
||||
with open(self.path) as f:
|
||||
tmp = dict(json.load(f))
|
||||
for key in tmp:
|
||||
super().__setitem__(key, tmp[key])
|
||||
self.last_action = "r"
|
||||
|
||||
|
||||
## extended dictionary - logic
|
||||
def __setitem__(self, key, value):
|
||||
if self.last_action != "r":
|
||||
self.read_dict()
|
||||
# not sure if the step to read is necessary, but I'll keep it for safety
|
||||
super().__setitem__(key,value)
|
||||
self.write_dict() # writes 'self' to a json file.
|
||||
|
||||
def __getitem__(self, key):
|
||||
if self.last_action != "r":
|
||||
self.read_dict()
|
||||
|
||||
ret_val = super().__getitem__(key)
|
||||
if type(ret_val) != int and type(ret_val) != str:
|
||||
ret_val = create_struct(type(ret_val), key, self, ret_val)
|
||||
|
||||
return ret_val
|
||||
|
||||
def clear(self):
|
||||
super().clear()
|
||||
self.write_dict()
|
||||
return {}
|
||||
|
||||
|
||||
|
||||
def create_struct(struct_type, own_name, parent_name, *args, **kwargs):
|
||||
class HookedStruct(struct_type):
|
||||
|
||||
def __init__(self, own_name, parent_name, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.name = own_name
|
||||
self.parent = parent_name
|
||||
|
||||
def __setitem__(self, *args, **kwargs):
|
||||
super().__setitem__(*args, **kwargs)
|
||||
self.parent.__setitem__(self.name, self)
|
||||
|
||||
def __getitem__(self, *args, **kwargs):
|
||||
ret_val = super().__getitem__(*args, **kwargs)
|
||||
if type(ret_val) != int and type(ret_val) != str:
|
||||
ret_val = create_struct(type(ret_val), args[0], self, ret_val)
|
||||
return ret_val
|
||||
|
||||
def pop(self, *args):
|
||||
retvalue = super().pop(*args)
|
||||
self.parent.__setitem__(self.name, self)
|
||||
return retvalue
|
||||
|
||||
def append(self, *args):
|
||||
super().append(*args)
|
||||
self.parent.__setitem__(self.name, self)
|
||||
|
||||
return HookedStruct(own_name, parent_name, *args, **kwargs)
|
||||
import json
|
||||
import os
|
||||
|
||||
|
||||
class PersistentDict(dict):
|
||||
"""Extended dict that writes its content to a file every time a value is changed"""
|
||||
|
||||
def __init__(self, file_name, *args, **kwargs):
|
||||
"""initialization of the dict and of the required files"""
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
self.path = file_name
|
||||
self.last_action = ""
|
||||
try:
|
||||
self.read_dict()
|
||||
except:
|
||||
with open(self.path, "a") as f:
|
||||
f.write("{}")
|
||||
|
||||
|
||||
## helper - functions
|
||||
def write_dict(self):
|
||||
with open(self.path, "w") as f:
|
||||
json.dump(self, f)
|
||||
self.last_action = "w"
|
||||
|
||||
def read_dict(self):
|
||||
with open(self.path) as f:
|
||||
tmp = dict(json.load(f))
|
||||
for key in tmp:
|
||||
super().__setitem__(key, tmp[key])
|
||||
self.last_action = "r"
|
||||
|
||||
|
||||
## extended dictionary - logic
|
||||
def __setitem__(self, key, value):
|
||||
if self.last_action != "r":
|
||||
self.read_dict()
|
||||
# not sure if the step to read is necessary, but I'll keep it for safety
|
||||
super().__setitem__(key,value)
|
||||
self.write_dict() # writes 'self' to a json file.
|
||||
|
||||
def __getitem__(self, key):
|
||||
if self.last_action != "r":
|
||||
self.read_dict()
|
||||
|
||||
ret_val = super().__getitem__(key)
|
||||
if type(ret_val) != int and type(ret_val) != str:
|
||||
ret_val = create_struct(type(ret_val), key, self, ret_val)
|
||||
|
||||
return ret_val
|
||||
|
||||
def clear(self):
|
||||
super().clear()
|
||||
self.write_dict()
|
||||
return {}
|
||||
|
||||
|
||||
|
||||
def create_struct(struct_type, own_name, parent_name, *args, **kwargs):
|
||||
class HookedStruct(struct_type):
|
||||
|
||||
def __init__(self, own_name, parent_name, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.name = own_name
|
||||
self.parent = parent_name
|
||||
|
||||
def __setitem__(self, *args, **kwargs):
|
||||
super().__setitem__(*args, **kwargs)
|
||||
self.parent.__setitem__(self.name, self)
|
||||
|
||||
def __getitem__(self, *args, **kwargs):
|
||||
ret_val = super().__getitem__(*args, **kwargs)
|
||||
if type(ret_val) != int and type(ret_val) != str:
|
||||
ret_val = create_struct(type(ret_val), args[0], self, ret_val)
|
||||
return ret_val
|
||||
|
||||
def pop(self, *args):
|
||||
retvalue = super().pop(*args)
|
||||
self.parent.__setitem__(self.name, self)
|
||||
return retvalue
|
||||
|
||||
def append(self, *args):
|
||||
super().append(*args)
|
||||
self.parent.__setitem__(self.name, self)
|
||||
|
||||
return HookedStruct(own_name, parent_name, *args, **kwargs)
|
||||
|
||||
@@ -1,116 +0,0 @@
|
||||
from . import models
|
||||
from peewee import *
|
||||
# from playhouse.pool import PooledMySQLDatabase
|
||||
from playhouse.shortcuts import ReconnectMixin
|
||||
import inspect
|
||||
|
||||
from . import keys
|
||||
dbk = keys.db_keys
|
||||
|
||||
|
||||
class ReconnectDataBase(ReconnectMixin, MySQLDatabase):
|
||||
pass
|
||||
|
||||
|
||||
|
||||
class DBConnector:
|
||||
"""Create a connection to a remote database and log some quantities that will be visualized otherwhere"""
|
||||
def __init__(self):
|
||||
self.db = models.db
|
||||
|
||||
self.sensors = models.SensorMetric
|
||||
self.chats = models.ChatMetric
|
||||
self.errors = models.ErrorMetric
|
||||
self.lists = models.List
|
||||
|
||||
self.create_tables()
|
||||
|
||||
def create_tables(self):
|
||||
self.db.create_tables([self.sensors, self.chats, self.errors, self.lists])
|
||||
|
||||
|
||||
|
||||
class DataBaseConnector:
|
||||
def __init__(self) -> None:
|
||||
self.db_object = models.ReconnectDataBase(
|
||||
dbk["name"],
|
||||
user=dbk["username"],
|
||||
password=dbk["password"],
|
||||
host=dbk["url"],
|
||||
port=dbk["port"],
|
||||
autorollback=True
|
||||
)
|
||||
models.db.initialize(self.db_object)
|
||||
|
||||
# self.sensors = models.SensorMetric
|
||||
# self.chats = models.ChatMetric
|
||||
# self.errors = models.ErrorMetric
|
||||
# self.lists = models.List
|
||||
## Set as property methods instead
|
||||
|
||||
self.db_object.create_tables([self.sensors, self.chats, self.errors, self.lists])
|
||||
|
||||
@property
|
||||
def sensors(self):
|
||||
self.connect_first()
|
||||
return models.SensorMetric
|
||||
|
||||
@property
|
||||
def chats(self):
|
||||
self.connect_first()
|
||||
return models.ChatMetric
|
||||
|
||||
@property
|
||||
def errors(self):
|
||||
self.connect_first()
|
||||
return models.ErrorMetric
|
||||
|
||||
@property
|
||||
def lists(self):
|
||||
self.connect_first()
|
||||
return models.List
|
||||
|
||||
def connect_first(self):
|
||||
# if self.db_object.is_closed():
|
||||
# self.db_object.connect()
|
||||
self.db_object.connect(reuse_if_open=True)
|
||||
|
||||
|
||||
|
||||
|
||||
# def auto_reconnect(func, *args, **kwargs):
|
||||
# return func
|
||||
|
||||
# def classwide_decorator(decorator):
|
||||
# def decorate(cls):
|
||||
# for attr in inspect.getmembers(cls, inspect.ismethod): # there's propably a better way to do this
|
||||
# # TODO: filter init
|
||||
# print(attr)
|
||||
# if callable(getattr(cls, attr)):
|
||||
# setattr(cls, attr, decorator(getattr(cls, attr)))
|
||||
# return cls
|
||||
# return decorate
|
||||
|
||||
# # apply auto_reconnect to every method so that every method first checks the db connection and reconnects if necessary
|
||||
# @classwide_decorator(auto_reconnect)
|
||||
# class DataBaseConnector(ReconnectMixin, MySQLDatabase):
|
||||
# def __init__(self, *args, **kwargs):
|
||||
# super().__init__(
|
||||
# dbk["name"],
|
||||
# user=dbk["username"],
|
||||
# password=dbk["password"],
|
||||
# host=dbk["url"],
|
||||
# port=dbk["port"],
|
||||
# autorollback=True,
|
||||
# *args, **kwargs)
|
||||
|
||||
# models.db.initialize(self)
|
||||
# self.sensors = models.SensorMetric
|
||||
# self.chats = models.ChatMetric
|
||||
# self.errors = models.ErrorMetric
|
||||
# self.lists = models.List
|
||||
|
||||
# self.create_tables([self.sensors, self.chats, self.errors, self.lists])
|
||||
|
||||
# def m1(self): pass
|
||||
# def m2(self, x): pass
|
||||
Reference in New Issue
Block a user