Dockerized and fixed errors

app/persistence/README.md

@@ -0,0 +1,12 @@
+## What is happening here?
+
+This "persistence"-module aims to standardize 2 things:
+* the creation of a common set of variables that survives a potential (let's face it, likely) crash
+* advanced logging and analytics
+
+### Common variables
+These are saved as a json file and are handled internally as a dict. Each change in the dict triggers a write to the file.
+
+
+### Logging
+A chunky sqlite-db which periodically gets new entries. From all modules. Ideally this db is then visualized through grafana. WIP

app/persistence/__init__.py

@@ -0,0 +1 @@
+#placeholder

app/persistence/database.py

@@ -0,0 +1,98 @@
+from . import models
+from peewee import *
+from playhouse.pool import PooledMySQLDatabase
+
+
+import logging
+logger = logging.getLogger(__name__)
+
+
+import os
+if os.getenv("dockerized", "") == "true":
+    import sys
+    sys.path.append("/keys")
+    import db_keys as keys
+else:
+    from . import keys
+dbk = keys.db_access
+
+
+
+db_connection = PooledMySQLDatabase(
+    dbk["name"],
+    user=dbk["username"],
+    password=dbk["password"],
+    host=dbk["url"],
+    port=dbk["port"],
+    autorollback=True
+)
+
+
+def auto_connect_db(func):
+    def wrapper(*args, **kwargs):
+        #before:
+        db_connection.connect()
+        ret = func(*args, **kwargs)
+        #after:
+        db_connection.close()
+        # also, action is in scope now
+        return ret
+    
+    return wrapper
+
+
+
+class DatabaseUtils:
+    """This object is the only database-related stuff getting exposed to the other modules. It must explicitly handle the connections!"""
+    def __init__(self) -> None:
+        # TODO specify arguments!
+        models.db.initialize(db_connection)
+        models.create_tables(db_connection)
+
+
+    @auto_connect_db
+    def chat_count(self, attribute):
+        if attribute == "read":
+            return models.ChatMetric.select().where(models.ChatMetric.read == True).count()
+        elif attribute == "send":
+            return models.ChatMetric.select().where(models.ChatMetric.send == True).count()
+        elif attribute == "execute":
+            return models.ChatMetric.select().where(models.ChatMetric.execute == True).count()
+        else: # does not exist
+            return -1
+
+    @auto_connect_db
+    def chat_log(self, **kwargs):
+        models.ChatMetric(**kwargs)
+
+    @auto_connect_db
+    def list_get(self, list_name=""):
+        if not list_name: # return all
+            cursor = models.List.select(models.List.name).execute()
+            return [k.name for k in cursor]
+        else:
+            return models.List.get(models.List.name == list_name).content_as_list
+        
+    @auto_connect_db    
+    def list_update(self, list_name, append="", replace=None):
+        if replace != None:
+            models.List.get(models.List.name == list_name).set_content(replace)
+        elif append:
+            l_obj = models.List.get(models.List.name == list_name)
+            l = l_obj.content_as_list
+            l.append(append)
+            l_obj.set_content(l)
+        else:
+            logger.warning("Empty update_list() query was made. Ignoring")
+
+    @auto_connect_db
+    def list_create(self, list_name):
+        models.List(name=list_name).save()
+
+    @auto_connect_db
+    def list_delete(self, list_name):
+        models.List.delete().where(models.List.name == list_name).execute()
+
+    @auto_connect_db
+    def sensor_log(self, **kwargs):
+        models.SensorMetric(**kwargs).save()

app/persistence/local_io.py

@@ -0,0 +1,87 @@
+import json
+import os
+
+
+class PersistentDict(dict):
+    """Extended dict that writes its content to a file every time a value is changed"""
+
+    def __init__(self, file_name, *args, **kwargs):
+        """initialization of the dict and of the required files"""
+        super().__init__(*args, **kwargs)
+
+        self.path = file_name
+        self.last_action = ""
+        try:
+            self.read_dict()
+        except:
+            with open(self.path, "a") as f:
+                f.write("{}")
+        
+
+    ## helper - functions
+    def write_dict(self):
+        with open(self.path, "w") as f:
+            json.dump(self, f)
+        self.last_action = "w"
+
+    def read_dict(self):
+        with open(self.path) as f:
+            tmp = dict(json.load(f))
+            for key in tmp:
+                super().__setitem__(key, tmp[key])
+        self.last_action = "r"
+
+
+    ## extended dictionary - logic
+    def __setitem__(self, key, value):
+        if self.last_action != "r":
+            self.read_dict()
+        # not sure if the step to read is necessary, but I'll keep it for safety
+        super().__setitem__(key,value)
+        self.write_dict() # writes 'self' to a json file.
+
+    def __getitem__(self, key):
+        if self.last_action != "r":
+            self.read_dict()
+        
+        ret_val = super().__getitem__(key)
+        if type(ret_val) != int and type(ret_val) != str:
+            ret_val = create_struct(type(ret_val), key, self, ret_val)
+
+        return ret_val
+    
+    def clear(self):
+        super().clear()
+        self.write_dict()
+        return {}
+
+
+
+def create_struct(struct_type, own_name, parent_name, *args, **kwargs):
+    class HookedStruct(struct_type):
+
+        def __init__(self, own_name, parent_name, *args, **kwargs):
+            super().__init__(*args, **kwargs)
+            self.name = own_name
+            self.parent = parent_name
+        
+        def __setitem__(self, *args, **kwargs):
+            super().__setitem__(*args, **kwargs)
+            self.parent.__setitem__(self.name, self)
+
+        def __getitem__(self, *args, **kwargs):
+            ret_val = super().__getitem__(*args, **kwargs)
+            if type(ret_val) != int and type(ret_val) != str:
+                ret_val = create_struct(type(ret_val), args[0], self, ret_val)
+            return ret_val
+
+        def pop(self, *args):
+            retvalue = super().pop(*args)
+            self.parent.__setitem__(self.name, self)
+            return retvalue
+
+        def append(self, *args):
+            super().append(*args)
+            self.parent.__setitem__(self.name, self)
+
+    return HookedStruct(own_name, parent_name, *args, **kwargs)

app/persistence/models.py

@@ -0,0 +1,56 @@
+from datetime import datetime
+from peewee import *
+import logging
+import json
+
+logger = logging.getLogger(__name__)
+
+
+def create_tables(db):
+    db.create_tables([SensorMetric, ChatMetric, ErrorMetric, List])
+
+db = DatabaseProxy()
+# set the nature of the db at runtime
+
+
+class DBModel(Model):
+    # specific to the above DB
+    class Meta:
+        database = db
+
+class Metric(DBModel):
+    time = DateTimeField(default = datetime.now())
+        
+
+
+### Actual metrics:
+
+class SensorMetric(Metric):
+    # this is a continuous metric
+    temperature = FloatField()
+    humidity = FloatField()
+    luminosity = FloatField()
+
+
+class ChatMetric(Metric):
+    read = BooleanField()
+    send = BooleanField()
+    execute = BooleanField()
+
+
+class ErrorMetric(Metric):
+    # same as above
+    error = TextField()
+
+
+class List(DBModel):
+    name = CharField(unique=True)
+    content = TextField(default="") # unlimited length, use to serialise list into
+
+    @property
+    def content_as_list(self):
+        return json.loads(self.content or '[]')
+    
+    def set_content(self, list_content):
+        self.content = json.dumps(list_content)
+        self.save()