Many bug fixes

app/utils_storage/migrations/migration.001.py

@@ -0,0 +1,67 @@
+from playhouse.migrate import *
+
+
+"""
+This migration assumes that downloads.db kept the exact same structure as before.
+messages.db should drop the table articlemodelreference in favor of a new field article in the thread-table
+Since each thread is constrained to exactly one article this makes the most sense.
+
+This migration assumes that messages.db gets a new field in the table thread:
+id  |   thread_ts | article_id
+
+We now need to migrate from the table articlemodelreference and then delete it.
+"""
+
+
+db = SqliteDatabase("/code/.dev/messages.db")
+migrator = SqliteMigrator(db)
+
+
+article_field = IntegerField(null=True)
+
+
+migrate(
+    migrator.add_column('thread', 'article_id', article_field),
+    # migrator.drop_column('some_table', 'old_column'),
+)
+
+
+
+# these are the old models, adapted to the migration
+
+class BaseModel(Model):
+    class Meta:
+        database = db
+
+class User(BaseModel):
+    user_id = CharField(default='', unique=True)   
+
+class Thread(BaseModel):
+    """The threads that concern us are only created if the messages that contain urls"""
+    thread_ts = FloatField(default = 0)
+    article_id = IntegerField(null=True)
+
+    
+class Message(BaseModel):
+    ts = FloatField(unique=True) #for sorting
+    channel_id = CharField(default='')
+    user = ForeignKeyField(User, backref="messages")
+    text = TextField(default='')
+    thread = ForeignKeyField(Thread, backref="messages", default=None)
+    file_type = CharField(default='')
+    perma_link = CharField(default='')
+    is_processed_override = BooleanField(default=False)
+
+
+class ArticleModelReference(BaseModel):
+    message = ForeignKeyField(Message, backref='article_model_references')
+    article_model_id = IntegerField(default = 0)
+
+
+
+
+for ref in ArticleModelReference.select():
+    ref.message.thread.article_id = ref.article_model_id
+    ref.message.thread.save()
+
+db.drop_tables((ArticleModelReference))

app/utils_storage/models.py

@@ -12,15 +12,23 @@ config = configuration.parsed["DOWNLOADS"]
 slack_config = configuration.parsed["SLACK"]
 
 ## Helpers
-db = DatabaseProxy()
+chat_db = DatabaseProxy()
+download_db = DatabaseProxy()
+
 # set the nature of the db at runtime
-class BaseModel(Model):
+
+class DownloadBaseModel(Model):
     class Meta:
-        database = db
+        database = download_db
+
+class ChatBaseModel(Model):
+    class Meta:
+        database = chat_db
+
 
 
 ## == Article related models == ##
-class ArticleDownload(BaseModel):
+class ArticleDownload(DownloadBaseModel):
     title = CharField(default='')
     pub_date = DateField(default = '')
     download_date = DateField(default = datetime.date.today)
@@ -55,7 +63,7 @@ class ArticleDownload(BaseModel):
 
     @property
     def fname_template(self):
-        if self.source_name == "youtube.com":
+        if "youtube.com" in self.source_name or "youtu.be" in self.source_name:
             fname = "{} -- {}".format(self.source_name, self.title)
         else:
             fname = "{} -- {}.pdf".format(self.source_name, self.title)
@@ -155,23 +163,23 @@ class ArticleDownload(BaseModel):
         return True, {}
 
 
-class ArticleKeyword(BaseModel):
+class ArticleKeyword(DownloadBaseModel):
     # instance gets created for every one keyword -> flexible in size
     article = ForeignKeyField(ArticleDownload, backref='keywords')
     keyword = CharField()
 
 
-class ArticleAuthor(BaseModel):
+class ArticleAuthor(DownloadBaseModel):
     article = ForeignKeyField(ArticleDownload, backref='authors')
     author = CharField()
 
 
-class ArticleReference(BaseModel):
+class ArticleReference(DownloadBaseModel):
     article = ForeignKeyField(ArticleDownload, backref='references')
     reference_url = TextField(default = '')
 
 
-class ArticleRelated(BaseModel):
+class ArticleRelated(DownloadBaseModel):
     article = ForeignKeyField(ArticleDownload, backref='related')
     related_file_name = TextField(default = '')
 
@@ -179,13 +187,13 @@ class ArticleRelated(BaseModel):
 
 
 ## == Slack-thread related models == ##
-class User(BaseModel):
+class User(ChatBaseModel):
     user_id = CharField(default='', unique=True)
     # messages
 
 
-class Thread(BaseModel):
-    """The threads that concern us are only created if the messages that contain urls"""
+class Thread(ChatBaseModel):
+    """The threads that concern us are only created if the base massage contains a url"""
     thread_ts = FloatField(default = 0)
     article = ForeignKeyField(ArticleDownload, backref="slack_thread", null=True, default=None)
     # provides, ts, user, models
@@ -227,7 +235,7 @@ class Thread(BaseModel):
 
 
     
-class Message(BaseModel):
+class Message(ChatBaseModel):
     ts = FloatField(unique=True) #for sorting
     channel_id = CharField(default='')
     user = ForeignKeyField(User, backref="messages")
@@ -275,7 +283,7 @@ class Message(BaseModel):
         return len(self.urls) == 1
 
 
-class Reaction(BaseModel):
+class Reaction(ChatBaseModel):
     type = CharField(default = "")
     message = ForeignKeyField(Message, backref="reaction")
 
@@ -286,17 +294,16 @@ class Reaction(BaseModel):
 
 
 
-
-
-
-
 def create_tables():
-    with db:
-        db.create_tables([ArticleDownload, ArticleKeyword, ArticleAuthor, ArticleReference, ArticleRelated, User, Message, Thread, Reaction])
+    with download_db:
+        download_db.create_tables([ArticleDownload, ArticleKeyword, ArticleAuthor, ArticleReference, ArticleRelated])
+    with chat_db:
+        chat_db.create_tables([User, Message, Thread, Reaction])
 
 
-def set_db(db_object):
-    db.initialize(db_object)
+def set_db(chat_db_object, download_db_object):
+    chat_db.initialize(chat_db_object)
+    download_db.initialize(download_db_object)
     create_tables()
 
 def clear_path_name(path):