A working app with a few bugs sprinkled in. Configuration saved externally

misc/exctract_from_mail_backup.py

@@ -0,0 +1,23 @@
+import os
+import re
+import json
+
+os.chdir("/home/remy/Documents/mails2/")
+
+regex = "(?P<url>https?://[^\s]+)"
+
+all_files = os.listdir(".")
+all_urls = []
+
+for f in all_files:
+    with open(f, "r", encoding="utf8") as mail:
+        content = mail.readlines()
+    
+    search = "".join(content)
+    urls = re.findall(regex, search)
+    all_urls += urls
+
+print("Saved {} urls".format(len(all_urls)))
+
+with open("mails_export.json", "w") as f:
+    json.dump(all_urls, f)  

misc/hotfix_mails.py

@@ -0,0 +1,38 @@
+import logging
+import keys
+from peewee import SqliteDatabase
+
+from persistence import article_models
+from archiving_utils import runner as archive_runner
+from mail_utils import runner as mail_runner
+
+# Global logger setup:
+logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S')
+logger = logging.getLogger("MailThread")
+
+
+# Constant values...
+DOWNLOADS_DB = "/app/file_storage/downloads.db"
+
+
+# DB Setup:
+article_models.set_db(SqliteDatabase(
+    DOWNLOADS_DB,
+    pragmas = {'journal_mode': 'wal'} # mutliple threads can access at once
+))
+
+
+mail_worker = mail_runner.MailSender(keys.MAIL_UNAME, keys.MAIL_PASSWORD, keys.MAIL_SENDER, keys.MAIL_RECIPIENT)
+dl_worker = archive_runner.ArchivingThread(article_models, mail_worker)
+dl_worker.start()
+
+
+
+# Retroactively sends a message to DIRK for messages that were archived using slack, but when the mail-reply was not yet implemented
+
+
+
+url_list = []
+
+for url in url_list:
+    dl_worker.get_or_save(url)

misc/hotfix_missed_messages.py

@@ -0,0 +1,88 @@
+import time
+import keys
+import slack_sdk
+from slack_sdk.errors import SlackApiError
+from peewee import SqliteDatabase
+
+from persistence import  message_models
+# from bot_utils import messages
+
+
+
+# Constant values...
+MESSAGES_DB = "/app/file_storage/messages.db"
+
+BOT_ID = "U02MR1R8UJH"
+ARCHIVE_ID = "C02MM7YG1V4"
+DEBUG_ID = "C02NM2H9J5Q"
+
+
+
+client = slack_sdk.WebClient(token=keys.OAUTH_TOKEN)
+
+message_models.set_db(SqliteDatabase(MESSAGES_DB))
+
+
+def message_dict_to_model(message):
+    if message["type"] == "message":
+        thread_ts = message["thread_ts"] if "thread_ts" in message else message["ts"]
+        uid = message.get("user", "BAD USER")
+        user, _ = message_models.User.get_or_create(user_id = uid)
+        thread, _ = message_models.Thread.get_or_create(thread_ts = thread_ts)
+        m, new = message_models.Message.get_or_create(
+            user = user,
+            thread = thread,
+            ts = message["ts"],
+            channel_id = ARCHIVE_ID,
+            text = message["text"]
+        )
+        print("Saved (text) {} (new={})".format(m, new))
+
+        for f in message.get("files", []): #default: []
+            m.file_type = f["filetype"]
+            m.perma_link = f["url_private_download"]
+            m.save()
+            print("Saved permalink {} to {} (possibly overwriting)".format(f["name"], m))
+        if new:
+            return m
+        else:
+            return None
+    else:
+        print("What should I do of {}".format(message))
+        return None
+
+
+def check_all_past_messages():
+    last_ts = 0
+    
+    result = client.conversations_history(
+        channel=ARCHIVE_ID,
+        oldest=last_ts
+    )
+
+    new_messages = result.get("messages", []) # fetches 100 messages by default
+
+    new_fetches = []
+    for m in new_messages:
+        new_fetches.append(message_dict_to_model(m))
+    # print(result)
+    refetch = result.get("has_more", False)
+    print(f"Refetching : {refetch}")
+    while refetch: # we have not actually fetched them all
+        try:
+            result = client.conversations_history(
+                channel = ARCHIVE_ID,
+                cursor = result["response_metadata"]["next_cursor"],
+                oldest = last_ts
+            ) # refetches in batches of 100 messages
+            refetch = result.get("has_more", False)
+            new_messages = result.get("messages", [])
+            for m in new_messages:
+                new_fetches.append(message_dict_to_model(m))
+        except SlackApiError: # Most likely a rate-limit
+            print("Error while fetching channel messages. (likely rate limit) Retrying in {} seconds...".format(30))
+            time.sleep(30)
+            refetch = True
+
+
+check_all_past_messages()

misc/hotfix_reactions.py

@@ -0,0 +1,38 @@
+from peewee import SqliteDatabase
+
+from persistence import article_models, message_models
+
+# Global logger setup:
+
+
+# Constant values...
+DOWNLOADS_DB = "../container_data/downloads.db"
+MESSAGES_DB = "../container_data/messages.db"
+
+BOT_ID = "U02MR1R8UJH"
+ARCHIVE_ID = "C02MM7YG1V4"
+DEBUG_ID = "C02NM2H9J5Q"
+
+
+# DB Setup:
+article_models.set_db(SqliteDatabase(
+    DOWNLOADS_DB,
+    pragmas = {'journal_mode': 'wal'} # mutliple threads can access at once
+))
+
+message_models.set_db(SqliteDatabase(MESSAGES_DB))
+
+
+
+for reaction in message_models.Reaction.select():
+    print(reaction)        
+    thread = reaction.message.thread
+    articles = message_models.get_referenced_articles(thread, article_models.ArticleDownload)
+    for a in articles:
+        print(a)
+    reaction = reaction.type
+    status = 1 if reaction == "white_check_mark" else -1
+    print(status)
+    for article in articles:
+        article.verified = status
+        article.save()