From 024da446e77c2801dcebd7825b70111a03ab0bf1 Mon Sep 17 00:00:00 2001 From: Remy Moll Date: Wed, 20 Apr 2022 19:16:58 +0200 Subject: [PATCH] more bugs. Cleaner dockerfile --- Dockerfile | 3 ++- README.md | 2 +- app/runner.py | 13 +++++++++---- app/utils_slack/message_helpers.py | 10 ++++++---- app/utils_slack/runner.py | 8 ++++---- app/utils_storage/models.py | 3 ++- app/utils_worker/compress/runner.py | 2 +- 7 files changed, 25 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index 093f2f1..a5057aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,8 @@ FROM python:latest +RUN echo "deb http://deb.debian.org/debian/ unstable main contrib non-free" >> /etc/apt/sources.list RUN apt-get update && apt-get install -y \ -evince libcanberra-gtk-module \ +evince \ # for checking xauth wget tar firefox \ # for geckodriver + gui diff --git a/README.md b/README.md index 3bb1884..ab427c3 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ where the `Dockerfile` has to be in the working directory `docker run -it -v /mnt/Data/COSS/CONTAINERDATA/:/app/file_storage/ -v /mnt/Data/COSS/auto_news/app:/code --entrypoint /bin/bash auto_news` -`docker run -it -v /mnt/Data/COSS/DOWNLOADS/auto_news/container_data/:/app/file_storage/ -e DISPLAY=":0" --network host -v \$XAUTHORITY:/root/.Xauthority auto_news check` +`docker run -it -v /mnt/Data/COSS/CONTAINERDATA/:/app/file_storage/ -e DISPLAY=":0" --network host -v XAUTHORITY:/root/.Xauthority auto_news check` diff --git a/app/runner.py b/app/runner.py index ed96736..a51bbd2 100644 --- a/app/runner.py +++ b/app/runner.py @@ -2,6 +2,7 @@ import configuration models = configuration.models import sys +from threading import Thread import logging logger = logging.getLogger(__name__) @@ -96,10 +97,10 @@ class ArticleWatcher: -class Coordinator: +class Coordinator(Thread): def __init__(self, **kwargs) -> None: """Launcher calls this Coordinator as the main thread to handle connections between the other workers (threaded).""" - pass + super().__init__(target = self.launch) def add_workers(self, **kwargs): self.worker_slack = kwargs.pop("worker_slack", None) @@ -112,7 +113,8 @@ class Coordinator: self.kwargs = kwargs - for w in [self.worker_slack, self.worker_download, self.worker_fetch, self.worker_upload, self.worker_compress]: + def launch(self) -> None: + for w in [self.worker_download, self.worker_fetch, self.worker_upload, self.worker_compress]: if not w is None: w.start() @@ -170,12 +172,15 @@ if __name__ == "__main__": check_runner.verify_unchecked() else: # launch with full action + slack_runner = slack_runner.BotRunner(coordinator.incoming_request) kwargs = { "worker_download" : DownloadWorker(), "worker_fetch" : FetchWorker(), "worker_upload" : UploadWorker(), "worker_compress" : CompressWorker(), - "worker_slack" : slack_runner.BotRunner(coordinator.incoming_request), + "worker_slack" : slack_runner, "worker_mail" : mail_runner, } coordinator.add_workers(**kwargs) + coordinator.start() + slack_runner.start() diff --git a/app/utils_slack/message_helpers.py b/app/utils_slack/message_helpers.py index aeb71c1..782be69 100644 --- a/app/utils_slack/message_helpers.py +++ b/app/utils_slack/message_helpers.py @@ -123,7 +123,7 @@ async def fetch_missed_thread_messages(): )["messages"] except SlackApiError: logger.error("Hit rate limit while querying threaded messages, retrying in {}s ({}/{} queries elapsed)".format(config["api_wait_time"], i, len(threads))) - await asyncio.sleep(config["api_wait_time"]) + await asyncio.sleep(int(config["api_wait_time"])) messages = slack_client.conversations_replies( channel = config["archive_id"], ts = t.slack_ts, @@ -152,7 +152,7 @@ async def fetch_missed_channel_reactions(): reactions = query["message"].get("reactions", []) # default = [] except SlackApiError: # probably a rate_limit: logger.error("Hit rate limit while querying reactions. retrying in {}s ({}/{} queries elapsed)".format(config["api_wait_time"], i, len(threads))) - await asyncio.sleep(config["api_wait_time"]) + await asyncio.sleep(int(config["api_wait_time"])) reactions = query["message"].get("reactions", []) for r in reactions: @@ -202,11 +202,13 @@ def message_dict_to_model(message): ) logger.info("Saved (text) {} (new={})".format(m, new)) - for f in message.get("files", []): #default: [] + files = message.get("files", []) + if len(files) >= 1: + f = files[0] #default: [] m.file_type = f["filetype"] m.perma_link = f["url_private_download"] m.save() - logger.info("Saved permalink {} to {} (possibly overwriting)".format(f["name"], m)) + logger.info("Saved permalink {} to {}".format(f["name"], m)) if new: return m else: diff --git a/app/utils_slack/runner.py b/app/utils_slack/runner.py index 2e20ea5..1b146fa 100644 --- a/app/utils_slack/runner.py +++ b/app/utils_slack/runner.py @@ -164,7 +164,7 @@ class BotApp(App): -class BotRunner(Thread): +class BotRunner(): """Stupid encapsulation so that we can apply the slack decorators to the BotApp""" def __init__(self, callback, *args, **kwargs) -> None: self.bot_worker = BotApp(callback, token=config["auth_token"]) @@ -177,11 +177,11 @@ class BotRunner(Thread): def handle_incoming_reaction(event, say): return self.bot_worker.handle_incoming_reaction(event) - target = self.launch - super().__init__(target=target) + # target = self.launch + # super().__init__(target=target) - def launch(self): + def start(self): self.bot_worker.start() SocketModeHandler(self.bot_worker, config["app_token"]).start() diff --git a/app/utils_storage/models.py b/app/utils_storage/models.py index c01b0e9..7d2efd4 100644 --- a/app/utils_storage/models.py +++ b/app/utils_storage/models.py @@ -207,7 +207,7 @@ class Thread(ChatBaseModel): @property def initiator_message(self): - return self.messages[0] # todo check if this needs sorting + return self.messages[0] # TODO check if this needs sorting @property def message_count(self): @@ -241,6 +241,7 @@ class Message(ChatBaseModel): user = ForeignKeyField(User, backref="messages") text = TextField(default='') thread = ForeignKeyField(Thread, backref="messages", default=None) + file_type = CharField(default='') perma_link = CharField(default='') is_processed_override = BooleanField(default=False) # reaction diff --git a/app/utils_worker/compress/runner.py b/app/utils_worker/compress/runner.py index 8a99fcb..993ccc0 100644 --- a/app/utils_worker/compress/runner.py +++ b/app/utils_worker/compress/runner.py @@ -13,7 +13,7 @@ def shrink_pdf(article): return article # it probably was a youtube video c = subprocess.run( - ["gs", "-sDEVICE=pdfwrite", "-dPDFSETTINGS=/screen", "-dNOPAUSE", "-dBATCH", f"-sOutputFile={config['default_download_path']}/compressed.pdf", f'"{article.save_path + article.file_name}"'], + ["gs", "-sDEVICE=pdfwrite", "-dPDFSETTINGS=/screen", "-dNOPAUSE", "-dBATCH", f"-sOutputFile={config['default_download_path']}/compressed.pdf", f"{article.save_path + article.file_name}"], stdout=subprocess.PIPE, stderr=subprocess.PIPE )