remoll/coss_archiving
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

more bugs. Cleaner dockerfile

Browse Source
This commit is contained in:
Remy Moll 2022-04-20 19:16:58 +02:00
parent 8f3ea25662
commit 024da446e7
7 changed files with 25 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Dockerfile
View File

@ -1,7 +1,8 @@
FROM python:latest
RUN echo "deb http://deb.debian.org/debian/ unstable main contrib non-free" >> /etc/apt/sources.list
RUN apt-get update && apt-get install -y \
evince libcanberra-gtk-module \
evince \
# for checking
xauth wget tar firefox \
# for geckodriver + gui

2
README.md
View File

@ -45,7 +45,7 @@ where the `Dockerfile` has to be in the working directory
`docker run -it -v /mnt/Data/COSS/CONTAINERDATA/:/app/file_storage/ -v /mnt/Data/COSS/auto_news/app:/code --entrypoint /bin/bash auto_news`
`docker run -it -v /mnt/Data/COSS/DOWNLOADS/auto_news/container_data/:/app/file_storage/ -e DISPLAY=":0" --network host -v \$XAUTHORITY:/root/.Xauthority auto_news check`
`docker run -it -v /mnt/Data/COSS/CONTAINERDATA/:/app/file_storage/ -e DISPLAY=":0" --network host -v XAUTHORITY:/root/.Xauthority auto_news check`

13
app/runner.py
View File

@ -2,6 +2,7 @@
import configuration
models = configuration.models
import sys
from threading import Thread
import logging
logger = logging.getLogger(__name__)
@ -96,10 +97,10 @@ class ArticleWatcher:
class Coordinator:
class Coordinator(Thread):
def __init__(self, **kwargs) -> None:
"""Launcher calls this Coordinator as the main thread to handle connections between the other workers (threaded)."""
pass
super().__init__(target = self.launch)
def add_workers(self, **kwargs):
self.worker_slack = kwargs.pop("worker_slack", None)
@ -112,7 +113,8 @@ class Coordinator:
self.kwargs = kwargs
for w in [self.worker_slack, self.worker_download, self.worker_fetch, self.worker_upload, self.worker_compress]:
def launch(self) -> None:
for w in [self.worker_download, self.worker_fetch, self.worker_upload, self.worker_compress]:
if not w is None:
w.start()
@ -170,12 +172,15 @@ if __name__ == "__main__":
check_runner.verify_unchecked()
else: # launch with full action
slack_runner = slack_runner.BotRunner(coordinator.incoming_request)
kwargs = {
"worker_download" : DownloadWorker(),
"worker_fetch" : FetchWorker(),
"worker_upload" : UploadWorker(),
"worker_compress" : CompressWorker(),
"worker_slack" : slack_runner.BotRunner(coordinator.incoming_request),
"worker_slack" : slack_runner,
"worker_mail" : mail_runner,
}
coordinator.add_workers(**kwargs)
coordinator.start()
slack_runner.start()

10
app/utils_slack/message_helpers.py
View File

@ -123,7 +123,7 @@ async def fetch_missed_thread_messages():
)["messages"]
except SlackApiError:
logger.error("Hit rate limit while querying threaded messages, retrying in {}s ({}/{} queries elapsed)".format(config["api_wait_time"], i, len(threads)))
await asyncio.sleep(config["api_wait_time"])
await asyncio.sleep(int(config["api_wait_time"]))
messages = slack_client.conversations_replies(
channel = config["archive_id"],
ts = t.slack_ts,
@ -152,7 +152,7 @@ async def fetch_missed_channel_reactions():
reactions = query["message"].get("reactions", []) # default = []
except SlackApiError: # probably a rate_limit:
logger.error("Hit rate limit while querying reactions. retrying in {}s ({}/{} queries elapsed)".format(config["api_wait_time"], i, len(threads)))
await asyncio.sleep(config["api_wait_time"])
await asyncio.sleep(int(config["api_wait_time"]))
reactions = query["message"].get("reactions", [])
for r in reactions:
@ -202,11 +202,13 @@ def message_dict_to_model(message):
)
logger.info("Saved (text) {} (new={})".format(m, new))
for f in message.get("files", []): #default: []
files = message.get("files", [])
if len(files) >= 1:
f = files[0] #default: []
m.file_type = f["filetype"]
m.perma_link = f["url_private_download"]
m.save()
logger.info("Saved permalink {} to {} (possibly overwriting)".format(f["name"], m))
logger.info("Saved permalink {} to {}".format(f["name"], m))
if new:
return m
else:

8
app/utils_slack/runner.py
View File

@ -164,7 +164,7 @@ class BotApp(App):
class BotRunner(Thread):
class BotRunner():
"""Stupid encapsulation so that we can apply the slack decorators to the BotApp"""
def __init__(self, callback, *args, **kwargs) -> None:
self.bot_worker = BotApp(callback, token=config["auth_token"])
@ -177,11 +177,11 @@ class BotRunner(Thread):
def handle_incoming_reaction(event, say):
return self.bot_worker.handle_incoming_reaction(event)
target = self.launch
super().__init__(target=target)
# target = self.launch
# super().__init__(target=target)
def launch(self):
def start(self):
self.bot_worker.start()
SocketModeHandler(self.bot_worker, config["app_token"]).start()

3
app/utils_storage/models.py
View File

@ -207,7 +207,7 @@ class Thread(ChatBaseModel):
@property
def initiator_message(self):
return self.messages[0] # todo check if this needs sorting
return self.messages[0] # TODO check if this needs sorting
@property
def message_count(self):
@ -241,6 +241,7 @@ class Message(ChatBaseModel):
user = ForeignKeyField(User, backref="messages")
text = TextField(default='')
thread = ForeignKeyField(Thread, backref="messages", default=None)
file_type = CharField(default='')
perma_link = CharField(default='')
is_processed_override = BooleanField(default=False)
# reaction

2
app/utils_worker/compress/runner.py
View File

@ -13,7 +13,7 @@ def shrink_pdf(article):
return article # it probably was a youtube video
c = subprocess.run(
["gs", "-sDEVICE=pdfwrite", "-dPDFSETTINGS=/screen", "-dNOPAUSE", "-dBATCH", f"-sOutputFile={config['default_download_path']}/compressed.pdf", f'"{article.save_path + article.file_name}"'],
["gs", "-sDEVICE=pdfwrite", "-dPDFSETTINGS=/screen", "-dNOPAUSE", "-dBATCH", f"-sOutputFile={config['default_download_path']}/compressed.pdf", f"{article.save_path + article.file_name}"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
)