Split geckodriver and python for cleaner containers

Dockerfile

@@ -2,21 +2,24 @@ FROM python:latest
 
 ENV TZ Euopre/Zurich
 
-RUN echo "deb http://deb.debian.org/debian/ unstable main contrib non-free" >> /etc/apt/sources.list
+# RUN echo "deb http://deb.debian.org/debian/ unstable main contrib non-free" >> /etc/apt/sources.list
+# allows the installation of the latest firefox-release (debian is not usually a rolling release)
 RUN apt-get update && apt-get install -y \
 evince \
 # for checking
-xauth wget tar firefox \
-# for geckodriver + gui
+xauth \
+#for gui
+# wget tar firefox \
+# for geckodriver
 ghostscript
 # for compression
 
 
 # Download gecko (firefox) driver for selenium
-RUN wget https://github.com/mozilla/geckodriver/releases/download/v0.31.0/geckodriver-v0.31.0-linux64.tar.gz
-RUN tar -x geckodriver -zf geckodriver-v0.31.0-linux64.tar.gz -O > /usr/bin/geckodriver
-RUN chmod +x /usr/bin/geckodriver
-RUN rm geckodriver-v0.31.0-linux64.tar.gz
+# RUN wget https://github.com/mozilla/geckodriver/releases/download/v0.31.0/geckodriver-v0.31.0-linux64.tar.gz
+# RUN tar -x geckodriver -zf geckodriver-v0.31.0-linux64.tar.gz -O > /usr/bin/geckodriver
+# RUN chmod +x /usr/bin/geckodriver
+# RUN rm geckodriver-v0.31.0-linux64.tar.gz
 
 
 RUN useradd --create-home --shell /bin/bash --uid 1001 autonews

README.md

@@ -51,7 +51,8 @@ All relevant passthroughs and mounts are specified through the env-file, for whi
 
 For the debug env-file, you will likely want interactivity, so you need to run:
 
-`docker compose --env-file env/debug run auto_news`
+`docker compose --env-file env/debug  up -d && docker compose --env-file env/debug exec auto_news bash && docker compose --env-file env/debug down`
+
 
 <!-- > Note:
 >

app/configuration.py

@@ -1,3 +1,4 @@
+from ast import parse
 import os
 import configparser
 import logging
@@ -20,10 +21,11 @@ parsed.read("/app/file_storage/config.ini")
 
 if os.getenv("DEBUG", "false") == "true":
     logger.warning("Found 'DEBUG=true', setting up dummy databases")
-
+    
     db_base_path = parsed["DATABASE"]["db_path_dev"]
     parsed["SLACK"]["archive_id"] = parsed["SLACK"]["debug_id"]
-    parsed["MAIL"]["recipient"] = parsed["MAIL"]["sender"] 
+    parsed["MAIL"]["recipient"] = parsed["MAIL"]["sender"]
+    parsed["DOWNLOADS"]["local_storage_path"] = parsed["DATABASE"]["db_path_dev"]
 else:
     logger.warning("Found 'DEBUG=false' and running on production databases, I hope you know what you're doing...")
 

app/utils_worker/download/browser.py

@@ -40,11 +40,16 @@ class PDFDownloader:
         options.set_preference("browser.download.dir", config["default_download_path"])
 
         self.logger.info("Starting gecko driver")
-        self.driver = webdriver.Firefox(
+        # self.driver = webdriver.Firefox(
+        #     options = options,
+        #     service = webdriver.firefox.service.Service(
+        #         log_path = f'{config["local_storage_path"]}/geckodriver.log'
+        # ))
+        self.driver = webdriver.Remote(
+            command_executor = 'http://localhost:4444',
             options = options,
-            service = webdriver.firefox.service.Service(
-                log_path = f'{config["local_storage_path"]}/geckodriver.log'
-        ))
+            # can't set log path...
+        )
         
         residues = os.listdir(config["default_download_path"])
         for res in residues:

app/utils_worker/workers.py

@@ -5,6 +5,7 @@ from .fetch.runner import get_description
 from .upload.runner import upload_to_archive as run_upload
 from .compress.runner import shrink_pdf
 
+import time
 import logging
 logger = logging.getLogger(__name__)
 
@@ -46,6 +47,7 @@ class UploadWorker(TemplateWorker):
     def _handle_article(self, article_watcher):
         action = run_upload # function
         super()._handle_article(article_watcher, action)
+        time.sleep(4) # Archive Uploads rate limited to 15/minute
         article_watcher.upload_completed = True
 
 

docker-compose.yaml

@@ -1,6 +1,5 @@
 # docker compose --env-file env/debug up 
 
-
 version: "3.9"
 services:
   auto_news:
@@ -26,16 +25,13 @@ services:
     stdin_open: ${INTERACTIVE:-false} # docker run -i
     tty: ${INTERACTIVE:-false}        # docker run -t
 
-
-  # geckodriver:
-  #   image: selenium/standalone-firefox:100.0
-  #   volumes: 
-  #     
-  #     - ${CONTAINER_DATA-/dev/null}:/app/file_storage
-  #     - ${FIREFOX_PROFILE}:/auto_news.profile
-  #     - ${HOSTS_FILE}:/etc/hosts
-  #   environment:
-  #     - DISPLAY=$DISPLAY
-  #     - START_XVFB=false
-
-  #   network_mode: host
+  geckodriver:
+    image: selenium/standalone-firefox:101.0
+    volumes:
+      - ${XAUTHORITY-/dev/null}:/home/seluser/.Xauthority
+    environment:
+      - DISPLAY=$DISPLAY
+      - START_VNC=false
+      - START_XVFB=false
+    user: 1001:1001
+    network_mode: host