Split geckodriver and python for cleaner containers

app/configuration.py

@@ -1,3 +1,4 @@
+from ast import parse
 import os
 import configparser
 import logging
@@ -20,10 +21,11 @@ parsed.read("/app/file_storage/config.ini")
 
 if os.getenv("DEBUG", "false") == "true":
     logger.warning("Found 'DEBUG=true', setting up dummy databases")
-
+    
     db_base_path = parsed["DATABASE"]["db_path_dev"]
     parsed["SLACK"]["archive_id"] = parsed["SLACK"]["debug_id"]
-    parsed["MAIL"]["recipient"] = parsed["MAIL"]["sender"] 
+    parsed["MAIL"]["recipient"] = parsed["MAIL"]["sender"]
+    parsed["DOWNLOADS"]["local_storage_path"] = parsed["DATABASE"]["db_path_dev"]
 else:
     logger.warning("Found 'DEBUG=false' and running on production databases, I hope you know what you're doing...")
 

app/utils_worker/download/browser.py

@@ -40,11 +40,16 @@ class PDFDownloader:
         options.set_preference("browser.download.dir", config["default_download_path"])
 
         self.logger.info("Starting gecko driver")
-        self.driver = webdriver.Firefox(
+        # self.driver = webdriver.Firefox(
+        #     options = options,
+        #     service = webdriver.firefox.service.Service(
+        #         log_path = f'{config["local_storage_path"]}/geckodriver.log'
+        # ))
+        self.driver = webdriver.Remote(
+            command_executor = 'http://localhost:4444',
             options = options,
-            service = webdriver.firefox.service.Service(
-                log_path = f'{config["local_storage_path"]}/geckodriver.log'
-        ))
+            # can't set log path...
+        )
         
         residues = os.listdir(config["default_download_path"])
         for res in residues:

app/utils_worker/workers.py

@@ -5,6 +5,7 @@ from .fetch.runner import get_description
 from .upload.runner import upload_to_archive as run_upload
 from .compress.runner import shrink_pdf
 
+import time
 import logging
 logger = logging.getLogger(__name__)
 
@@ -46,6 +47,7 @@ class UploadWorker(TemplateWorker):
     def _handle_article(self, article_watcher):
         action = run_upload # function
         super()._handle_article(article_watcher, action)
+        time.sleep(4) # Archive Uploads rate limited to 15/minute
         article_watcher.upload_completed = True