remoll/coss_archiving
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Split geckodriver and python for cleaner containers

Browse Source
This commit is contained in:
Remy Moll 2022-06-16 11:58:28 +02:00
parent 87d65fc988
commit ac9e988af3
6 changed files with 37 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
Dockerfile
View File

@ -2,21 +2,24 @@ FROM python:latest
ENV TZ Euopre/Zurich ENV TZ Euopre/Zurich
RUN echo "deb http://deb.debian.org/debian/ unstable main contrib non-free" >> /etc/apt/sources.list # RUN echo "deb http://deb.debian.org/debian/ unstable main contrib non-free" >> /etc/apt/sources.list
# allows the installation of the latest firefox-release (debian is not usually a rolling release)
RUN apt-get update && apt-get install -y \ RUN apt-get update && apt-get install -y \
evince \ evince \
# for checking # for checking
xauth wget tar firefox \ xauth \
# for geckodriver + gui #for gui
# wget tar firefox \
# for geckodriver
ghostscript ghostscript
# for compression # for compression
# Download gecko (firefox) driver for selenium # Download gecko (firefox) driver for selenium
RUN wget https://github.com/mozilla/geckodriver/releases/download/v0.31.0/geckodriver-v0.31.0-linux64.tar.gz # RUN wget https://github.com/mozilla/geckodriver/releases/download/v0.31.0/geckodriver-v0.31.0-linux64.tar.gz
RUN tar -x geckodriver -zf geckodriver-v0.31.0-linux64.tar.gz -O > /usr/bin/geckodriver # RUN tar -x geckodriver -zf geckodriver-v0.31.0-linux64.tar.gz -O > /usr/bin/geckodriver
RUN chmod +x /usr/bin/geckodriver # RUN chmod +x /usr/bin/geckodriver
RUN rm geckodriver-v0.31.0-linux64.tar.gz # RUN rm geckodriver-v0.31.0-linux64.tar.gz
RUN useradd --create-home --shell /bin/bash --uid 1001 autonews RUN useradd --create-home --shell /bin/bash --uid 1001 autonews

3
README.md
View File

@ -51,7 +51,8 @@ All relevant passthroughs and mounts are specified through the env-file, for whi
For the debug env-file, you will likely want interactivity, so you need to run: For the debug env-file, you will likely want interactivity, so you need to run:
`docker compose --env-file env/debug run auto_news` `docker compose --env-file env/debug up -d && docker compose --env-file env/debug exec auto_news bash && docker compose --env-file env/debug down`
<!-- > Note: <!-- > Note:
> >

2
app/configuration.py
View File

@ -1,3 +1,4 @@
from ast import parse
import os import os
import configparser import configparser
import logging import logging
@ -24,6 +25,7 @@ if os.getenv("DEBUG", "false") == "true":
db_base_path = parsed["DATABASE"]["db_path_dev"] db_base_path = parsed["DATABASE"]["db_path_dev"]
parsed["SLACK"]["archive_id"] = parsed["SLACK"]["debug_id"] parsed["SLACK"]["archive_id"] = parsed["SLACK"]["debug_id"]
parsed["MAIL"]["recipient"] = parsed["MAIL"]["sender"] parsed["MAIL"]["recipient"] = parsed["MAIL"]["sender"]
parsed["DOWNLOADS"]["local_storage_path"] = parsed["DATABASE"]["db_path_dev"]
else: else:
logger.warning("Found 'DEBUG=false' and running on production databases, I hope you know what you're doing...") logger.warning("Found 'DEBUG=false' and running on production databases, I hope you know what you're doing...")

13
app/utils_worker/download/browser.py
View File

@ -40,11 +40,16 @@ class PDFDownloader:
options.set_preference("browser.download.dir", config["default_download_path"]) options.set_preference("browser.download.dir", config["default_download_path"])
self.logger.info("Starting gecko driver") self.logger.info("Starting gecko driver")
self.driver = webdriver.Firefox( # self.driver = webdriver.Firefox(
# options = options,
# service = webdriver.firefox.service.Service(
# log_path = f'{config["local_storage_path"]}/geckodriver.log'
# ))
self.driver = webdriver.Remote(
command_executor = 'http://localhost:4444',
options = options, options = options,
service = webdriver.firefox.service.Service( # can't set log path...
log_path = f'{config["local_storage_path"]}/geckodriver.log' )
))
residues = os.listdir(config["default_download_path"]) residues = os.listdir(config["default_download_path"])
for res in residues: for res in residues:

2
app/utils_worker/workers.py
View File

@ -5,6 +5,7 @@ from .fetch.runner import get_description
from .upload.runner import upload_to_archive as run_upload from .upload.runner import upload_to_archive as run_upload
from .compress.runner import shrink_pdf from .compress.runner import shrink_pdf
import time
import logging import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -46,6 +47,7 @@ class UploadWorker(TemplateWorker):
def _handle_article(self, article_watcher): def _handle_article(self, article_watcher):
action = run_upload # function action = run_upload # function
super()._handle_article(article_watcher, action) super()._handle_article(article_watcher, action)
time.sleep(4) # Archive Uploads rate limited to 15/minute
article_watcher.upload_completed = True article_watcher.upload_completed = True

24
docker-compose.yaml
View File

@ -1,6 +1,5 @@
# docker compose --env-file env/debug up # docker compose --env-file env/debug up
version: "3.9" version: "3.9"
services: services:
auto_news: auto_news:
@ -26,16 +25,13 @@ services:
stdin_open: ${INTERACTIVE:-false} # docker run -i stdin_open: ${INTERACTIVE:-false} # docker run -i
tty: ${INTERACTIVE:-false} # docker run -t tty: ${INTERACTIVE:-false} # docker run -t
geckodriver:
# geckodriver: image: selenium/standalone-firefox:101.0
# image: selenium/standalone-firefox:100.0 volumes:
# volumes: - ${XAUTHORITY-/dev/null}:/home/seluser/.Xauthority
# environment:
# - ${CONTAINER_DATA-/dev/null}:/app/file_storage - DISPLAY=$DISPLAY
# - ${FIREFOX_PROFILE}:/auto_news.profile - START_VNC=false
# - ${HOSTS_FILE}:/etc/hosts - START_XVFB=false
# environment: user: 1001:1001
# - DISPLAY=$DISPLAY network_mode: host
# - START_XVFB=false
# network_mode: host