Update usage instructions

.gitignore

@@ -4,6 +4,9 @@
 __pycache__/
 
 
+config/container.yaml
+config/local.env
+
 
 ## svelte:
 # Logs

Makefile

@@ -0,0 +1,87 @@
+include config/local.env
+export
+
+build:
+	@echo "Building..."
+	docker compose build $(flags)
+
+
+down:
+	@echo "Stopping containers..."
+	docker compose down -t 0 --volumes
+
+
+# Variables specific to debug
+debug: export DEBUG=true
+debug: export HEADFULL=true
+debug: export ENTRYPOINT=/bin/bash
+debug: export CODE=./
+debug:
+	@echo "Running in debug mode..."
+	docker compose up -d geckodriver
+	docker compose run -it --service-ports $(target) $(flags) || true
+	make down
+
+
+production: export DEBUG=false
+production:
+	@echo "Running in production mode..."
+	docker compose run -it --service-ports $(target) $(flags) || true
+	make down
+
+
+nas_sync:
+	@echo "Syncing NAS..."
+	SYNC_FOLDER=$(folder) docker compose run -it nas_sync $(flags) || true
+	docker compose down
+	docker container prune -f
+	make down
+
+
+
+
+## Misc:
+edit_profile: export CODE=./
+edit_profile: export HEADFULL=true
+edit_profile:
+	@echo "Editing profile..."
+	docker compose up -d geckodriver
+	sleep 5
+	docker compose exec geckodriver /bin/bash /code/geckodriver/edit_profile.sh || true
+	# runs inside the container
+	make down
+
+
+
+db_interface:
+	docker create \
+	--name pgadmin \
+	-p 8080:80 \
+	-e 'PGADMIN_DEFAULT_EMAIL=${UNAME}@test.com' \
+	-e 'PGADMIN_DEFAULT_PASSWORD=password' \
+	-e 'PGADMIN_CONFIG_ENHANCED_COOKIE_PROTECTION=True' \
+	-e 'PGADMIN_CONFIG_LOGIN_BANNER="Authorised users only!"' \
+	dpage/pgadmin4
+
+	docker start pgadmin
+
+	sleep 5
+
+	# TODO auto add the server to the list displayed in the browser
+	# docker exec pgadmin sh -c "echo ${SERVER_DATA} > /tmp/servers.json"
+	# docker exec pgadmin sh -c "/venv/bin/python setup.py --load-servers /tmp/servers.json --user remy@test.com"
+	@echo "Go to http://localhost:8080 to access the database interface"
+	@echo "Username: ${UNAME}@test.com"
+	@echo "Password: password"
+	@echo "Hit any key to stop (not ctrl+c)"
+	read STOP
+
+	docker stop pgadmin
+	docker rm pgadmin
+
+
+logs:
+	docker compose logs -f $(target) $(flags)
+
+
+	make down

README.md

@@ -12,45 +12,103 @@ A utility to
 
 ---
 
-## Running - through launch file
-> Prerequisite: make `launch.cexecutable:
-> 
-> `chmod +x launch`
+## Running - through makefile
 
-Execute the file by runnning `./launch`. This won't do anything in itself. You need to specify a mode, and then a command
+Execute the file by runnning `make`. This won't do anything in itself. For the main usage you need to specify a mode and a target.
 
-`./launch <mode> <command> <command options>`
+`make <mode> target=<target>`
 
 ### Overview of the modes
 
-The production mode performs all automatic actions and therfore does not require any manual intervention. It queries the slack workspace, adds the new requests to the database, downloads all files and metadata, uploads the urls to archive.org and sends out the downloaded article. As a last step the newly created file is synced to the COSS-NAS.
+The production mode performs all automatic actions and therefore does not require any manual intervention. It queries the slack workspace, adds the new requests to the database, downloads all files and metadata, uploads the urls to archive.org and sends out the downloaded article. As a last step the newly created file is synced to the COSS-NAS.
 
 The debug mode is more sophisticated and allows for big code changes without the need to recompile. It directly mounts the code-directory into the cotainer. As a failsafe the environment-variable `DEBUG=true` is set. The whole utility is then run on a sandbox environment (slack-channel, database, email) so that Dirk is not affected by any mishaps.
 
 Two additional 'modes' are `build` and `down`. Build rebuilds the container, which is necessary after code changes. Down ensures a clean shutdown of *all* containers. Usually the launch-script handles this already but it sometimes fails, in which case `down` needs to be called again.
 
 
-### Overview of the commands
+### Overview of the targets
 
-In essence a command is simply a service from docker-compose, which is run in an interactive environment. As such all services defined in `docker-compose.yaml` can be called as commands. Only two of them will be of real use:
+In essence a target is simply a service from docker-compose, which is run in an interactive environment. As such all services defined in `docker-compose.yaml` can be called as target. Only two of them will be of real use:
 
-`news_fetch` does the majority of the actions mentionned above. By default, that is without any options, it runs a metadata-fetch, download, and upload to archive.org. The upload is usually the slowest which is why articles that are processed but don't yet have an archive.org url tend to pile up. You can therefore specify the option `upload` which only starts the upload for the concerned articles, as a catch-up if you will.
+`news_fetch` does the majority of the actions mentioned above. By default, that is without any options, it runs a metadata-fetch, download, and upload to archive.org. The upload is usually the slowest which is why articles that are processed but don't yet have an archive.org url tend to pile up. You can therefore specify the option `upload` which only starts the upload for the concerned articles, as a catch-up if you will.
 
 Example usage:
 
 ```bash
-./launch production news_fetch # full mode
-./launch production news_fetch upload # upload mode (lighter resource usage)
-./launch debug news_fetch # debug mode, which drops you inside a new shell
+make production target=news_fetch # full mode
+make production target=news_fetch flags=upload # upload mode (lighter resource usage)
+make debug target=news_fetch # debug mode, which drops you inside a new shell
 
-./launch production news_check
+make production target=news_check
 ```
 
 `news_check` starts a webapp, accessible under [http://localhost:8080](http://localhost:8080) and allows you to easily check the downloaded articles.
 
+### Synchronising changes with NAS
 
-## (Running - Docker compose) 
-> I strongly recommend sticking to the usage of `./launch`.
+I recommend `rsync`.
+
+From within the ETH-network you can launch
+```
+make nas_sync folder=<target>
+```
+this will launch a docker container running `rsync` and connected to both the COSS NAS-share and your local files. Specifying a folder restricts the files that are watched for changes.
+
+example: `make nas_sync folder=2022/September` will take significantly less time than `make nas_sync folder=2022` but only considers files written to the September folder.
+
+> Please check the logs for any suspicious messages. `rsync`ing to smb shares is prone to errors.
+
+
+### Misc. usage:
+
+```bash
+make build # rebuilds all containers to reflect code changes
+make down # shuts down all containers (usually not necessary since this occurs automatically)
+make edit_profile # opens a firefox window under localhost:7900 to edit the profile used by news_fetch
+make db_interfacce # opens a postgres-interface to view the remote database (localhost:8080)
+```
+
+## First run:
+> The program relies on a functioning firefox profile!
+
+For the first run ever, run 
+
+`make edit_profile`
+
+This will generate a new firefox profile under `coss_archiving/dependencies/news_fetch.profile`.
+You can then go to [http://localhost:7900](http://localhost:7900) in your browser. Check the profile (under firefox://profile-internals).
+
+Now install two addons: Idontcareaboutcookies and bypass paywalls clean (from firefox://extensions). They ensure that most sites just work out of the box. You can additionally install adblockers such as ublock origin.
+
+You can then use this profile to further tweak various sites. The state of the sites (namely their cookies) will be used by `news_fetch`.
+
+> Whenever you need to make changes to the profile, for instance re-log in to websites, just rerun `make edit_profile`.
+
+
+## Building
+
+> The software **will** change. Because the images referenced in docker compose are usually the `latest` ones, it is sufficient to update the containers.
+
+In docker compose, run 
+
+`docker compose --env-file env/production build`
+
+Or simpler, just run
+
+`make build` (should issues occur you can also run `make build flags=--no-cache`)
+
+
+## Roadmap:
+
+- [ ] handle paywalled sites like faz, spiegel, ... through their dedicated sites (see nexisuni.com for instance), available through the ETH network
+- [ ] improve reliability of nas_sync. (+ logging)
+- [ ] divide month folders into smaller ones
+
+
+
+## Appendix: (Running - Docker compose) 
+> I strongly recommend sticking to the usage of `make`.
 
 Instead of using the launch file you can manually issue `docker compose` comands. Example: check for logs.
 
@@ -75,53 +133,3 @@ docker compose --env-file env/production up # starts all services and shows thei
 docker compose --env-file env/production logs -f news_fetch # follows along with the logs of only one service
 docker compose --env-file env/production down
 ```
-
-### First run:
-> The program relies on a functioning firefox profile!
-
-For the first run ever, run 
-
-`./launch edit_profile`
-
-This will generate a new firefox profile under `coss_archiving/dependencies/news_fetch.profile`.
-You can then go to [http://localhost:7900](http://localhost:7900) in your browser. Check the profile (under firefox://profile-internals).
-
-Now install two addons: Idontcareaboutcookies and bypass paywalls clean (from firefox://extensions). They ensure that most sites just work out of the box. You can additionally install adblockers such as ublock origin.
-
-You can then use this profile to further tweak various sites. The state of the sites (namely their cookies) will be used by `news_fetch`.
-
-> Whenever you need to make changes to the profile, for instance re-log in to websites, just rerun `./launch edit_profile`.
-
-Exit the mode by closing the firefox window. You can then run `./launch down` and then proceed normally.
-
-
-## Building
-
-> The software **will** change. Because the images referenced in docker compose are usually the `latest` ones, it is sufficient to update the containers.
-
-In docker compose, run 
-
-`docker compose --env-file env/production build`
-
-Or simpler, just run
-
-`./launch build` (should issues occur you can also run `./launch build --no-cache`)
-
-
-
-## Roadmap:
-
-- [ ] handle paywalled sites like faz, spiegel, ... through their dedicated sites (see nexisuni.com for instance), available through the ETH network
-- [ ] improve reliability of nas_sync. (+ logging)
-- [ ] divide month folders into smaller ones
-
-
-
-## Manual Sync to NAS:
-Manual sync is sadly still necessary, as the lsync client, sometimes gets overwhelmed by quick writes.
-
-I use `rsync`. Mounting the NAS locally, I navigate to the location of the local folder (notice the trailing slash). Then run
-`rsync -Razq --no-perms --no-owner --no-group --temp-dir=/tmp --progress --log-file=rsync.log <local folder>/ "<remote>"`
-where `<remote>` is the location where the NAS is mounted. (options:`R` - relative paths  , `a` - archive mode (multiple actions), `z` - ??, `q` - quiet. We also don't copy most of the metadata and we keep a log of the transfers.)
-
-You can also use your OS' native copy option and select *de not overwrite*. This should only copy the missing files, significantly speeding up the operation.

config/README.md

@@ -1,8 +0,0 @@
-## Configuration: example
-The files inside this directory (not the ones in `env/`) are a sample of the required configuration.
-
-Please create a copy of these files under `<location of downloads>/config/...`.
-
-> Note:
->
-> Some of the fields are blank, please fill them in as needed.

config/container.yaml

@@ -0,0 +1,37 @@
+mail:
+  smtp_server: smtp.ethz.ch
+  port: 587
+  sender: "****************"
+  recipient: "****************"
+  uname: "****************"
+  password: "************"
+
+
+slack:
+  bot_id: U02MR1R8UJH
+  archive_id: C02MM7YG1V4
+  debug_id: C02NM2H9J5Q
+  api_wait_time: 90
+  auth_token: "****************"
+  app_token: "****************"
+
+
+database:
+  debug_db: /app/containerdata/debug/downloads.db
+  db_printout: /app/containerdata/backups
+  production_db_name: coss_archiving
+  production_user_name: "ca_rw"
+  production_password: "****************"
+
+  ## user_name: ca_ro
+  ## password: "****************"
+
+
+downloads:
+  local_storage_path: /app/containerdata/files
+  debug_storage_path: /app/containerdata/debug/
+  default_download_path: /app/containerdata/tmp
+  remote_storage_path: /helbing_support/Archiving-Pipeline
+  browser_profile_path: /app/containerdata/dependencies/news_fetch.profile
+  # please keep this exact name
+  browser_print_delay: 3

config/db.config.ini

@@ -1,7 +0,0 @@
-[DATABASE]
-db_name: coss_archiving
-user_name: ****************
-password: ****************
-
-## user_name: ca_ro
-## password: #TK5cLxA^YyoxWjR6

config/local.env

@@ -0,0 +1,18 @@
+CONTAINER_DATA=***********
+UNAME=***********
+U_ID=***********
+
+DB_HOST=***********
+
+
+OPENCONNECT_URL=***********
+OPENCONNECT_USER=***********
+OPENCONNECT_PASSWORD=***********
+OPENCONNECT_OPTIONS=--authgroup student-net
+
+
+NAS_HOST=***********
+NAS_PATH=/gess_coss_1/helbing_support/Archiving-Pipeline
+NAS_USERNAME=***********
+NAS_PASSWORD=***********
+# Special characters like # need to be escaped (write: \#) 

config/nas_login.config

@@ -1,3 +0,0 @@
-user=remoll
-domain=D
-password=****************

config/nas_sync.config

@@ -1,12 +0,0 @@
-settings {
-   logfile    = "/tmp/lsyncd.log",
-   statusFile = "/tmp/lsyncd.status",
-   nodaemon   = true,
-}
-
-sync {
-   default.rsync,
-   source = "/sync/local_files",
-   target = "/sync/remote_files",
-   init = false,
-}

config/news_fetch.config.ini

@@ -1,31 +0,0 @@
-[MAIL]
-smtp_server: smtp.ethz.ch
-port: 587
-sender: ****************
-recipient: ****************
-uname: ****************
-password: ****************
-
-
-[SLACK]
-bot_id: U02MR1R8UJH
-archive_id: C02MM7YG1V4
-debug_id: C02NM2H9J5Q
-api_wait_time: 90
-auth_token: ****************
-app_token: ****************
-
-
-[DATABASE]
-download_db_debug: /app/containerdata/debug/downloads.db
-db_printout: /app/containerdata/backups
-
-
-[DOWNLOADS]
-local_storage_path: /app/containerdata/files
-debug_storage_path: /app/containerdata/debug/
-default_download_path: /app/containerdata/tmp
-remote_storage_path: /helbing_support/Archiving-Pipeline
-browser_profile_path: /app/containerdata/dependencies/news_fetch.profile
-# please keep this exact name
-browser_print_delay: 3

config/vpn.config

@@ -1,4 +0,0 @@
-OPENCONNECT_URL=sslvpn.ethz.ch/student-net
-OPENCONNECT_USER=****************
-OPENCONNECT_PASSWORD=****************
-OPENCONNECT_OPTIONS=--authgroup student-net

docker-compose.yaml

@@ -4,33 +4,17 @@ services:
 
   vpn: # Creates a connection behind the ETH Firewall to access NAS and Postgres
     image: wazum/openconnect-proxy:latest
-    env_file:
-      - ${CONTAINER_DATA}/config/vpn.config
+    environment:
+      - OPENCONNECT_URL=${OPENCONNECT_URL}
+      - OPENCONNECT_USER=${OPENCONNECT_USER}
+      - OPENCONNECT_PASSWORD=${OPENCONNECT_PASSWORD}
+      - OPENCONNECT_OPTIONS=${OPENCONNECT_OPTIONS}
     cap_add:
     - NET_ADMIN
     volumes:
       - /dev/net/tun:/dev/net/tun
     # alternative to cap_add & volumes: specify privileged: true
     expose: ["5432"] # exposed here because db_passhtrough uses this network. See below for more details
-    
-
-  nas_sync: # Syncs locally downloaded files with the NAS-share on nas22.ethz.ch/...
-    depends_on:
-      - vpn
-    network_mode: "service:vpn" # used to establish a connection to the SMB server from inside ETH network
-    build: nas_sync # local folder to build
-    image: nas_sync:latest
-    cap_add: # capabilities needed for mounting the SMB share
-      - SYS_ADMIN
-      - DAC_READ_SEARCH
-    volumes:
-      - ${CONTAINER_DATA}/files:/sync/local_files
-      - ${CONTAINER_DATA}/config/nas_sync.config:/sync/nas_sync.config
-      - ${CONTAINER_DATA}/config/nas_login.config:/sync/nas_login.config
-    command:
-      - nas22.ethz.ch/gess_coss_1/helbing_support/Archiving-Pipeline # first command is the target mount path
-      - lsyncd
-      - /sync/nas_sync.config
 
 
   geckodriver: # separate docker container for pdf-download. This hugely improves stability (and creates shorter build times for the containers)
@@ -40,7 +24,6 @@ services:
       - START_VNC=${HEADFULL-false} # as opposed to headless, used when requiring supervision (eg. for websites that crash)
       - START_XVFB=${HEADFULL-false}
       - SE_VNC_NO_PASSWORD=1
-      # - SE_OPTS="--profile /user_data/news_fetch.profile.firefox"
     volumes:
       - ${CONTAINER_DATA}/dependencies:/firefox_profile/
       - ${CODE:-/dev/null}:/code
@@ -53,7 +36,7 @@ services:
   db_passthrough: # Allows a container on the local network to connect to a service (here postgres) through the vpn
     network_mode: "service:vpn"
     image: alpine/socat:latest
-    command: ["tcp-listen:5432,reuseaddr,fork", "tcp-connect:id-hdb-psgr-cp48.ethz.ch:5432"]
+    command: ["tcp-listen:5432,reuseaddr,fork", "tcp-connect:${DB_HOST}:5432"]
     # expose: ["5432"] We would want this passthrough to expose its ports to the other containers
     # BUT since it uses the same network as the vpn-service, it can't expose ports on its own. 5432 is therefore exposed under service.vpn.expose 
 
@@ -62,14 +45,14 @@ services:
     build: news_fetch
     image: news_fetch:latest
     depends_on: # when using docker compose run news_fetch, the dependencies are started as well
-      - nas_sync
       - geckodriver
       - db_passthrough
-
     volumes:
       - ${CONTAINER_DATA}:/app/containerdata # always set
+      - ./config/container.yaml:/app/config.yaml
       - ${CODE:-/dev/null}:/code # not set in prod, defaults to /dev/null
     environment:
+      - CONFIG_FILE=/app/config.yaml
       - DEBUG=${DEBUG}
       - UNAME=${UNAME}
     user: ${U_ID}:${U_ID} # since the app writes files to the local filesystem, it must be run as the current user
@@ -86,10 +69,33 @@ services:
       - db_passthrough
     volumes:
       - ${CONTAINER_DATA}:/app/containerdata # always set
+      - ./config/container.yaml:/app/config.yaml
       - ${CODE:-/dev/null}:/code # not set in prod, defaults to /dev/null
     environment:
+      - CONFIG_FILE=/app/config.yaml
       - UNAME=${UNAME}
     ports:
       - "8080:80" # 80 inside container
     entrypoint: ${ENTRYPOINT:-python app.py} # by default launch workers as defined in the Dockerfile
-    tty: true
+
+
+  nas_sync:
+    image: alpine:latest
+    volumes:
+      - ${CONTAINER_DATA}/files:/sync/local_files
+      - coss_smb_share:/sync/remote_files
+    command:
+      - /bin/sh
+      - -c
+      - |
+        apk add rsync
+        rsync -av --no-perms --no-owner --no-group --progress /sync/local_files/${SYNC_FOLDER}/ /sync/remote_files/${SYNC_FOLDER} -n
+
+
+volumes:
+  coss_smb_share:
+    driver: local
+    driver_opts:
+      type: cifs
+      o: "addr=${NAS_HOST},nounix,file_mode=0777,dir_mode=0777,domain=D,username=${NAS_USERNAME},password=${NAS_PASSWORD}"
+      device: //${NAS_HOST}${NAS_PATH}

launch

@@ -1,70 +0,0 @@
-#!/bin/bash
-set -e
-set -o ignoreeof
-
-echo "Bash script launching COSS_ARCHIVING..."
-
-
-# CHANGE ME ONCE!
-export CONTAINER_DATA=/mnt/media/@Bulk/COSS/Downloads/coss_archiving
-export UNAME=remy
-export U_ID=1000
-
-
-### Main use cases ###
-if [[ $1 == "debug" ]]
-then
-    export DEBUG=true
-    export HEADFULL=true
-    export CODE=./
-    export ENTRYPOINT=/bin/bash
-    # since service ports does not open ports on implicitly started containers, also start geckodriver:
-    docker compose up -d geckodriver
-
-elif [[ $1 == "production" ]]
-then
-    export DEBUG=false
-
-elif [[ $1 == "build" ]]
-then
-    export DEBUG=false
-    shift
-    docker compose build "$@"
-    exit 0
-
-
-### Manual Shutdown ###
-elif [[ $1 == "down" ]]
-then
-    docker compose down -t 0
-    exit 0
-
-
-
-### Edge cases -> for firefox ###
-elif [[ $1 == "edit_profile" ]]
-then
-    export CODE=./
-    export HEADFULL=true
-
-    docker compose up -d geckodriver
-    sleep 5
-    docker compose exec  geckodriver /bin/bash /code/geckodriver/edit_profile.sh # inside the container
-    docker compose down -t 0
-
-
-### Fallback ####
-else
-    echo "Please specify the execution mode (debug/production/build/edit_profile/down) as the first argument"
-    exit 1
-fi
-
-
-
-shift # consumes the variable set in $1 so that $@ only contains the remaining arguments
-
-docker compose run -it --service-ports "$@"
-
-echo "Docker run finished, shutting down containers..."
-docker compose down -t 0
-echo "Bye!"

manual/gather_media_files.py

@@ -2,66 +2,69 @@
 Runs the news_fetch pipeline against a manually curated list of urls and saves them locally
 """
 import sys
-sys.path.append("../app/news_fetch")
+sys.path.append("../news_fetch")
 import runner
+import os
 import logging
 logger = logging.getLogger()
-import json
 
-from rich.console import Console
-from rich.table import Table
-console = Console()
 
-logger.info("Overwriting production values for single time media-fetch")
-runner.configuration.models.set_db(
-    runner.configuration.SqliteDatabase("../.dev/media_downloads.db")
-)
-runner.configuration.main_config["DOWNLOADS"]["local_storage_path"] = "../.dev/"
+class DummyMessage:
+    """Required by the dispatcher"""
+    ts = 0
+    def __init__(self, url):
+        self.urls = [url]
 
 
 def fetch():
     dispatcher = runner.Dispatcher()
 
-    dispatcher.workers_in = [{"FetchWorker": runner.FetchWorker(), "DownloadWorker": runner.DownloadWorker()}]
-    dispatcher.workers_out = [{"PrintWorker": runner.PrintWorker()}]
+    dispatcher.workers_in = [
+        {"FetchWorker": runner.FetchWorker(), "DownloadWorker": runner.DownloadWorker()},
+        {"UploadWorker": runner.UploadWorker()}
+    ]
+    print_worker = runner.PrintWorker("Finished processing", sent = True)
+    dispatcher.workers_out = [{"PrintWorker": print_worker}]
 
     dispatcher.start()
 
-    with open("media_urls.json", "r") as f:
-        url_list = json.loads(f.read()) 
 
-    logger.info(f"Found {len(url_list)} media urls")
-    for u in url_list:
-        msg_text = f"<{u}|dummy preview text>"
-        dispatcher.incoming_request(msg)
+    with open("media_urls.txt", "r") as f:
+        url_list = [l.replace("\n", "") for l in f.readlines()]
+    with open("media_urls.txt", "w") as f:
+        f.write("") # empty the file once it is read so that it does not get processed again
 
+    if url_list:
+        logger.info(f"Found {len(url_list)} media urls")
+        for u in url_list:
+            dispatcher.incoming_request(DummyMessage(u))
+    else:
+        logger.info(f"No additional media urls found. Running the pipeline with messages from db.")
+
+    print_worker.keep_alive()
 
 
 def show():
+    for a in runner.models.ArticleDownload.select():
+        print(f"""
+        URL: {a.article_url}
+        ARCHIVE_URL: {a.archive_url}
+        ARTICLE_SOURCE: {a.source_name}
+        FILE_NAME: {a.file_name}
+        """)
 
-    t = Table(
-        title = "ArticleDownloads",
-        row_styles = ["white", "bright_black"],
+
+if __name__ == "__main__":
+    logger.info("Overwriting production values for single time media-fetch")
+    if not os.path.exists("../.dev/"):
+        os.mkdir("../.dev/")
+    runner.configuration.models.set_db(
+        runner.configuration.SqliteDatabase("../.dev/media_downloads.db")
     )
-    
-    entries = ["title", "article_url", "archive_url", "authors"]
-
-    for e in entries:
-        t.add_column(e, justify = "right")
-
-    sel = runner.models.ArticleDownload.select()
-
-    for s in sel:
-        c = [getattr(s, e) for e in entries]#
-        c[-1] = str([a.author for a in c[-1]])
-        print(c)
-        t.add_row(*c)
-
-    
-    console.print(t)
+    runner.configuration.main_config["downloads"]["local_storage_path"] = "../.dev/"
 
 
-
-
-# fetch()
-show()
+    if len(sys.argv) == 1: # no additional arguments
+        fetch()
+    elif sys.argv[1] == "show":
+        show()

nas_sync/Dockerfile

@@ -1,9 +0,0 @@
-FROM bash:latest
-# alpine with bash instead of sh
-ENV TZ=Europe/Berlin
-RUN apk add lsyncd cifs-utils rsync
-RUN mkdir -p /sync/remote_files
-COPY entrypoint.sh /sync/entrypoint.sh
-
-
-ENTRYPOINT ["bash", "/sync/entrypoint.sh"]

nas_sync/entrypoint.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-set -e
-
-sleep 5 # waits for the vpn to have an established connection
-echo "Starting NAS sync"
-mount -t cifs "//$1" -o credentials=/sync/nas_login.config /sync/remote_files
-echo "Successfully mounted SAMBA remote: $1 --> /sync/remote_files"
-shift # consumes the variable set in $1 so tat $@ only contains the remaining arguments
-
-exec "$@"

news_check/requirements.txt

@@ -1,4 +1,5 @@
 flask
 peewee
 markdown
-psycopg2
+psycopg2
+pyyaml

news_check/server/configuration.py

@@ -1,17 +1,16 @@
 from peewee import PostgresqlDatabase
-import configparser
 import time
+import yaml
+import os
 
-main_config = configparser.ConfigParser()
-main_config.read("/app/containerdata/config/news_fetch.config.ini")
+config_location = os.getenv("CONFIG_FILE")
+with open(config_location, "r") as f:
+    config = yaml.safe_load(f)
 
-db_config = configparser.ConfigParser()
-db_config.read("/app/containerdata/config/db.config.ini")
-
-cred = db_config["DATABASE"]
+cred = config["database"]
 time.sleep(10) # wait for the vpn to connect (can't use a healthcheck because there is no depends_on)
 db = PostgresqlDatabase(
-    cred["db_name"], user=cred["user_name"], password=cred["password"], host="vpn", port=5432
+    cred["production_db_name"], user=cred["production_user_name"], password=cred["production_password"], host="vpn", port=5432
 )
 
 import models

news_check/server/models.py

@@ -6,7 +6,7 @@ import os
 import datetime
 import configuration
 
-config = configuration.main_config["DOWNLOADS"]
+downloads_config = configuration.config["downloads"]
 
 # set the nature of the db at runtime
 download_db = DatabaseProxy()
@@ -34,14 +34,14 @@ class ArticleDownload(DownloadBaseModel):
     file_name = TextField(default = '')
     @property
     def save_path(self):
-        return f"{config['local_storage_path']}/{self.download_date.year}/{self.download_date.strftime('%B')}/"
+        return f"{downloads_config['local_storage_path']}/{self.download_date.year}/{self.download_date.strftime('%B')}/"
     @property
     def fname_nas(self, file_name=""):
         if self.download_date:
             if file_name:
-                return f"NAS: {config['remote_storage_path']}/{self.download_date.year}/{self.download_date.strftime('%B')}/{file_name}"
+                return f"NAS: {downloads_config['remote_storage_path']}/{self.download_date.year}/{self.download_date.strftime('%B')}/{file_name}"
             else: # return the self. name
-                return f"NAS: {config['remote_storage_path']}/{self.download_date.year}/{self.download_date.strftime('%B')}/{self.file_name}"
+                return f"NAS: {downloads_config['remote_storage_path']}/{self.download_date.year}/{self.download_date.strftime('%B')}/{self.file_name}"
         else:
             return None
 

news_fetch/configuration.py

@@ -1,9 +1,7 @@
-import os
-import configparser
-import logging
 import time
-# import shutil
-# from datetime import datetime
+import os
+import logging
+import yaml
 from peewee import SqliteDatabase, PostgresqlDatabase
 from rich.logging import RichHandler
 
@@ -19,22 +17,21 @@ logger = logging.getLogger(__name__)
 
 
 # load config file containing constants and secrets
-main_config = configparser.ConfigParser()
-main_config.read("/app/containerdata/config/news_fetch.config.ini")
-db_config = configparser.ConfigParser()
-db_config.read("/app/containerdata/config/db.config.ini")
+config_location = os.getenv("CONFIG_FILE")
+with open(config_location, "r") as f:
+    config = yaml.safe_load(f)
 
 
 # DEBUG MODE:
 if os.getenv("DEBUG", "false") == "true":
     logger.warning("Found 'DEBUG=true', setting up dummy databases")
     
-    main_config["SLACK"]["archive_id"] = main_config["SLACK"]["debug_id"]
-    main_config["MAIL"]["recipient"] = main_config["MAIL"]["sender"]
-    main_config["DOWNLOADS"]["local_storage_path"] = main_config["DOWNLOADS"]["debug_storage_path"]
+    config["slack"]["archive_id"] = config["slack"]["debug_id"]
+    config["mail"]["recipient"] = config["mail"]["sender"]
+    config["downloads"]["local_storage_path"] = config["downloads"]["debug_storage_path"]
 
     download_db = SqliteDatabase(
-        main_config["DATABASE"]["download_db_debug"],
+        config["database"]["debug_db"],
         pragmas = {'journal_mode': 'wal'} # mutliple threads can read at once
     )
 
@@ -43,9 +40,9 @@ else:
     logger.warning("Found 'DEBUG=false' and running on production databases, I hope you know what you're doing...")
     
     time.sleep(10) # wait for the vpn to connect (can't use a healthcheck because there is no depends_on)
-    cred = db_config["DATABASE"]
+    cred = config["database"]
     download_db = PostgresqlDatabase(
-        cred["db_name"], user=cred["user_name"], password=cred["password"], host="vpn", port=5432
+        cred["production_db_name"], user=cred["production_user_name"], password=cred["production_password"], host="vpn", port=5432
     )
     # TODO Reimplement backup/printout
     # logger.info("Backing up databases")

news_fetch/requirements.txt

@@ -8,4 +8,6 @@ newspaper3k
 htmldate
 markdown
 rich
-psycopg2
+psycopg2
+unidecode
+pyyaml

news_fetch/runner.py

@@ -128,8 +128,14 @@ class Dispatcher(Thread):
 
 
 class PrintWorker:
+    def __init__(self, action, sent = False) -> None:
+        self.action = action
+        self.sent = sent
     def send(self, article):
-        print(f"Uploaded article {article}")
+        print(f"{self.action} article {article}")
+        if self.sent:
+            article.sent = True
+            article.save()
     def keep_alive(self): # keeps script running, because there is nothing else in the main thread
         while True: sleep(1)
 
@@ -144,11 +150,12 @@ if __name__ == "__main__":
         logger.info(f"Launching upload to archive for {len(articles)} articles.")
 
         dispatcher.workers_in = [{"UploadWorker": UploadWorker()}]
-        dispatcher.workers_out = [{"PrintWorker": PrintWorker()}]
+        print_worker = PrintWorker("Uploaded")
+        dispatcher.workers_out = [{"PrintWorker": print_worker}]
         dispatcher.start()
         for a in articles:
             dispatcher.incoming_request(article=a)
-        PrintWorker().keep_alive()
+        print_worker.keep_alive()
 
     else: # launch with full action
         try:

news_fetch/utils_mail/runner.py

@@ -7,16 +7,20 @@ import logging
 import configuration
 
 logger = logging.getLogger(__name__)
-config = configuration.main_config["MAIL"]
+mail_config = configuration.config["mail"]
 
 def send(article_model):
     mail = MIMEMultipart()
     mail['Subject'] = "{} -- {}".format(article_model.source_name, article_model.title)
-    mail['From'] = config["sender"]
-    mail['To'] = config["recipient"]
-
-    msg, files = article_model.mail_info() # this is html
+    mail['From'] = mail_config["sender"]
+    mail['To'] = mail_config["recipient"]
 
+    try:
+        msg, files = article_model.mail_info() # this is html
+    except: # Raised by model if article has no associated file
+        logger.info("Skipping mail sending")
+        return
+        
     content = MIMEText(msg, "html")
     mail.attach(content)
 
@@ -29,14 +33,14 @@ def send(article_model):
 
     try:
         try:
-            smtp = smtplib.SMTP(config["smtp_server"], config["port"])
+            smtp = smtplib.SMTP(mail_config["smtp_server"], mail_config["port"])
         except ConnectionRefusedError:
             logger.error("Server refused connection. Is this an error on your side?")
             return False
 
         smtp.starttls()
-        smtp.login(config["uname"], config["password"])
-        smtp.sendmail(config["sender"], config["recipient"], mail.as_string())
+        smtp.login(mail_config["uname"], mail_config["password"])
+        smtp.sendmail(mail_config["sender"], mail_config["recipient"], mail.as_string())
         smtp.quit()
         logger.info("Mail successfully sent.")
     except smtplib.SMTPException as e:

news_fetch/utils_slack/runner.py

@@ -7,7 +7,7 @@ import re
 import time
 
 import configuration
-config = configuration.main_config["SLACK"]
+slack_config = configuration.config["slack"]
 models = configuration.models
 
 class MessageIsUnwanted(Exception):
@@ -61,7 +61,7 @@ class Message:
 
     @property
     def is_by_human(self):
-        return self.user.user_id != config["bot_id"]
+        return self.user.user_id != slack_config["bot_id"]
 
     
     @property
@@ -87,7 +87,7 @@ class BotApp(App):
 
     def say_substitute(self, *args, **kwargs):
         self.client.chat_postMessage(
-            channel=config["archive_id"],
+            channel=slack_config["archive_id"],
             text=" - ".join(args),
             **kwargs
         )
@@ -101,7 +101,7 @@ class BotApp(App):
             last_ts = presaved.slack_ts_full
 
         result = self.client.conversations_history(
-            channel=config["archive_id"],
+            channel=slack_config["archive_id"],
             oldest=last_ts
         )
 
@@ -116,7 +116,7 @@ class BotApp(App):
         while refetch: # we have not actually fetched them all
             try:
                 result = self.client.conversations_history(
-                    channel = config["archive_id"],
+                    channel = slack_config["archive_id"],
                     cursor = result["response_metadata"]["next_cursor"],
                     oldest = last_ts
                 ) # fetches 100 messages, older than the [-1](=oldest) element of new_fetches
@@ -126,8 +126,8 @@ class BotApp(App):
                 for m in new_messages:
                     return_messages.append(Message(m))
             except SlackApiError: # Most likely a rate-limit
-                self.logger.error("Error while fetching channel messages. (likely rate limit) Retrying in {} seconds...".format(config["api_wait_time"]))
-                time.sleep(config["api_wait_time"])
+                self.logger.error("Error while fetching channel messages. (likely rate limit) Retrying in {} seconds...".format(slack_config["api_wait_time"]))
+                time.sleep(slack_config["api_wait_time"])
                 refetch = True
         
         self.logger.info(f"Fetched {len(return_messages)} new channel messages.")
@@ -181,7 +181,7 @@ class BotRunner():
 
     """Stupid encapsulation so that we can apply the slack decorators to the BotApp"""
     def __init__(self, callback, *args, **kwargs) -> None:
-        self.bot_worker = BotApp(callback, token=config["auth_token"])
+        self.bot_worker = BotApp(callback, token=slack_config["auth_token"])
 
         @self.bot_worker.event(event="message", matchers=[is_message_in_archiving])
         def handle_incoming_message(message, say):
@@ -195,7 +195,7 @@ class BotRunner():
         def handle_all_other_reactions(event, say):
             self.logger.log("Ignoring slack event that isn't a message")
 
-        self.handler = SocketModeHandler(self.bot_worker, config["app_token"])
+        self.handler = SocketModeHandler(self.bot_worker, slack_config["app_token"])
 
 
     def start(self):
@@ -215,5 +215,5 @@ class BotRunner():
 
 
 def is_message_in_archiving(message) -> bool:
-    return message["channel"] == config["archive_id"]
+    return message["channel"] == slack_config["archive_id"]
 

news_fetch/utils_storage/helpers.py

@@ -1,7 +1,11 @@
+import unidecode
+KEEPCHARACTERS = (' ','.','_', '-')
+
 def clear_path_name(path):
-    keepcharacters = (' ','.','_', '-')
-    converted = "".join([c if (c.isalnum() or c in keepcharacters) else "_" for c in path]).rstrip()
-    return converted
+    path = unidecode.unidecode(path) # remove umlauts, accents and others
+    path = "".join([c if (c.isalnum() or c in KEEPCHARACTERS) else "_" for c in path]) # remove all non-alphanumeric characters
+    path = path.rstrip() # remove trailing spaces
+    return path
 
 def shorten_name(name, offset = 50):
     if len(name) > offset:

news_fetch/utils_storage/models.py

@@ -8,8 +8,7 @@ import configuration
 import datetime
 
 from . import helpers
-config = configuration.main_config["DOWNLOADS"]
-slack_config = configuration.main_config["SLACK"]
+downloads_config = configuration.config["downloads"]
 FILE_SIZE_THRESHOLD = 15 * 1024 * 1024 # 15MB
 
 
@@ -34,7 +33,8 @@ class ArticleDownload(DownloadBaseModel):
     def is_title_bad(self):  # add incrementally
         return "PUR-Abo" in self.title \
             or "Redirecting" in self.title \
-            or "Error while running fetch" in self.title
+            or "Error while running fetch" in self.title \
+            or self.title == ""
 
     summary = TextField(default = '')
     source_name = CharField(default = '')
@@ -44,14 +44,14 @@ class ArticleDownload(DownloadBaseModel):
     file_name = TextField(default = '')
     @property
     def save_path(self):
-        return f"{config['local_storage_path']}/{self.download_date.year}/{self.download_date.strftime('%B')}/"
+        return f"{downloads_config['local_storage_path']}/{self.download_date.year}/{self.download_date.strftime('%B')}/"
     @property
     def fname_nas(self, file_name=""):
         if self.download_date:
             if file_name:
-                return f"NAS: {config['remote_storage_path']}/{self.download_date.year}/{self.download_date.strftime('%B')}/{file_name}"
+                return f"NAS: {downloads_config['remote_storage_path']}/{self.download_date.year}/{self.download_date.strftime('%B')}/{file_name}"
             else: # return the self. name
-                return f"NAS: {config['remote_storage_path']}/{self.download_date.year}/{self.download_date.strftime('%B')}/{self.file_name}"
+                return f"NAS: {downloads_config['remote_storage_path']}/{self.download_date.year}/{self.download_date.strftime('%B')}/{self.file_name}"
         else:
             return None
     @property
@@ -102,18 +102,22 @@ class ArticleDownload(DownloadBaseModel):
         answer_files = []
         # displays the summary in a blockquote
 
-        status = self.file_status
-        if status == 1: # file_name was empty
-            return None # there has been an error do not send any message
-        elif status == 2: # no file found at specified location
-            answer_text += f"*{self.title}*\n{summary}\nFilename: {self.file_name}"
-        elif status == 3: # file found but deemed too big
-            location = f"File not sent directly. Location on NAS:\n`{self.fname_nas}`"
-            answer_text += f"*{self.title}*\n{summary}\n{location}"
-        else: # everything nominal
+        try:
+            self.ensure_file_present()
             answer_text += f"*{self.title}*\n{summary}"
             answer_files.append(self.save_path + self.file_name)
 
+        except Exception as e:
+            msg = e.args[0]
+            logger.error(f"Article {self} has file-issues: {msg}")
+            if "file too big" in msg:
+                location = f"File too big to send directly. Location on NAS:\n`{self.fname_nas}`"
+                answer_text += f"*{self.title}*\n{summary}\n{location}"
+                
+            else: # file not found, or filename not set
+                raise e
+                # reraise the exception, so that the caller can handle it
+
         # then the related files
         if self.related:
             rel_text = "Related files on NAS:"
@@ -144,19 +148,14 @@ class ArticleDownload(DownloadBaseModel):
                 related_file_name = r
             )
     
-    @property
-    def file_status(self):
-        """0 = file exists, 1 = no file name!, 2 = file does not exit,3 = file exists but is too large"""
+    def ensure_file_present(self):
         if not self.file_name:
-            logger.error(f"Article {self} has no filename!")
-            return 2
+            raise Exception("no filename")
         file_path_abs = self.save_path + self.file_name
         if not os.path.exists(file_path_abs):
-            logger.error(f"Article {self} has a filename, but the file does not exist at that location!")
-            return 2
+            raise Exception("file not found")
         if (os.path.splitext(file_path_abs)[1] != ".pdf") or (os.path.getsize(file_path_abs) > FILE_SIZE_THRESHOLD):
-            logger.warning(f"Article {self} has a file that exceeds the file size limit.")
-            return 3
+            raise Exception("file too big")
 
 
 

news_fetch/utils_worker/download/browser.py

@@ -11,7 +11,7 @@ from selenium import webdriver
 
 import configuration
 
-config = configuration.main_config["DOWNLOADS"]
+download_config = configuration.config["downloads"]
 
 def driver_running(f):
     def wrapper(*args, **kwargs):
@@ -66,74 +66,88 @@ class PDFDownloader:
 
     @driver_running
     def download(self, article_object):
-        sleep_time = int(config["browser_print_delay"])
         url = article_object.article_url
 
+
+        if url[-4:] == ".pdf": # calling the ususal pdf generation would not yield a nice pdf, just download it directly
+            self.logger.info("Downloading existing pdf")
+            success = self.get_exisiting_pdf(article_object)
+            # get a page title if required
+            if article_object.is_title_bad:
+                article_object.title = self.driver.title.replace(".pdf", "") # some titles end with .pdf
+                # will be propagated to the saved file (dst) as well
+        else:
+            success = self.get_new_pdf(article_object)
+
+        if not success:
+            self.logger.error("Download failed")
+        # TODO: need to reset the file name to empty?
+        return article_object # changes to this are saved later by the external caller
+
+
+    def get_exisiting_pdf(self, article_object):
+        # get a better page title if required
+        if article_object.is_title_bad:
+            article_object.title = article_object.article_url.split("/")[-1].split(".pdf")[0]
         try:
-            self.driver.get(url)
+            r = requests.get(article_object.article_url)
+            bytes = r.content
+        except:
+            return False
+        return self.write_pdf(bytes, article_object)
+
+
+    def get_new_pdf(self, article_object):
+        sleep_time = int(download_config["browser_print_delay"])
+
+        try:
+            self.driver.get(article_object.article_url)
         except Exception as e:
             self.logger.critical("Selenium .get(url) failed with error {}".format(e))
             self.finish()
-            return article_object  # without changes
+            return False
         
         time.sleep(sleep_time)
         # leave the page time to do any funky business
 
-        # in the mean time, get a page title if required
         if article_object.is_title_bad:
-            article_object.title = self.driver.title.replace(".pdf", "") # some titles end with .pdf
-            # will be propagated to the saved file (dst) as well
+            article_object.title = self.driver.title
 
+        try:
+            result = self.driver.print_page()
+            bytes = base64.b64decode(result, validate=True)
+        except:
+            self.logger.error("Failed, probably because the driver went extinct.")
+            return False
+
+        return self.write_pdf(bytes, article_object)
+
+
+    def get_file_destination(self, article_object):
         fname = article_object.fname_template
         fname = ensure_unique(article_object.save_path, fname)
         dst = os.path.join(article_object.save_path, fname)
+        return dst, fname
 
 
-        if url[-4:] == ".pdf": # calling the ususal pdf generation would not yield a nice pdf, just download it directly
-            success = self.get_exisiting_pdf(url, dst)
-        else:
-            success = self.get_new_pdf(dst)
-
-        if success:
-            article_object.file_name = fname
-        else:
-            article_object.file_name = ""
-        
-        return article_object # this change is saved later by the external caller
-
-
-    def get_exisiting_pdf(self, url, dst):
-        try:
-            r = requests.get(url)
-            bytes = r.content
-        except:
-            return False
-        return self.get_new_pdf(dst, other_bytes=bytes)
-
-
-    def get_new_pdf(self, dst, other_bytes=None):
+    def write_pdf(self, content, article_object):
+        dst, fname = self.get_file_destination(article_object)
         os.makedirs(os.path.dirname(dst), exist_ok=True)
-
-        if other_bytes is None:
-            try:
-                result = self.driver.print_page()
-                bytes = base64.b64decode(result, validate=True)
-            except:
-                self.logger.error("Failed, probably because the driver went extinct.")
-                return False
-        else:
-            bytes = other_bytes
-
+        
         try:
             with open(dst, "wb+") as f:
-                f.write(bytes)
+                f.write(content)
+            
+            article_object.file_name = fname
             return True
         except Exception as e:
             self.logger.error(f"Failed, because of FS-operation: {e}")
             return False
 
 
-    def create_tmp_profile(self, full_profile_path: Path = Path(config["browser_profile_path"])) -> Path:
+
+        
+    def create_tmp_profile(self, full_profile_path: Path = Path(download_config["browser_profile_path"])) -> Path:
         reduced_profile_path = Path(f"/tmp/firefox_profile_{uuid.uuid4()}")
         os.mkdir(reduced_profile_path)
         # copy needed directories

news_fetch/utils_worker/download/youtube.py

@@ -1,10 +1,11 @@
 import youtube_dl
 import os
 import logging
+import configuration
 
+download_config = configuration.config["downloads"]
 logger = logging.getLogger(__name__)
 
-
 class MyLogger(object):
     def debug(self, msg): pass
     def warning(self, msg): pass
@@ -19,7 +20,6 @@ class YouTubeDownloader:
 
 
     def post_download_hook(self, ret_code):
-        # print(ret_code)
         if ret_code['status'] == 'finished':
             file_loc = ret_code["filename"]
             fname = os.path.basename(file_loc)
@@ -35,9 +35,11 @@ class YouTubeDownloader:
         ydl_opts = {
             'format': 'best[height<=720]',
             'outtmpl': f"{file_path}.%(ext)s", # basically the filename from the object, but with a custom extension depending on the download
-            'logger': MyLogger(),
+            'logger': MyLogger(), # supress verbosity
             'progress_hooks': [self.post_download_hook],
-            'updatetime': False
+            'updatetime': False,
+            # File is also used by firefox so make sure to not write to it!
+            # youtube dl apparenlty does not support cookies.sqlite and the documentation is not clear on how to use cookies.txt
         }
         try:
             with youtube_dl.YoutubeDL(ydl_opts) as ydl:
@@ -46,5 +48,9 @@ class YouTubeDownloader:
         except Exception as e:
             logger.error(f"Youtube download crashed: {e}")
             article_object.file_name = ""
+            logfile = os.path.join(download_config["local_storage_path"], "failed_downloads.csv")
+            logger.info(f"Logging youtube errors seperately to {logfile}")
+            with open(logfile, "a+") as f:
+                f.write(f"{url}\n")
 
         return article_object