From 79e3f54955450d462f4175f513f3aff5bad404d0 Mon Sep 17 00:00:00 2001 From: Remy Moll Date: Sat, 25 Jun 2022 16:06:45 +0200 Subject: [PATCH] Better documentation, smoother checks --- README.md | 68 +++++++++------------------------------ app/utils_check/runner.py | 6 ++-- env/check | 5 ++- 3 files changed, 23 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index 0839c3d..8fac278 100644 --- a/README.md +++ b/README.md @@ -3,63 +3,28 @@ A utility to fetch article requests from slack and generate pdfs for them, fully automatically. -## Running - Pure docker -> I recommend running with docker compose instead -### How to run - auto archiving mode -In this mode the program is launched as a docker container, in a headless mode. For persistence purposes a local storage volume is required, but that's it! - -`docker run -it -v :/app/file_storage/ auto_news` - -You can specify additional parameters: - -`docker run -it -v :/app/file_storage/ auto_news debug` runs with debug values (does not write to prod db, does not send mails) - -`docker run -it -v :/app/file_storage/ auto_news upload` catches up on incomplete uploads to archive. - -`docker run -it -v :/app/file_storage/ auto_news reducedfetch` makes assumption about the status of the slack chat and greatly reduces the number of api calls (faster start up). - -These parameters can be combined (mostyl for testing I guess) - -Finally for manual file verification: - -`docker run -it -v :/app/file_storage/ -e DISPLAY=":0" --network host -v \$XAUTHORITY:/root/.Xauthority auto_news check` lets you visually verify the downloaded files. The additional parameters are required in order to open guis on the host. - - -### How to run - development mode -In this mode, a docker container is launched with an additional volume, the local code. You can test your code without the need to rebuild the image. - -`docker run -it -v :/app/file_storage/ -v :/code/ --entry-point /bin/bash auto_news` -You are droppped into a bash shell, in which you can navigate to the `/code` directory and then test live. - -### Cheat-sheet Remy: - -`docker run -it -v /mnt/Data/COSS/Downloads/auto_news.container/:/app/file_storage/ auto_news` - -`docker run -it -v /mnt/Data/COSS/Downloads/auto_news.container/:/app/file_storage/ -v /mnt/Data/COSS/Development/auto_news/app:/code --entrypoint /bin/bash auto_news` - - -`docker run -it -v /mnt/Data/COSS/Downloads/auto_news.container/:/app/file_storage/ -e DISPLAY=":0" --network host -v XAUTHORITY:/root/.Xauthority auto_news check` - - ## Running - Docker compose -I also wrote a rudimentary docker compose file which makes running much more simple. Just run +A rudimentary docker compose file makes for much simpler command linde calls. -`docker compose --env-file up` +* For normal, `production` mode and `upload` mode, run: -All relevant passthroughs and mounts are specified through the env-file, for which I configured 4 versions: production, debug (development in general), upload and check. These files will have to be adapted to your individual setup but can be reused more easily. + `docker compose --env-file env/ up` -For the debug env-file, you will likely want interactivity, so you need to run: + All relevant passthroughs and mounts are specified through the env-file, for which I configured 4 versions: production, debug (development in general), upload and check. These files will have to be adapted to your individual setup but can be reused more easily. -`docker compose --env-file env/debug up -d && docker compose --env-file env/debug exec auto_news bash && docker compose --env-file env/debug down` +* For `debug` mode, you will likely want interactivity, so you need to run: + `docker compose --env-file env/debug up -d && docker compose --env-file env/debug exec auto_news bash && docker compose --env-file env/debug down` - + which should automatically shutdown the containers once you are done. (`ctrl+d` to exit the container shell). If not, re-run `docker compose --env-file env/debug down` manually. + > Note: + > The live-mounted code is then under `/code`. Note that the `DEBUG=true` environment variable is still set. If you want to test things on production, run `export DEBUG=false`. Running `python runner.py` will now run the newly written code but, with the production database and storage. +* For `check` mode, some env-variables are also changed and you still require interactivity. You don't need the geckodriver service however. The simplest way is to run + +`docker compose --env-file env/check run auto_news` ## Building @@ -71,9 +36,9 @@ In docker, simply run: where the `Dockerfile` has to be in the working directory -In docker compose, run the usual command, but append +In docker compose, run -`docker compose ... up --build` +`docker compose --env-file env/production build` @@ -82,7 +47,6 @@ In docker compose, run the usual command, but append ## Roadmap: -[ ] automatically upload files to NAS +:x: automatically upload files to NAS -[ ] handle paywalled sites like faz, spiegel, .. through their dedicated edu-friendly sites -... \ No newline at end of file +:x: handle paywalled sites like faz, spiegel, .. through their dedicated edu-friendly sites diff --git a/app/utils_check/runner.py b/app/utils_check/runner.py index 10feab3..5af8420 100644 --- a/app/utils_check/runner.py +++ b/app/utils_check/runner.py @@ -35,7 +35,7 @@ def file_overview(file_url: str, file_attributes: list, options: dict) -> None: file_table = Table( title = file_url, row_styles = ["white", "bright_black"], - min_width = 150 + min_width = 100 ) file_table.add_column("Attribute", justify = "right", no_wrap = True) @@ -158,11 +158,11 @@ def verify_unchecked(): try: # close any previously opened windows: - subprocess.call(["kill", "`pgrep evince`"]) + # subprocess.call(["kill", "`pgrep evince`"]) + os.system("pkill evince") # then open a new one subprocess.Popen(["evince", f"file://{os.path.join(article.save_path, article.file_name)}"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) # supress evince gtk warnings - print("done") except Exception as e: print(e) continue diff --git a/env/check b/env/check index 3a40348..d9be696 100644 --- a/env/check +++ b/env/check @@ -10,4 +10,7 @@ DEBUG=false CHECK=true HEADLESS=true UPLOAD=false -REDUCEDFETCH=false \ No newline at end of file +REDUCEDFETCH=false + +# ENTRYPOINT="/bin/bash" +INTERACTIVE=true \ No newline at end of file