From d3d44dcdc9818875e7049c770fb8adaba4357387 Mon Sep 17 00:00:00 2001 From: Remy Moll Date: Fri, 9 Sep 2022 22:32:22 +0200 Subject: [PATCH] working (feature complete) news_fetch --- news_check/client/public/favicon.png | Bin 3127 -> 0 bytes news_check/client/public/global.css | 63 ---------- news_check/client/src/App.svelte | 21 +++- .../client/src/ArticleOperations.svelte | 116 +++++++++++------- news_check/client/src/ArticleStatus.svelte | 16 ++- news_check/client/src/Toast.svelte | 34 +++++ news_check/client/src/main.js | 3 - news_check/server/app.py | 45 +++++-- news_check/server/configuration.py | 2 + news_fetch/configuration.py | 4 +- news_fetch/runner.py | 10 +- 11 files changed, 177 insertions(+), 137 deletions(-) delete mode 100644 news_check/client/public/favicon.png delete mode 100644 news_check/client/public/global.css create mode 100644 news_check/client/src/Toast.svelte diff --git a/news_check/client/public/favicon.png b/news_check/client/public/favicon.png deleted file mode 100644 index 7e6f5eb5a2f1f1c882d265cf479de25caa925645..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3127 zcmV-749N3|P)i z7)}s4L53SJCkR}iVi00SFk;`MXX*#X*kkwKs@nFGS}c;=?XFjU|G$3t^5sjIVS2G+ zw)WGF83CpoGXhLGW(1gW%uV|X7>1P6VhCX=Ux)Lb!*DZ%@I3!{Gsf7d?gtIQ%nQiK z3%(LUSkBji;C5Rfgd6$VsF@H`Pk@xtY6t<>FNR-pD}=C~$?)9pdm3XZ36N5PNWYjb z$xd$yNQR9N!dfj-Vd@BwQo^FIIWPPmT&sZyQ$v81(sCBV=PGy{0wltEjB%~h157*t zvbe_!{=I_783x!0t1-r#-d{Y?ae$Q4N_Nd^Ui^@y(%)Gjou6y<3^XJdu{rmUf-Me?)zZ>9OR&6U5H*cK; z$gUlB{g0O4gN0sLSO|Of?hU(l?;h(jA3uH!Z{EBKuV23ouU@^Y6#%v+QG;>e*E}%?wlu-NT4DG zs)z)7WbLr)vGAu(ohrKc^em@OpO&f~6_>E61n_e0_V3@{U3^O;j{`^mNCJUj_>;7v zsMs6Hu3g7+@v+lSo;=yTYFqq}jZmQ-BK8K{C4kqi_i*jBaQE(Au0607V-zKeT;EPg zX(`vrn=L+e74+-Tqeok@_`tDa$G9I|$nTU5H*2V8@y()n*zqM?J1G!-1aX;CfDC9B zTnJ#j_%*n8Qb1)re*Bno7g0RG{Eb;IK14irJYJp$5Z6ac9~b_P?+5t~95~SRG$g?1 znFJ7p$xV&GZ18m~79TGRdfsc-BcX$9yXTR*n)mPD@1~O(_?cT$ZvFPucRmGlq&se0 zKrcUf^k}4hM*biEJOWKzz!qQe;CB_ZtSOO9Owg#lZAc=s65^rb{fZe(TYu_rk!wKkEf}RIt=#Om( zR8mN`DM<^xj~59euMMspBolVN zAPTr8sSDI104orIAdmL$uOXn*6hga1G+0WD0E?UtabxC#VC~vf3|10|phW;yQ3CY8 z2CM=)ErF;xq-YJ5G|um}>*1#E+O_Mu|Nr#qQ&G1P-NMq@f?@*XUcSbV?tX=)ilM-Q zBZP|!Bpv0V;#ojKcpc7$=eqO;#Uy~#?^kNI{vSZfLx&DEt~LTmaKWXcx=joubklI<*Aw z>LtMaQ7DR<1I2LkWvwyu#Rwn~;ezT}_g(@5l3h?W%-a86Y-t#O1PubP+z<%?V5D(U zy57A6{h+{?kOZp7&WKZR+=sznMJ}+Dnpo=C_0%R_x_t~J5T?E_{+))l5v1%52>)d-`iiZyx|5!%M2Fb2dU zW3~MwwpEH9Rhue+k$UIOoo($Ds!NbOyMR36fRHu;*15(YcA7siIZk#%JWz>P!qX1?IUojG&nKR>^gArBt2 zit(ETyZ=@V&7mv_Fi4bABcnwP+jzQuHcfU&BrAV91u-rFvEi7y-KnWsvHH=d2 zgAk(GKm_S8RcTJ>2N3~&Hbwp{Z3NF_Xeh}g4Eke)V&dY{W(3&b1j9t4yK_aYJisZZ{1rcU5- z;eD>K;ndPq&B-8yA_S0F!4ThA&{1{x)H<#?k9a#6Pc6L?V^s0``ynL&D;p(!Nmx`Y zFkHex{4p!Ggm^@DlehW}iHHVi}~u=$&N? z(NEBLQ#UxxAkdW>X9LnqUr#t4Lu0=9L8&o>JsqTtT5|%gb3QA~hr0pED71+iFFr)dZ=Q=E6ng{NE{Z~0)C?deO#?Aj zSDQ$z#TeC2T^|=}6GBo-&$;E{HL3!q3Z-szuf)O=G#zDjin4SSP%o%6+2IT#sLjQa ziyxFFz~LMjWY+_a5H!U6%a<=b7QVP^ z*90a62;bVq{?@)P6^DWd^Yilq4|YTV2Nw!Yu;a1lPI-sxR)rf@Fe5DhDP7FH zZZ%4S*1C30P;|O+jB!1;m|rXT90Sm5*RBbQN`PKu+hDD*S^yE(CdtSfg=z>u$cIj> z { + let interfaceState = updateInterface() + + async function updateInterface () { let url = ''; if (current_id == 0) { url = '/api/article/first'; @@ -19,12 +24,14 @@ const article_response = await fetch(article_url); const article_data = await article_response.json(); return article_data; - })() - - + } + + function triggerUpdate () { + interfaceState = updateInterface(); + } -{#await updateInterface} +{#await interfaceState} ... {:then article_data}
@@ -33,7 +40,9 @@
- +
{/await} + + \ No newline at end of file diff --git a/news_check/client/src/ArticleOperations.svelte b/news_check/client/src/ArticleOperations.svelte index e1db46e..de55266 100644 --- a/news_check/client/src/ArticleOperations.svelte +++ b/news_check/client/src/ArticleOperations.svelte @@ -1,55 +1,85 @@ @@ -58,21 +88,22 @@

Your options: (click on action or use keyboard)

- + - {#each actions as action} - + {#each actions as action} - + - {/each}
Action Keyboard shortcut
{ action.kbd } + { action.kbd } + {#if action.comment}({ action.comment }){/if} +
@@ -80,14 +111,9 @@
+ + + + -{#if toast_visible} -
-
-
- { toast_state.text }. -
-
-
-{/if} \ No newline at end of file diff --git a/news_check/client/src/ArticleStatus.svelte b/news_check/client/src/ArticleStatus.svelte index 3ffb9db..6c426dc 100644 --- a/news_check/client/src/ArticleStatus.svelte +++ b/news_check/client/src/ArticleStatus.svelte @@ -2,13 +2,21 @@ export let article_data; const status_items = [ {name: 'Title', value: article_data.title}, + {name: 'Url', value: article_data.article_url}, + {name: 'Source', value: article_data.source_name}, {name: 'Filename', value: article_data.file_name}, + {name: 'Location', value: article_data.save_path}, {name: 'Language', value: article_data.language}, {name: 'Authors', value: article_data.authors}, {name: "Related", value: article_data.related}, ] +

Article overview:

@@ -23,16 +31,18 @@ {#each status_items as item} { item.name } - {#if item.value != ""} + {#if item.name == "Url"} + { item.value } + {:else} { item.value } + {/if} {:else} - { item.value } + not set {/if} {/each}
-
\ No newline at end of file diff --git a/news_check/client/src/Toast.svelte b/news_check/client/src/Toast.svelte new file mode 100644 index 0000000..0c13236 --- /dev/null +++ b/news_check/client/src/Toast.svelte @@ -0,0 +1,34 @@ + + +
+ {#each $toasts as toast} +
+
{ toast.text }.
+
+ {/each} +
diff --git a/news_check/client/src/main.js b/news_check/client/src/main.js index d6cacbb..d80e9a3 100644 --- a/news_check/client/src/main.js +++ b/news_check/client/src/main.js @@ -2,9 +2,6 @@ import App from './App.svelte'; const app = new App({ target: document.body, - props: { - name: 'world' - } }); export default app; \ No newline at end of file diff --git a/news_check/server/app.py b/news_check/server/app.py index 418d854..9d99796 100644 --- a/news_check/server/app.py +++ b/news_check/server/app.py @@ -1,5 +1,7 @@ from flask import Flask, send_from_directory, request +import os import configuration + models = configuration.models db = configuration.db app = Flask(__name__) @@ -30,9 +32,9 @@ def get_article_by_id(id): return article.to_dict() @app.route("/api/article/first") -def get_article_first(): +def get_article_first(min_id=0): with db: - article = models.ArticleDownload.select(models.ArticleDownload.id).where(models.ArticleDownload.verified == 0).order_by(models.ArticleDownload.id).first() + article = models.ArticleDownload.select(models.ArticleDownload.id).where((models.ArticleDownload.verified == 0) & (models.ArticleDownload.id > min_id)).order_by(models.ArticleDownload.id).first() return {"id" : article.id} @app.route("/api/article//next") @@ -41,27 +43,44 @@ def get_article_next(id): if models.ArticleDownload.get_by_id(id + 1).verified == 0: return {"id" : id + 1} else: - return get_article_first() + return get_article_first(min_id=id) # if the current article was skipped, but the +1 is already verified, get_first will return the same article again. so specify min id. @app.route("/api/article//set", methods=['POST']) def set_article(id): - action = request.json['action'] + try: + action = request.json.get('action', None) + except Exception as e: + print(f"Exception in set_article {e}") + action = None with db: article = models.ArticleDownload.get_by_id(id) - if action == "a": - article.verified = 1 - elif action == "b": - article.verified = -1 - elif action == "r": - article.set_related() - article.save() - return "ok" + if action: + if action == "a": + article.verified = 1 + elif action == "b": + article.verified = -1 + else: # implicitly action == "r": + print(request.files) + file = request.files.get("file", None) + if file is None: # upload tends to crash + return "No file uploaded", 400 + artname, _ = os.path.splitext(article.file_name) + fname = f"{artname} -- related_{article.related.count() + 1}.{file.filename.split('.')[-1]}" + fpath = os.path.join(article.save_path, fname) + print(fpath) + file.save(fpath) + article.set_related([fname]) + return {"file_path": fpath} + + article.save() + return "ok" if __name__ == "__main__": - app.run(host="0.0.0.0", port="80") + debug = os.getenv("DEBUG", "false") == "true" + app.run(host="0.0.0.0", port="80", debug=debug) diff --git a/news_check/server/configuration.py b/news_check/server/configuration.py index c6cd798..f3214da 100644 --- a/news_check/server/configuration.py +++ b/news_check/server/configuration.py @@ -1,5 +1,6 @@ from peewee import PostgresqlDatabase import configparser +import time main_config = configparser.ConfigParser() main_config.read("/app/containerdata/config/news_fetch.config.ini") @@ -8,6 +9,7 @@ db_config = configparser.ConfigParser() db_config.read("/app/containerdata/config/db.config.ini") cred = db_config["DATABASE"] +time.sleep(10) # wait for the vpn to connect (can't use a healthcheck because there is no depends_on) db = PostgresqlDatabase( cred["db_name"], user=cred["user_name"], password=cred["password"], host="vpn", port=5432 ) diff --git a/news_fetch/configuration.py b/news_fetch/configuration.py index 01eb52e..49ba39e 100644 --- a/news_fetch/configuration.py +++ b/news_fetch/configuration.py @@ -2,8 +2,8 @@ import os import configparser import logging import time -import shutil -from datetime import datetime +# import shutil +# from datetime import datetime from peewee import SqliteDatabase, PostgresqlDatabase from rich.logging import RichHandler diff --git a/news_fetch/runner.py b/news_fetch/runner.py index d3baf18..7e17cb4 100644 --- a/news_fetch/runner.py +++ b/news_fetch/runner.py @@ -1,4 +1,5 @@ """Main coordination of other util classes. Handles inbound and outbound calls""" +from time import sleep import configuration models = configuration.models from threading import Thread @@ -110,7 +111,8 @@ class Dispatcher(Thread): logger.error("Dispatcher.incoming_request called with no arguments") return - if is_new or (article.file_name == "" and article.verified == 0): + if is_new or (article.file_name == "" and article.verified == 0) \ + or (not is_new and len(self.workers_in) == 1): # this is for upload # check for models that were created but were abandonned. This means they have missing information, most importantly no associated file # this overwrites previously set information, but that should not be too important ArticleWatcher( @@ -121,7 +123,6 @@ class Dispatcher(Thread): else: # manually trigger notification immediatly logger.info(f"Found existing article {article}. Now sending") - self.article_complete_notifier(article) @@ -142,6 +143,8 @@ if __name__ == "__main__": class PrintWorker: def send(self, article): print(f"Uploaded article {article}") + def keep_alive(self): # keeps script running, because there is nothing else in the main thread + while True: sleep(1) articles = models.ArticleDownload.select().where(models.ArticleDownload.archive_url == "" or models.ArticleDownload.archive_url == "TODO:UPLOAD").execute() logger.info(f"Launching upload to archive for {len(articles)} articles.") @@ -149,6 +152,9 @@ if __name__ == "__main__": dispatcher.workers_in = [{"UploadWorker": UploadWorker()}] dispatcher.workers_out = [{"PrintWorker": PrintWorker()}] dispatcher.start() + for a in articles: + dispatcher.incoming_request(article=a) + PrintWorker().keep_alive() else: # launch with full action try: