working (feature complete) news_fetch

news_check/client/public/global.css

@@ -1,63 +0,0 @@
-html, body {
-	position: relative;
-	width: 100%;
-	height: 100%;
-}
-
-body {
-	color: #333;
-	margin: 0;
-	padding: 8px;
-	box-sizing: border-box;
-	font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen-Sans, Ubuntu, Cantarell, "Helvetica Neue", sans-serif;
-}
-
-a {
-	color: rgb(0,100,200);
-	text-decoration: none;
-}
-
-a:hover {
-	text-decoration: underline;
-}
-
-a:visited {
-	color: rgb(0,80,160);
-}
-
-label {
-	display: block;
-}
-
-input, button, select, textarea {
-	font-family: inherit;
-	font-size: inherit;
-	-webkit-padding: 0.4em 0;
-	padding: 0.4em;
-	margin: 0 0 0.5em 0;
-	box-sizing: border-box;
-	border: 1px solid #ccc;
-	border-radius: 2px;
-}
-
-input:disabled {
-	color: #ccc;
-}
-
-button {
-	color: #333;
-	background-color: #f4f4f4;
-	outline: none;
-}
-
-button:disabled {
-	color: #999;
-}
-
-button:not(:disabled):active {
-	background-color: #ddd;
-}
-
-button:focus {
-	border-color: #666;
-}

news_check/client/src/App.svelte

@@ -2,10 +2,15 @@
 	import PDFView from './PDFView.svelte';
 	import ArticleStatus from './ArticleStatus.svelte';
 	import ArticleOperations from './ArticleOperations.svelte';
+	
+	import Toast from './Toast.svelte';
+
 
 	let current_id = 0; 
 	
-	const updateInterface = (async () => {
+	let interfaceState = updateInterface()
+	
+	async function updateInterface () {
 		let url = '';
 		if (current_id == 0) {
 			url = '/api/article/first';
@@ -19,12 +24,14 @@
 		const article_response = await fetch(article_url);
 		const article_data = await article_response.json();
 		return article_data;
-	})()
+	}
-	
+
-	
+	function triggerUpdate () {
+		interfaceState = updateInterface();
+	}
 </script>
 
-{#await updateInterface}
+{#await interfaceState}
 ...
 {:then article_data}
 	<div class="flex w-full h-screen gap-5 p-5">
@@ -33,7 +40,9 @@
 		<div class="w-2/5">
 			<ArticleStatus article_data={article_data}/>
 			<div class="divider divider-vertical"></div> 
-			<ArticleOperations article_data={article_data}/>
+			<ArticleOperations article_data={article_data} callback={triggerUpdate}/>
 		</div>
 	</div>
 {/await}
+
+<Toast/>

news_check/client/src/ArticleOperations.svelte

@@ -1,55 +1,85 @@
 <script>
-    import {fade} from 'svelte/transition';
-    
     export let article_data;
+    export let callback;
+    window.focus()  
+    import { addToast } from './Toast.svelte';
     
     const actions = [
       {name: 'Mark as good (and skip to next)', kbd: 'A'},
       {name: 'Mark as bad (and skip to next)', kbd: 'B'},
-      {name: 'Upload related file', kbd: 'R'},
+      {name: 'Upload related file', kbd: 'R', comment: "can be used multiple times"},
-      {name: 'Skip', kbd: 'ctrl'},
+      {name: 'Skip', kbd: 'S'},
     ]
 
-    const toast_states = {
-      'success' : {class: 'alert-success', text: 'Article updated successfully'},
-      'error' : {class: 'alert-error', text: 'Article update failed'},
-    }
-    let toast_state = {};
-    let toast_visible = false;
 
-    
+    let fileInput = document.createElement('input');
+    fileInput.type = 'file';
+
+    fileInput.onchange = e => {
+      let result = (async () => {
+        uploadRelatedFile(e.target.files[0]);
+      })()
+    }
+        
+
     function onKeyDown(e) {apiAction(e.key)}
     function apiAction(key) {
       if (actions.map(d => d.kbd.toLowerCase()).includes(key.toLowerCase())){ // ignore other keypresses
 
         const updateArticle = (async() => {
-          const response = await fetch('/api/article/' + article_data.id + '/set', {
+          let success
-            method: 'POST',
-            headers: {'Content-Type': 'application/json'},
-            body: JSON.stringify({
-              'action': key.toLowerCase(),
-            })
-          })
-          const success = response.status == 200;
-        
-          if (success){
-            showToast('success');
-          } else {
-            showToast('error');
-          }
 
+          if (key.toLowerCase() == "s") {
+            addToast('success', "Article skipped")
+            callback()
+            return
+          } else if (key.toLowerCase() == "r") {
+            fileInput.click() // this will trigger a change in fileInput, 
+            return
+          } else {
+            const response = await fetch('/api/article/' + article_data.id + '/set', {
+              method: 'POST',
+              headers: {'Content-Type': 'application/json'},
+              body: JSON.stringify({
+                'action': key.toLowerCase(),
+              })
+            })
+            success = response.status == 200  
+          }
+            if (success){
+              addToast('success')
+              callback()
+            } else {
+              addToast('error')
+            }
         })()
       }
     }
 
-    function showToast(state){  
-      toast_visible = true;
-      toast_state = toast_states[state];
-      setTimeout(() => {
-        toast_visible = false;
-      }, 1000)
 
+
+    async function uploadRelatedFile(file) {
+      const formData = new FormData()
+      formData.append('file', file)
+
+      const response = await fetch('/api/article/' + article_data.id + '/set', {
+        method: 'POST',
+        body : formData,
+      })
+      
+      const success =  response.status == 200;
+      if (success){
+        const data = await response.json()
+        let fname = data.file_path
+        addToast('success', "File uploaded as " + fname)
+      } else {
+        addToast('error', "File upload failed")
+      }
+      return success;
     }
+
+
+
 </script>
 
 
@@ -58,21 +88,22 @@
       <h2 class="card-title">Your options: (click on action or use keyboard)</h2>
       <div class="overflow-x-auto">
         <table class="table w-full table-compact">
-          <!-- head -->
           <thead>
             <tr>
                 <th>Action</th>
                 <th>Keyboard shortcut</th>
             </tr>
           </thead>
+
           <tbody>
-            {#each actions as action}
+            {#each actions as action}                
-                
                 <tr>
                     <td><button on:click={() => apiAction(action.kbd)}>{ action.name }</button></td>
-                    <td><kbd class="kbd">{ action.kbd }</kbd></td>
+                    <td><kbd class="kbd">
+                      { action.kbd }</kbd>
+                      {#if action.comment}({ action.comment }){/if}
+                    </td>
                 </tr>
-            
             {/each}
           </tbody>
         </table>
@@ -80,14 +111,9 @@
     </div>
 </div>
 
+
+
+
+<!-- Listen for keypresses -->
 <svelte:window on:keydown|preventDefault={onKeyDown} />
 
-{#if toast_visible}
-<div class="toast" transition:fade>
-  <div class="alert { toast_state.class }">
-      <div>
-      <span>{ toast_state.text }.</span>
-      </div>
-  </div>
-</div>
-{/if}

news_check/client/src/ArticleStatus.svelte

@@ -2,13 +2,21 @@
     export let article_data;
     const status_items = [
         {name: 'Title', value: article_data.title},
+        {name: 'Url', value: article_data.article_url},
+        {name: 'Source', value: article_data.source_name},
         {name: 'Filename', value: article_data.file_name},
+        {name: 'Location', value: article_data.save_path},
         {name: 'Language', value: article_data.language},
         {name: 'Authors', value: article_data.authors},
         {name: "Related", value: article_data.related},
     ]
 </script>
 
+<style>
+  a {
+    word-break: break-all;
+  }
+</style>
 <div class="card bg-neutral-300 shadow-xl overflow-x-auto">
     <div class="card-body">
       <h2 class="card-title">Article overview:</h2>
@@ -23,16 +31,18 @@
             {#each status_items as item}
                 <tr>
                     <td>{ item.name }</td>
-                    <!-- <td>Quality Control Specialist</td> -->
                     {#if item.value != ""}
+                      {#if item.name == "Url"}
+                        <td class='bg-emerald-200'><a href="{ item.value }">{ item.value }</a></td>
+                      {:else}
                         <td class='bg-emerald-200' style="white-space: normal; width:70%">{ item.value }</td>
+                      {/if}
                     {:else}
-                        <td class='bg-red-200'>{ item.value }</td>
+                      <td class='bg-red-200'>not set</td>
                     {/if}
                 </tr>
             {/each}
           </tbody>
         </table>
       </div>
-    
 </div>

news_check/client/src/Toast.svelte

@@ -0,0 +1,34 @@
+<script  context="module">
+    import {fade} from 'svelte/transition';
+    import { writable } from 'svelte/store';
+
+
+    let toasts = writable([])
+
+    export function addToast (type, message="") {
+        if (message == "") {
+            message = toast_states[type]["text"]
+        }
+        toasts.update((all) => [{"class" : toast_states[type]["class"], "text": message}, ...all]);
+        toasts = toasts;
+        setTimeout(() => {
+            toasts.update((all) => all.slice(0, -1));
+        }, 2000);
+    }
+
+
+
+    const toast_states = {
+      'success' : {class: 'alert-success', text: 'Article updated successfully'},
+      'error' : {class: 'alert-error', text: 'Article update failed'},
+    }
+
+</script>
+
+<div class="toast">
+    {#each $toasts as toast}
+        <div class="alert { toast.class }" transition:fade>
+            <div> <span>{ toast.text }.</span> </div>
+        </div>
+    {/each}
+</div>

news_check/client/src/main.js

@@ -2,9 +2,6 @@ import App from './App.svelte';
 
 const app = new App({
 	target: document.body,
-	props: {
-		name: 'world'
-	}
 });
 
 export default app;

news_check/server/app.py

@@ -1,5 +1,7 @@
 from flask import Flask, send_from_directory, request
+import os
 import configuration
+
 models = configuration.models
 db = configuration.db
 app = Flask(__name__)
@@ -30,9 +32,9 @@ def get_article_by_id(id):
         return article.to_dict()
 
 @app.route("/api/article/first")
-def get_article_first():
+def get_article_first(min_id=0):
     with db:
-        article = models.ArticleDownload.select(models.ArticleDownload.id).where(models.ArticleDownload.verified == 0).order_by(models.ArticleDownload.id).first()
+        article = models.ArticleDownload.select(models.ArticleDownload.id).where((models.ArticleDownload.verified == 0) & (models.ArticleDownload.id > min_id)).order_by(models.ArticleDownload.id).first()
         return {"id" : article.id}
 
 @app.route("/api/article/<int:id>/next")
@@ -41,27 +43,44 @@ def get_article_next(id):
         if models.ArticleDownload.get_by_id(id + 1).verified == 0:
             return {"id" : id + 1}
         else:
-            return get_article_first()
+            return get_article_first(min_id=id) # if the current article was skipped, but the +1 is already verified, get_first will return the same article again. so specify min id.
 
 
 
 @app.route("/api/article/<int:id>/set", methods=['POST'])
 def set_article(id):
-    action = request.json['action']
+    try:
+        action = request.json.get('action', None)
+    except Exception as e:
+        print(f"Exception in set_article {e}")
+        action = None
     with db:
         article = models.ArticleDownload.get_by_id(id)
-        if action == "a":
+        if action:
-            article.verified = 1
+            if action  == "a":
-        elif action == "b":
+                article.verified = 1
-            article.verified = -1
+            elif action == "b":
-        elif action == "r":
+                article.verified = -1
-            article.set_related()
+        else: # implicitly action == "r":
-        article.save()
+            print(request.files)
-        return "ok"
+            file = request.files.get("file", None)
+            if file is None: # upload tends to crash
+                return "No file uploaded", 400
 
+            artname, _ = os.path.splitext(article.file_name)
+            fname =  f"{artname} -- related_{article.related.count() + 1}.{file.filename.split('.')[-1]}"
+            fpath = os.path.join(article.save_path, fname)
+            print(fpath)
+            file.save(fpath)
+            article.set_related([fname])
+            return {"file_path": fpath}
+
+        article.save()
+    return "ok"
 
 
 
 
 if __name__ == "__main__":
-    app.run(host="0.0.0.0", port="80")
+    debug = os.getenv("DEBUG", "false") == "true"
+    app.run(host="0.0.0.0", port="80", debug=debug)

news_check/server/configuration.py

@@ -1,5 +1,6 @@
 from peewee import PostgresqlDatabase
 import configparser
+import time
 
 main_config = configparser.ConfigParser()
 main_config.read("/app/containerdata/config/news_fetch.config.ini")
@@ -8,6 +9,7 @@ db_config = configparser.ConfigParser()
 db_config.read("/app/containerdata/config/db.config.ini")
 
 cred = db_config["DATABASE"]
+time.sleep(10) # wait for the vpn to connect (can't use a healthcheck because there is no depends_on)
 db = PostgresqlDatabase(
     cred["db_name"], user=cred["user_name"], password=cred["password"], host="vpn", port=5432
 )

news_fetch/configuration.py

@@ -2,8 +2,8 @@ import os
 import configparser
 import logging
 import time
-import shutil
+# import shutil
-from datetime import datetime
+# from datetime import datetime
 from peewee import SqliteDatabase, PostgresqlDatabase
 from rich.logging import RichHandler
 

news_fetch/runner.py

@@ -1,4 +1,5 @@
 """Main coordination of other util classes. Handles inbound and outbound calls"""
+from time import sleep
 import configuration
 models = configuration.models
 from threading import Thread
@@ -110,7 +111,8 @@ class Dispatcher(Thread):
             logger.error("Dispatcher.incoming_request called with no arguments")
             return
 
-        if is_new or (article.file_name == "" and article.verified == 0):
+        if is_new or (article.file_name == "" and article.verified == 0) \
+            or (not is_new and len(self.workers_in) == 1): # this is for upload
             # check for models that were created but were abandonned. This means they have missing information, most importantly no associated file
             # this overwrites previously set information, but that should not be too important
             ArticleWatcher(
@@ -121,7 +123,6 @@ class Dispatcher(Thread):
 
         else: # manually trigger notification immediatly
             logger.info(f"Found existing article {article}. Now sending")
-            self.article_complete_notifier(article)
 
 
 
@@ -142,6 +143,8 @@ if __name__ == "__main__":
         class PrintWorker:
             def send(self, article):
                 print(f"Uploaded article {article}")
+            def keep_alive(self): # keeps script running, because there is nothing else in the main thread
+                while True: sleep(1)
 
         articles = models.ArticleDownload.select().where(models.ArticleDownload.archive_url == "" or models.ArticleDownload.archive_url == "TODO:UPLOAD").execute()
         logger.info(f"Launching upload to archive for {len(articles)} articles.")
@@ -149,6 +152,9 @@ if __name__ == "__main__":
         dispatcher.workers_in = [{"UploadWorker": UploadWorker()}]
         dispatcher.workers_out = [{"PrintWorker": PrintWorker()}]
         dispatcher.start()
+        for a in articles:
+            dispatcher.incoming_request(article=a)
+        PrintWorker().keep_alive()
 
     else: # launch with full action
         try: