remoll/coss_archiving
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

working (feature complete) news_fetch

Browse Source
This commit is contained in:
Remy Moll 2022-09-09 22:32:22 +02:00
parent afead44d6c
commit d3d44dcdc9
11 changed files with 177 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
news_check/client/public/favicon.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 3.1 KiB

63
news_check/client/public/global.css
View File

@ -1,63 +0,0 @@
html, body {
position: relative;
width: 100%;
height: 100%;
}
body {
color: #333;
margin: 0;
padding: 8px;
box-sizing: border-box;
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen-Sans, Ubuntu, Cantarell, "Helvetica Neue", sans-serif;
}
a {
color: rgb(0,100,200);
text-decoration: none;
}
a:hover {
text-decoration: underline;
}
a:visited {
color: rgb(0,80,160);
}
label {
display: block;
}
input, button, select, textarea {
font-family: inherit;
font-size: inherit;
-webkit-padding: 0.4em 0;
padding: 0.4em;
margin: 0 0 0.5em 0;
box-sizing: border-box;
border: 1px solid #ccc;
border-radius: 2px;
}
input:disabled {
color: #ccc;
}
button {
color: #333;
background-color: #f4f4f4;
outline: none;
}
button:disabled {
color: #999;
}
button:not(:disabled):active {
background-color: #ddd;
}
button:focus {
border-color: #666;
}

21
news_check/client/src/App.svelte
View File

@ -2,10 +2,15 @@
import PDFView from './PDFView.svelte';
import ArticleStatus from './ArticleStatus.svelte';
import ArticleOperations from './ArticleOperations.svelte';
import Toast from './Toast.svelte';
let current_id = 0;
const updateInterface = (async () => {
let interfaceState = updateInterface()
async function updateInterface () {
let url = '';
if (current_id == 0) {
url = '/api/article/first';
@ -19,12 +24,14 @@
const article_response = await fetch(article_url);
const article_data = await article_response.json();
return article_data;
})()
}
function triggerUpdate () {
interfaceState = updateInterface();
}
</script>
{#await updateInterface}
{#await interfaceState}
...
{:then article_data}
<div class="flex w-full h-screen gap-5 p-5">
@ -33,7 +40,9 @@
<div class="w-2/5">
<ArticleStatus article_data={article_data}/>
<div class="divider divider-vertical"></div>
<ArticleOperations article_data={article_data}/>
<ArticleOperations article_data={article_data} callback={triggerUpdate}/>
</div>
</div>
{/await}
<Toast/>

116
news_check/client/src/ArticleOperations.svelte
View File

@ -1,55 +1,85 @@
<script>
import {fade} from 'svelte/transition';
export let article_data;
export let callback;
window.focus()
import { addToast } from './Toast.svelte';
const actions = [
{name: 'Mark as good (and skip to next)', kbd: 'A'},
{name: 'Mark as bad (and skip to next)', kbd: 'B'},
{name: 'Upload related file', kbd: 'R'},
{name: 'Skip', kbd: 'ctrl'},
{name: 'Upload related file', kbd: 'R', comment: "can be used multiple times"},
{name: 'Skip', kbd: 'S'},
]
const toast_states = {
'success' : {class: 'alert-success', text: 'Article updated successfully'},
'error' : {class: 'alert-error', text: 'Article update failed'},
}
let toast_state = {};
let toast_visible = false;
let fileInput = document.createElement('input');
fileInput.type = 'file';
fileInput.onchange = e => {
let result = (async () => {
uploadRelatedFile(e.target.files[0]);
})()
}
function onKeyDown(e) {apiAction(e.key)}
function apiAction(key) {
if (actions.map(d => d.kbd.toLowerCase()).includes(key.toLowerCase())){ // ignore other keypresses
const updateArticle = (async() => {
const response = await fetch('/api/article/' + article_data.id + '/set', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({
'action': key.toLowerCase(),
})
})
const success = response.status == 200;
if (success){
showToast('success');
} else {
showToast('error');
}
let success
if (key.toLowerCase() == "s") {
addToast('success', "Article skipped")
callback()
return
} else if (key.toLowerCase() == "r") {
fileInput.click() // this will trigger a change in fileInput,
return
} else {
const response = await fetch('/api/article/' + article_data.id + '/set', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({
'action': key.toLowerCase(),
})
})
success = response.status == 200
}
if (success){
addToast('success')
callback()
} else {
addToast('error')
}
})()
}
}
function showToast(state){
toast_visible = true;
toast_state = toast_states[state];
setTimeout(() => {
toast_visible = false;
}, 1000)
async function uploadRelatedFile(file) {
const formData = new FormData()
formData.append('file', file)
const response = await fetch('/api/article/' + article_data.id + '/set', {
method: 'POST',
body : formData,
})
const success = response.status == 200;
if (success){
const data = await response.json()
let fname = data.file_path
addToast('success', "File uploaded as " + fname)
} else {
addToast('error', "File upload failed")
}
return success;
}
</script>
@ -58,21 +88,22 @@
<h2 class="card-title">Your options: (click on action or use keyboard)</h2>
<div class="overflow-x-auto">
<table class="table w-full table-compact">
<!-- head -->
<thead>
<tr>
<th>Action</th>
<th>Keyboard shortcut</th>
</tr>
</thead>
<tbody>
{#each actions as action}
{#each actions as action}
<tr>
<td><button on:click={() => apiAction(action.kbd)}>{ action.name }</button></td>
<td><kbd class="kbd">{ action.kbd }</kbd></td>
<td><kbd class="kbd">
{ action.kbd }</kbd>
{#if action.comment}({ action.comment }){/if}
</td>
</tr>
{/each}
</tbody>
</table>
@ -80,14 +111,9 @@
</div>
</div>
<!-- Listen for keypresses -->
<svelte:window on:keydown|preventDefault={onKeyDown} />
{#if toast_visible}
<div class="toast" transition:fade>
<div class="alert { toast_state.class }">
<div>
<span>{ toast_state.text }.</span>
</div>
</div>
</div>
{/if}

16
news_check/client/src/ArticleStatus.svelte
View File

@ -2,13 +2,21 @@
export let article_data;
const status_items = [
{name: 'Title', value: article_data.title},
{name: 'Url', value: article_data.article_url},
{name: 'Source', value: article_data.source_name},
{name: 'Filename', value: article_data.file_name},
{name: 'Location', value: article_data.save_path},
{name: 'Language', value: article_data.language},
{name: 'Authors', value: article_data.authors},
{name: "Related", value: article_data.related},
]
</script>
<style>
a {
word-break: break-all;
}
</style>
<div class="card bg-neutral-300 shadow-xl overflow-x-auto">
<div class="card-body">
<h2 class="card-title">Article overview:</h2>
@ -23,16 +31,18 @@
{#each status_items as item}
<tr>
<td>{ item.name }</td>
<!-- <td>Quality Control Specialist</td> -->
{#if item.value != ""}
{#if item.name == "Url"}
<td class='bg-emerald-200'><a href="{ item.value }">{ item.value }</a></td>
{:else}
<td class='bg-emerald-200' style="white-space: normal; width:70%">{ item.value }</td>
{/if}
{:else}
<td class='bg-red-200'>{ item.value }</td>
<td class='bg-red-200'>not set</td>
{/if}
</tr>
{/each}
</tbody>
</table>
</div>
</div>

34
news_check/client/src/Toast.svelte Normal file
View File

@ -0,0 +1,34 @@
<script context="module">
import {fade} from 'svelte/transition';
import { writable } from 'svelte/store';
let toasts = writable([])
export function addToast (type, message="") {
if (message == "") {
message = toast_states[type]["text"]
}
toasts.update((all) => [{"class" : toast_states[type]["class"], "text": message}, ...all]);
toasts = toasts;
setTimeout(() => {
toasts.update((all) => all.slice(0, -1));
}, 2000);
}
const toast_states = {
'success' : {class: 'alert-success', text: 'Article updated successfully'},
'error' : {class: 'alert-error', text: 'Article update failed'},
}
</script>
<div class="toast">
{#each $toasts as toast}
<div class="alert { toast.class }" transition:fade>
<div> <span>{ toast.text }.</span> </div>
</div>
{/each}
</div>

3
news_check/client/src/main.js
View File

@ -2,9 +2,6 @@ import App from './App.svelte';
const app = new App({
target: document.body,
props: {
name: 'world'
}
});
export default app;

45
news_check/server/app.py
View File

@ -1,5 +1,7 @@
from flask import Flask, send_from_directory, request
import os
import configuration
models = configuration.models
db = configuration.db
app = Flask(__name__)
@ -30,9 +32,9 @@ def get_article_by_id(id):
return article.to_dict()
@app.route("/api/article/first")
def get_article_first():
def get_article_first(min_id=0):
with db:
article = models.ArticleDownload.select(models.ArticleDownload.id).where(models.ArticleDownload.verified == 0).order_by(models.ArticleDownload.id).first()
article = models.ArticleDownload.select(models.ArticleDownload.id).where((models.ArticleDownload.verified == 0) & (models.ArticleDownload.id > min_id)).order_by(models.ArticleDownload.id).first()
return {"id" : article.id}
@app.route("/api/article/<int:id>/next")
@ -41,27 +43,44 @@ def get_article_next(id):
if models.ArticleDownload.get_by_id(id + 1).verified == 0:
return {"id" : id + 1}
else:
return get_article_first()
return get_article_first(min_id=id) # if the current article was skipped, but the +1 is already verified, get_first will return the same article again. so specify min id.
@app.route("/api/article/<int:id>/set", methods=['POST'])
def set_article(id):
action = request.json['action']
try:
action = request.json.get('action', None)
except Exception as e:
print(f"Exception in set_article {e}")
action = None
with db:
article = models.ArticleDownload.get_by_id(id)
if action == "a":
article.verified = 1
elif action == "b":
article.verified = -1
elif action == "r":
article.set_related()
article.save()
return "ok"
if action:
if action == "a":
article.verified = 1
elif action == "b":
article.verified = -1
else: # implicitly action == "r":
print(request.files)
file = request.files.get("file", None)
if file is None: # upload tends to crash
return "No file uploaded", 400
artname, _ = os.path.splitext(article.file_name)
fname = f"{artname} -- related_{article.related.count() + 1}.{file.filename.split('.')[-1]}"
fpath = os.path.join(article.save_path, fname)
print(fpath)
file.save(fpath)
article.set_related([fname])
return {"file_path": fpath}
article.save()
return "ok"
if __name__ == "__main__":
app.run(host="0.0.0.0", port="80")
debug = os.getenv("DEBUG", "false") == "true"
app.run(host="0.0.0.0", port="80", debug=debug)

2
news_check/server/configuration.py
View File

@ -1,5 +1,6 @@
from peewee import PostgresqlDatabase
import configparser
import time
main_config = configparser.ConfigParser()
main_config.read("/app/containerdata/config/news_fetch.config.ini")
@ -8,6 +9,7 @@ db_config = configparser.ConfigParser()
db_config.read("/app/containerdata/config/db.config.ini")
cred = db_config["DATABASE"]
time.sleep(10) # wait for the vpn to connect (can't use a healthcheck because there is no depends_on)
db = PostgresqlDatabase(
cred["db_name"], user=cred["user_name"], password=cred["password"], host="vpn", port=5432
)

4
news_fetch/configuration.py
View File

@ -2,8 +2,8 @@ import os
import configparser
import logging
import time
import shutil
from datetime import datetime
# import shutil
# from datetime import datetime
from peewee import SqliteDatabase, PostgresqlDatabase
from rich.logging import RichHandler

10
news_fetch/runner.py
View File

@ -1,4 +1,5 @@
"""Main coordination of other util classes. Handles inbound and outbound calls"""
from time import sleep
import configuration
models = configuration.models
from threading import Thread
@ -110,7 +111,8 @@ class Dispatcher(Thread):
logger.error("Dispatcher.incoming_request called with no arguments")
return
if is_new or (article.file_name == "" and article.verified == 0):
if is_new or (article.file_name == "" and article.verified == 0) \
or (not is_new and len(self.workers_in) == 1): # this is for upload
# check for models that were created but were abandonned. This means they have missing information, most importantly no associated file
# this overwrites previously set information, but that should not be too important
ArticleWatcher(
@ -121,7 +123,6 @@ class Dispatcher(Thread):
else: # manually trigger notification immediatly
logger.info(f"Found existing article {article}. Now sending")
self.article_complete_notifier(article)
@ -142,6 +143,8 @@ if __name__ == "__main__":
class PrintWorker:
def send(self, article):
print(f"Uploaded article {article}")
def keep_alive(self): # keeps script running, because there is nothing else in the main thread
while True: sleep(1)
articles = models.ArticleDownload.select().where(models.ArticleDownload.archive_url == "" or models.ArticleDownload.archive_url == "TODO:UPLOAD").execute()
logger.info(f"Launching upload to archive for {len(articles)} articles.")
@ -149,6 +152,9 @@ if __name__ == "__main__":
dispatcher.workers_in = [{"UploadWorker": UploadWorker()}]
dispatcher.workers_out = [{"PrintWorker": PrintWorker()}]
dispatcher.start()
for a in articles:
dispatcher.incoming_request(article=a)
PrintWorker().keep_alive()
else: # launch with full action
try: