few bugs in news_fetch left, news_chek wip
This commit is contained in:
@@ -2,16 +2,38 @@
|
||||
import PDFView from './PDFView.svelte';
|
||||
import ArticleStatus from './ArticleStatus.svelte';
|
||||
import ArticleOperations from './ArticleOperations.svelte';
|
||||
|
||||
let current_id = 0;
|
||||
|
||||
const updateInterface = (async () => {
|
||||
let url = '';
|
||||
if (current_id == 0) {
|
||||
url = '/api/article/first';
|
||||
} else {
|
||||
url = '/api/article/' + current_id + '/next';
|
||||
}
|
||||
const response = await fetch(url)
|
||||
const data = await response.json()
|
||||
current_id = data.id;
|
||||
let article_url = '/api/article/' + current_id + '/get';
|
||||
const article_response = await fetch(article_url);
|
||||
const article_data = await article_response.json();
|
||||
return article_data;
|
||||
})()
|
||||
|
||||
|
||||
</script>
|
||||
|
||||
|
||||
<div class="flex w-full h-full gap-5 p-5">
|
||||
<div class="w-3/5"><PDFView/></div>
|
||||
<div class="divider divider-horizontal"></div>
|
||||
<div class="w-2/5">
|
||||
<ArticleStatus article_id={42}/>
|
||||
<div class="divider divider-vertical"></div>
|
||||
<ArticleOperations/>
|
||||
{#await updateInterface}
|
||||
...
|
||||
{:then article_data}
|
||||
<div class="flex w-full h-screen gap-5 p-5">
|
||||
<div class="w-3/5"><PDFView article_data={article_data}/></div>
|
||||
<div class="divider divider-horizontal"></div>
|
||||
<div class="w-2/5">
|
||||
<ArticleStatus article_data={article_data}/>
|
||||
<div class="divider divider-vertical"></div>
|
||||
<ArticleOperations article_data={article_data}/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/await}
|
||||
|
||||
@@ -1,28 +1,93 @@
|
||||
<div class="toast">
|
||||
<div class="alert alert-info">
|
||||
<div>
|
||||
<span>New message arrived.</span>
|
||||
</div>
|
||||
<script>
|
||||
import {fade} from 'svelte/transition';
|
||||
|
||||
export let article_data;
|
||||
|
||||
const actions = [
|
||||
{name: 'Mark as good (and skip to next)', kbd: 'A'},
|
||||
{name: 'Mark as bad (and skip to next)', kbd: 'B'},
|
||||
{name: 'Upload related file', kbd: 'R'},
|
||||
{name: 'Skip', kbd: 'ctrl'},
|
||||
]
|
||||
|
||||
const toast_states = {
|
||||
'success' : {class: 'alert-success', text: 'Article updated successfully'},
|
||||
'error' : {class: 'alert-error', text: 'Article update failed'},
|
||||
}
|
||||
let toast_state = {};
|
||||
let toast_visible = false;
|
||||
|
||||
|
||||
function onKeyDown(e) {apiAction(e.key)}
|
||||
function apiAction(key) {
|
||||
if (actions.map(d => d.kbd.toLowerCase()).includes(key.toLowerCase())){ // ignore other keypresses
|
||||
|
||||
const updateArticle = (async() => {
|
||||
const response = await fetch('/api/article/' + article_data.id + '/set', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({
|
||||
'action': key.toLowerCase(),
|
||||
})
|
||||
})
|
||||
const success = response.status == 200;
|
||||
|
||||
if (success){
|
||||
showToast('success');
|
||||
} else {
|
||||
showToast('error');
|
||||
}
|
||||
|
||||
})()
|
||||
}
|
||||
}
|
||||
|
||||
function showToast(state){
|
||||
toast_visible = true;
|
||||
toast_state = toast_states[state];
|
||||
setTimeout(() => {
|
||||
toast_visible = false;
|
||||
}, 1000)
|
||||
|
||||
}
|
||||
</script>
|
||||
|
||||
|
||||
<div class="card bg-neutral-300 shadow-xl">
|
||||
<div class="card-body">
|
||||
<h2 class="card-title">Your options: (click on action or use keyboard)</h2>
|
||||
<div class="overflow-x-auto">
|
||||
<table class="table w-full table-compact">
|
||||
<!-- head -->
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Action</th>
|
||||
<th>Keyboard shortcut</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{#each actions as action}
|
||||
|
||||
<tr>
|
||||
<td><button on:click={() => apiAction(action.kbd)}>{ action.name }</button></td>
|
||||
<td><kbd class="kbd">{ action.kbd }</kbd></td>
|
||||
</tr>
|
||||
|
||||
{/each}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="grid grid-cols-3 gap-4">
|
||||
<div class="highlight">01</div>
|
||||
<div class="highlight">01</div>
|
||||
<div class="highlight">01</div>
|
||||
<div class="highlight">01</div>
|
||||
<div class="highlight">01</div>
|
||||
<div class="highlight">01</div>
|
||||
<div class="highlight">01</div>
|
||||
<div class="highlight">01</div>
|
||||
<div class="highlight">01</div>
|
||||
<svelte:window on:keydown|preventDefault={onKeyDown} />
|
||||
|
||||
{#if toast_visible}
|
||||
<div class="toast" transition:fade>
|
||||
<div class="alert { toast_state.class }">
|
||||
<div>
|
||||
<span>{ toast_state.text }.</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<style>
|
||||
.highlight {
|
||||
background-color: #f5f5f5;
|
||||
border-radius: 5px;
|
||||
padding: 10px;
|
||||
margin: 10px;
|
||||
}
|
||||
</style>
|
||||
{/if}
|
||||
@@ -1,25 +1,38 @@
|
||||
<script>
|
||||
export let article_id;
|
||||
const Article = (async () => {
|
||||
const response = await fetch('/api/article/' + article_id + '/get')
|
||||
return await response.json()
|
||||
})()
|
||||
console.log(Article)
|
||||
export let article_data;
|
||||
const status_items = [
|
||||
{name: 'Title', value: article_data.title},
|
||||
{name: 'Filename', value: article_data.file_name},
|
||||
{name: 'Language', value: article_data.language},
|
||||
{name: 'Authors', value: article_data.authors},
|
||||
{name: "Related", value: article_data.related},
|
||||
]
|
||||
</script>
|
||||
|
||||
<div class="mockup-window border bg-base-300">
|
||||
<h1 class="center">Article overview</h1>
|
||||
<ul tabindex="0" class="menu p-2 shadow bg-base-100 rounded-box w-52">
|
||||
{#await Article}
|
||||
<li>...waiting</li>
|
||||
{:then data}
|
||||
<li><a href="#">{data.value}</a></li>
|
||||
<li><a href="#">Item 2</a></li>
|
||||
{:catch error}
|
||||
<li>An error occurred!</li>
|
||||
{/await}
|
||||
|
||||
|
||||
</ul>
|
||||
<div class="card bg-neutral-300 shadow-xl overflow-x-auto">
|
||||
<div class="card-body">
|
||||
<h2 class="card-title">Article overview:</h2>
|
||||
<table class="table w-full table-compact" style="table-layout: fixed">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Attribute</th>
|
||||
<th>Value</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{#each status_items as item}
|
||||
<tr>
|
||||
<td>{ item.name }</td>
|
||||
<!-- <td>Quality Control Specialist</td> -->
|
||||
{#if item.value != ""}
|
||||
<td class='bg-emerald-200' style="white-space: normal">{ item.value }</td>
|
||||
{:else}
|
||||
<td class='bg-red-200'>{ item.value }</td>
|
||||
{/if}
|
||||
</tr>
|
||||
{/each}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
@@ -1,64 +1,10 @@
|
||||
<!--
|
||||
<script>
|
||||
var myState = {
|
||||
pdf: null,
|
||||
currentPage: 1,
|
||||
zoom: 1
|
||||
}
|
||||
|
||||
pdfjsLib.getDocument('test.pdf').then((pdf) => {
|
||||
|
||||
myState.pdf = pdf;
|
||||
render();
|
||||
|
||||
});
|
||||
|
||||
function render() {
|
||||
myState.pdf.getPage(myState.currentPage).then((page) => {
|
||||
|
||||
var canvas = document.getElementById("pdf_renderer");
|
||||
var ctx = canvas.getContext('2d');
|
||||
|
||||
var viewport = page.getViewport(myState.zoom);
|
||||
|
||||
canvas.width = viewport.width;
|
||||
canvas.height = viewport.height;
|
||||
|
||||
page.render({
|
||||
canvasContext: ctx,
|
||||
viewport: viewport
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
</script>
|
||||
-->
|
||||
<!-- <div id="my_pdf_viewer">
|
||||
<div class="mockup-window border bg-base-300">
|
||||
<div id="canvas_container" class="flex justify-center">
|
||||
<canvas id="pdf_renderer"></canvas>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div id="navigation_controls">
|
||||
<button id="go_previous">Previous</button>
|
||||
<input id="current_page" value="1" type="number"/>
|
||||
<button id="go_next">Next</button>
|
||||
</div>
|
||||
|
||||
<div id="zoom_controls">
|
||||
<button id="zoom_in">+</button>
|
||||
<button id="zoom_out">-</button>
|
||||
</div>
|
||||
</div> -->
|
||||
|
||||
<script>
|
||||
let pdf_file = 'test.pdf';
|
||||
export let article_data;
|
||||
</script>
|
||||
|
||||
<div class="mockup-window border bg-base-300 h-full w-full">
|
||||
<object class="pdf-view" data="{pdf_file}" title="Article PDF"> </object>
|
||||
<div class="h-full w-full shadow-xl">
|
||||
<object class="pdf-view" data="{article_data.save_path + article_data.file_name}" title="Article PDF"> </object>
|
||||
</div>
|
||||
|
||||
<style>
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import json
|
||||
from flask import Flask, send_from_directory, jsonify
|
||||
import random
|
||||
|
||||
from flask import Flask, send_from_directory, request
|
||||
import configuration
|
||||
models = configuration.models
|
||||
db = configuration.db
|
||||
app = Flask(__name__)
|
||||
|
||||
|
||||
@@ -9,26 +9,55 @@ app = Flask(__name__)
|
||||
# SVELTE 'STATIC' BACKEND. Always send index.html and the requested js-files. (compiled by npm)
|
||||
|
||||
@app.route("/") #index.html
|
||||
def base():
|
||||
def index():
|
||||
return send_from_directory('../client/public', 'index.html')
|
||||
@app.route("/<path:path>") #js-files
|
||||
def home(path):
|
||||
def js(path):
|
||||
return send_from_directory('../client/public', path)
|
||||
|
||||
@app.route("/app/containerdata/files/<path:path>")
|
||||
def static_pdf(path):
|
||||
return send_from_directory('/app/containerdata/files/', path)
|
||||
|
||||
|
||||
|
||||
###############################################################################
|
||||
# API for news_check.
|
||||
# (simple) API for news_check.
|
||||
|
||||
@app.route("/api/article/<int:id>/get")
|
||||
def get_article(id):
|
||||
res = {"value": id}
|
||||
return jsonify(res)
|
||||
def get_article_by_id(id):
|
||||
with db:
|
||||
article = models.ArticleDownload.get_by_id(id)
|
||||
return article.to_dict()
|
||||
|
||||
@app.route("/api/article/first")
|
||||
def get_article_first():
|
||||
with db:
|
||||
article = models.ArticleDownload.select(models.ArticleDownload.id).where(models.ArticleDownload.verified == 0).order_by(models.ArticleDownload.id).first()
|
||||
return {"id" : article.id}
|
||||
|
||||
@app.route("/api/article/<int:id>/next")
|
||||
def get_article_next(id):
|
||||
with db:
|
||||
if models.ArticleDownload.get_by_id(id + 1).verified == 0:
|
||||
return {"id" : id + 1}
|
||||
else:
|
||||
return get_article_first()
|
||||
|
||||
|
||||
|
||||
@app.route("/api/article/<int:id>/set", methods=['POST'])
|
||||
def set_article(id):
|
||||
return str(random.randint(0, 100))
|
||||
action = request.json['action']
|
||||
with db:
|
||||
article = models.ArticleDownload.get_by_id(id)
|
||||
if action == "a":
|
||||
article.verified = 1
|
||||
elif action == "b":
|
||||
article.verified = -1
|
||||
elif action == "r":
|
||||
article.set_related()
|
||||
article.save()
|
||||
return "ok"
|
||||
|
||||
|
||||
|
||||
|
||||
16
news_check/server/configuration.py
Normal file
16
news_check/server/configuration.py
Normal file
@@ -0,0 +1,16 @@
|
||||
from peewee import PostgresqlDatabase
|
||||
import configparser
|
||||
|
||||
main_config = configparser.ConfigParser()
|
||||
main_config.read("/app/containerdata/config/news_fetch.config.ini")
|
||||
|
||||
db_config = configparser.ConfigParser()
|
||||
db_config.read("/app/containerdata/config/db.config.ini")
|
||||
|
||||
cred = db_config["DATABASE"]
|
||||
db = PostgresqlDatabase(
|
||||
cred["db_name"], user=cred["user_name"], password=cred["password"], host="vpn", port=5432
|
||||
)
|
||||
|
||||
import models
|
||||
models.set_db(db)
|
||||
134
news_check/server/models.py
Normal file
134
news_check/server/models.py
Normal file
@@ -0,0 +1,134 @@
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
from peewee import *
|
||||
import os
|
||||
import datetime
|
||||
import configuration
|
||||
|
||||
config = configuration.main_config["DOWNLOADS"]
|
||||
|
||||
# set the nature of the db at runtime
|
||||
download_db = DatabaseProxy()
|
||||
|
||||
|
||||
class DownloadBaseModel(Model):
|
||||
class Meta:
|
||||
database = download_db
|
||||
|
||||
|
||||
|
||||
## == Article related models == ##
|
||||
class ArticleDownload(DownloadBaseModel):
|
||||
# in the beginning this is all we have
|
||||
article_url = TextField(default = '', unique=True)
|
||||
|
||||
# fetch then fills in the metadata
|
||||
title = TextField(default='')
|
||||
|
||||
summary = TextField(default = '')
|
||||
source_name = CharField(default = '')
|
||||
language = CharField(default = '')
|
||||
|
||||
|
||||
file_name = TextField(default = '')
|
||||
@property
|
||||
def save_path(self):
|
||||
return f"{config['local_storage_path']}/{self.download_date.year}/{self.download_date.strftime('%B')}/"
|
||||
@property
|
||||
def fname_nas(self, file_name=""):
|
||||
if self.download_date:
|
||||
if file_name:
|
||||
return f"NAS: {config['remote_storage_path']}/{self.download_date.year}/{self.download_date.strftime('%B')}/{file_name}"
|
||||
else: # return the self. name
|
||||
return f"NAS: {config['remote_storage_path']}/{self.download_date.year}/{self.download_date.strftime('%B')}/{self.file_name}"
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
archive_url = TextField(default = '')
|
||||
pub_date = DateField(default = datetime.date.fromtimestamp(0))
|
||||
download_date = DateField(default = datetime.date.today)
|
||||
|
||||
slack_ts = FloatField(default = 0) # should be a fixed-length string but float is easier to sort by
|
||||
|
||||
sent = BooleanField(default = False)
|
||||
|
||||
archived_by = CharField(default = os.getenv("UNAME"))
|
||||
# need to know who saved the message because the file needs to be on their computer in order to get verified
|
||||
# verification happens in a different app, but the model has the fields here as well
|
||||
comment = TextField(default = '')
|
||||
verified = IntegerField(default = 0) # 0 = not verified, 1 = verified, -1 = marked as bad
|
||||
|
||||
# authors
|
||||
# keywords
|
||||
# ... are added through foreignkeys
|
||||
# we will also add an attribute named message, to reference which message should be replied to. This attribute does not need to be saved in the db
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
"id": self.id,
|
||||
"article_url": self.article_url,
|
||||
"title": self.title,
|
||||
"summary": self.summary,
|
||||
"source_name": self.source_name,
|
||||
"language": self.language,
|
||||
"file_name": self.file_name,
|
||||
"save_path": self.save_path,
|
||||
"fname_nas": self.fname_nas,
|
||||
"archive_url": self.archive_url,
|
||||
"pub_date": self.pub_date.strftime("%Y-%m-%d"),
|
||||
"download_date": self.download_date.strftime("%Y-%m-%d"),
|
||||
"sent": self.sent,
|
||||
"comment": self.comment,
|
||||
"related": [r.related_file_name for r in self.related],
|
||||
"authors": [a.author for a in self.authors]
|
||||
}
|
||||
|
||||
|
||||
|
||||
def set_related(self, related):
|
||||
for r in related:
|
||||
if len(r) > 255:
|
||||
raise Exception("Related file name too long for POSTGRES")
|
||||
|
||||
ArticleRelated.create(
|
||||
article = self,
|
||||
related_file_name = r
|
||||
)
|
||||
|
||||
def file_status(self):
|
||||
if not self.file_name:
|
||||
logger.error(f"Article {self} has no filename!")
|
||||
return False, {"reply_text": "Download failed, no file was saved.", "file_path": None}
|
||||
|
||||
file_path_abs = self.save_path + self.file_name
|
||||
if not os.path.exists(file_path_abs):
|
||||
logger.error(f"Article {self} has a filename, but the file does not exist at that location!")
|
||||
return False, {"reply_text": "Can't find file. Either the download failed or the file was moved.", "file_path": None}
|
||||
|
||||
return True, {}
|
||||
|
||||
|
||||
class ArticleAuthor(DownloadBaseModel):
|
||||
article = ForeignKeyField(ArticleDownload, backref='authors')
|
||||
author = CharField()
|
||||
|
||||
|
||||
class ArticleRelated(DownloadBaseModel):
|
||||
# Related files, such as the full text of a paper, audio files, etc.
|
||||
article = ForeignKeyField(ArticleDownload, backref='related')
|
||||
related_file_name = TextField(default = '')
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def set_db(download_db_object):
|
||||
download_db.initialize(download_db_object)
|
||||
with download_db: # create tables (does nothing if they exist already)
|
||||
download_db.create_tables([ArticleDownload, ArticleAuthor, ArticleRelated])
|
||||
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
import peewee
|
||||
|
||||
db = peewee.PostgresqlDatabase('coss_archiving', user='ca_rw', password='pleasechangeit', host='vpn', port=5432)
|
||||
# db.connect()
|
||||
|
||||
|
||||
class Pet(peewee.Model):
|
||||
name = peewee.CharField()
|
||||
animal_type = peewee.CharField()
|
||||
|
||||
class Meta:
|
||||
database = db # this model uses the "people.db" database
|
||||
with db:
|
||||
db.create_tables([Pet])
|
||||
db.get_tables()
|
||||
|
||||
t = Pet.create(name="Test", animal_type="test")
|
||||
|
||||
for pet in Pet.select():
|
||||
print(pet.name)
|
||||
Reference in New Issue
Block a user