few bugs in news_fetch left, news_chek wip

news_check/server/app.py

@@ -1,7 +1,7 @@
-import json
-from flask import Flask, send_from_directory, jsonify
-import random
-
+from flask import Flask, send_from_directory, request
+import configuration
+models = configuration.models
+db = configuration.db
 app = Flask(__name__)
 
 
@@ -9,26 +9,55 @@ app = Flask(__name__)
 # SVELTE 'STATIC' BACKEND. Always send index.html and the requested js-files. (compiled by npm)
 
 @app.route("/") #index.html
-def base():
+def index():
     return send_from_directory('../client/public', 'index.html')
 @app.route("/<path:path>") #js-files
-def home(path):
+def js(path):
     return send_from_directory('../client/public', path)
-
+@app.route("/app/containerdata/files/<path:path>")
+def static_pdf(path):
+    return send_from_directory('/app/containerdata/files/', path)
 
 
 
 ###############################################################################
-# API for news_check.
+# (simple) API for news_check.
 
 @app.route("/api/article/<int:id>/get")
-def get_article(id):
-    res = {"value": id}
-    return jsonify(res)
+def get_article_by_id(id):
+    with db:
+        article = models.ArticleDownload.get_by_id(id)
+        return article.to_dict()
+
+@app.route("/api/article/first")
+def get_article_first():
+    with db:
+        article = models.ArticleDownload.select(models.ArticleDownload.id).where(models.ArticleDownload.verified == 0).order_by(models.ArticleDownload.id).first()
+        return {"id" : article.id}
+
+@app.route("/api/article/<int:id>/next")
+def get_article_next(id):
+    with db:
+        if models.ArticleDownload.get_by_id(id + 1).verified == 0:
+            return {"id" : id + 1}
+        else:
+            return get_article_first()
+
+
 
 @app.route("/api/article/<int:id>/set", methods=['POST'])
 def set_article(id):
-    return str(random.randint(0, 100))
+    action = request.json['action']
+    with db:
+        article = models.ArticleDownload.get_by_id(id)
+        if action == "a":
+            article.verified = 1
+        elif action == "b":
+            article.verified = -1
+        elif action == "r":
+            article.set_related()
+        article.save()
+        return "ok"
 
 
 

news_check/server/configuration.py

@@ -0,0 +1,16 @@
+from peewee import PostgresqlDatabase
+import configparser
+
+main_config = configparser.ConfigParser()
+main_config.read("/app/containerdata/config/news_fetch.config.ini")
+
+db_config = configparser.ConfigParser()
+db_config.read("/app/containerdata/config/db.config.ini")
+
+cred = db_config["DATABASE"]
+db = PostgresqlDatabase(
+    cred["db_name"], user=cred["user_name"], password=cred["password"], host="vpn", port=5432
+)
+
+import models
+models.set_db(db)

news_check/server/models.py

@@ -0,0 +1,134 @@
+import logging
+logger = logging.getLogger(__name__)
+
+from peewee import *
+import os
+import datetime
+import configuration
+
+config = configuration.main_config["DOWNLOADS"]
+
+# set the nature of the db at runtime
+download_db = DatabaseProxy()
+
+
+class DownloadBaseModel(Model):
+    class Meta:
+        database = download_db
+
+
+
+## == Article related models == ##
+class ArticleDownload(DownloadBaseModel):
+    # in the beginning this is all we have
+    article_url = TextField(default = '', unique=True)
+    
+    # fetch then fills in the metadata
+    title = TextField(default='')
+
+    summary = TextField(default = '')
+    source_name = CharField(default = '')
+    language = CharField(default = '')
+
+
+    file_name = TextField(default = '')
+    @property
+    def save_path(self):
+        return f"{config['local_storage_path']}/{self.download_date.year}/{self.download_date.strftime('%B')}/"
+    @property
+    def fname_nas(self, file_name=""):
+        if self.download_date:
+            if file_name:
+                return f"NAS: {config['remote_storage_path']}/{self.download_date.year}/{self.download_date.strftime('%B')}/{file_name}"
+            else: # return the self. name
+                return f"NAS: {config['remote_storage_path']}/{self.download_date.year}/{self.download_date.strftime('%B')}/{self.file_name}"
+        else:
+            return None
+
+    
+    archive_url = TextField(default = '')
+    pub_date = DateField(default = datetime.date.fromtimestamp(0))
+    download_date = DateField(default = datetime.date.today)
+
+    slack_ts = FloatField(default = 0) # should be a fixed-length string but float is easier to sort by
+
+    sent = BooleanField(default = False)
+    
+    archived_by = CharField(default = os.getenv("UNAME"))
+    # need to know who saved the message because the file needs to be on their computer in order to get verified
+    # verification happens in a different app, but the model has the fields here as well
+    comment = TextField(default = '')
+    verified = IntegerField(default = 0) # 0 = not verified, 1 = verified, -1 = marked as bad
+
+    # authors
+    # keywords
+    # ... are added through foreignkeys
+    # we will also add an attribute named message, to reference which message should be replied to. This attribute does not need to be saved in the db
+
+    def to_dict(self):
+        return {
+            "id": self.id,
+            "article_url": self.article_url,
+            "title": self.title,
+            "summary": self.summary,
+            "source_name": self.source_name,
+            "language": self.language,
+            "file_name": self.file_name,
+            "save_path": self.save_path,
+            "fname_nas": self.fname_nas,
+            "archive_url": self.archive_url,
+            "pub_date": self.pub_date.strftime("%Y-%m-%d"),
+            "download_date": self.download_date.strftime("%Y-%m-%d"),
+            "sent": self.sent,
+            "comment": self.comment,
+            "related": [r.related_file_name for r in self.related],
+            "authors": [a.author for a in self.authors]
+        }
+
+
+
+    def set_related(self, related):
+        for r in related:
+            if len(r) > 255:
+                raise Exception("Related file name too long for POSTGRES")
+
+            ArticleRelated.create(
+                article = self,
+                related_file_name = r
+            )
+
+    def file_status(self):
+        if not self.file_name:
+            logger.error(f"Article {self} has no filename!")
+            return False, {"reply_text": "Download failed, no file was saved.", "file_path": None}
+        
+        file_path_abs = self.save_path + self.file_name
+        if not os.path.exists(file_path_abs):
+            logger.error(f"Article {self} has a filename, but the file does not exist at that location!")
+            return False, {"reply_text": "Can't find file. Either the download failed or the file was moved.", "file_path": None}
+
+        return True, {}
+
+
+class ArticleAuthor(DownloadBaseModel):
+    article = ForeignKeyField(ArticleDownload, backref='authors')
+    author = CharField()
+
+
+class ArticleRelated(DownloadBaseModel):
+    # Related files, such as the full text of a paper, audio files, etc.
+    article = ForeignKeyField(ArticleDownload, backref='related')
+    related_file_name = TextField(default = '')
+
+
+
+
+
+
+
+def set_db(download_db_object):
+    download_db.initialize(download_db_object)
+    with download_db: # create tables (does nothing if they exist already)
+        download_db.create_tables([ArticleDownload, ArticleAuthor, ArticleRelated])
+
+