135 lines
4.4 KiB
Python
135 lines
4.4 KiB
Python
import logging
|
|
logger = logging.getLogger(__name__)
|
|
|
|
from peewee import *
|
|
import os
|
|
import datetime
|
|
import configuration
|
|
|
|
config = configuration.main_config["DOWNLOADS"]
|
|
|
|
# set the nature of the db at runtime
|
|
download_db = DatabaseProxy()
|
|
|
|
|
|
class DownloadBaseModel(Model):
|
|
class Meta:
|
|
database = download_db
|
|
|
|
|
|
|
|
## == Article related models == ##
|
|
class ArticleDownload(DownloadBaseModel):
|
|
# in the beginning this is all we have
|
|
article_url = TextField(default = '', unique=True)
|
|
|
|
# fetch then fills in the metadata
|
|
title = TextField(default='')
|
|
|
|
summary = TextField(default = '')
|
|
source_name = CharField(default = '')
|
|
language = CharField(default = '')
|
|
|
|
|
|
file_name = TextField(default = '')
|
|
@property
|
|
def save_path(self):
|
|
return f"{config['local_storage_path']}/{self.download_date.year}/{self.download_date.strftime('%B')}/"
|
|
@property
|
|
def fname_nas(self, file_name=""):
|
|
if self.download_date:
|
|
if file_name:
|
|
return f"NAS: {config['remote_storage_path']}/{self.download_date.year}/{self.download_date.strftime('%B')}/{file_name}"
|
|
else: # return the self. name
|
|
return f"NAS: {config['remote_storage_path']}/{self.download_date.year}/{self.download_date.strftime('%B')}/{self.file_name}"
|
|
else:
|
|
return None
|
|
|
|
|
|
archive_url = TextField(default = '')
|
|
pub_date = DateField(default = datetime.date.fromtimestamp(0))
|
|
download_date = DateField(default = datetime.date.today)
|
|
|
|
slack_ts = FloatField(default = 0) # should be a fixed-length string but float is easier to sort by
|
|
|
|
sent = BooleanField(default = False)
|
|
|
|
archived_by = CharField(default = os.getenv("UNAME"))
|
|
# need to know who saved the message because the file needs to be on their computer in order to get verified
|
|
# verification happens in a different app, but the model has the fields here as well
|
|
comment = TextField(default = '')
|
|
verified = IntegerField(default = 0) # 0 = not verified, 1 = verified, -1 = marked as bad
|
|
|
|
# authors
|
|
# keywords
|
|
# ... are added through foreignkeys
|
|
# we will also add an attribute named message, to reference which message should be replied to. This attribute does not need to be saved in the db
|
|
|
|
def to_dict(self):
|
|
return {
|
|
"id": self.id,
|
|
"article_url": self.article_url,
|
|
"title": self.title,
|
|
"summary": self.summary,
|
|
"source_name": self.source_name,
|
|
"language": self.language,
|
|
"file_name": self.file_name,
|
|
"save_path": self.save_path,
|
|
"fname_nas": self.fname_nas,
|
|
"archive_url": self.archive_url,
|
|
"pub_date": self.pub_date.strftime("%Y-%m-%d"),
|
|
"download_date": self.download_date.strftime("%Y-%m-%d"),
|
|
"sent": self.sent,
|
|
"comment": self.comment,
|
|
"related": [r.related_file_name for r in self.related],
|
|
"authors": [a.author for a in self.authors]
|
|
}
|
|
|
|
|
|
|
|
def set_related(self, related):
|
|
for r in related:
|
|
if len(r) > 255:
|
|
raise Exception("Related file name too long for POSTGRES")
|
|
|
|
ArticleRelated.create(
|
|
article = self,
|
|
related_file_name = r
|
|
)
|
|
|
|
def file_status(self):
|
|
if not self.file_name:
|
|
logger.error(f"Article {self} has no filename!")
|
|
return False, {"reply_text": "Download failed, no file was saved.", "file_path": None}
|
|
|
|
file_path_abs = self.save_path + self.file_name
|
|
if not os.path.exists(file_path_abs):
|
|
logger.error(f"Article {self} has a filename, but the file does not exist at that location!")
|
|
return False, {"reply_text": "Can't find file. Either the download failed or the file was moved.", "file_path": None}
|
|
|
|
return True, {}
|
|
|
|
|
|
class ArticleAuthor(DownloadBaseModel):
|
|
article = ForeignKeyField(ArticleDownload, backref='authors')
|
|
author = CharField()
|
|
|
|
|
|
class ArticleRelated(DownloadBaseModel):
|
|
# Related files, such as the full text of a paper, audio files, etc.
|
|
article = ForeignKeyField(ArticleDownload, backref='related')
|
|
related_file_name = TextField(default = '')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def set_db(download_db_object):
|
|
download_db.initialize(download_db_object)
|
|
with download_db: # create tables (does nothing if they exist already)
|
|
download_db.create_tables([ArticleDownload, ArticleAuthor, ArticleRelated])
|
|
|
|
|