import logging logger = logging.getLogger(__name__) from peewee import * import os import datetime import configuration config = configuration.main_config["DOWNLOADS"] # set the nature of the db at runtime download_db = DatabaseProxy() class DownloadBaseModel(Model): class Meta: database = download_db ## == Article related models == ## class ArticleDownload(DownloadBaseModel): # in the beginning this is all we have article_url = TextField(default = '', unique=True) # fetch then fills in the metadata title = TextField(default='') summary = TextField(default = '') source_name = CharField(default = '') language = CharField(default = '') file_name = TextField(default = '') @property def save_path(self): return f"{config['local_storage_path']}/{self.download_date.year}/{self.download_date.strftime('%B')}/" @property def fname_nas(self, file_name=""): if self.download_date: if file_name: return f"NAS: {config['remote_storage_path']}/{self.download_date.year}/{self.download_date.strftime('%B')}/{file_name}" else: # return the self. name return f"NAS: {config['remote_storage_path']}/{self.download_date.year}/{self.download_date.strftime('%B')}/{self.file_name}" else: return None archive_url = TextField(default = '') pub_date = DateField(default = datetime.date.fromtimestamp(0)) download_date = DateField(default = datetime.date.today) slack_ts = FloatField(default = 0) # should be a fixed-length string but float is easier to sort by sent = BooleanField(default = False) archived_by = CharField(default = os.getenv("UNAME")) # need to know who saved the message because the file needs to be on their computer in order to get verified # verification happens in a different app, but the model has the fields here as well comment = TextField(default = '') verified = IntegerField(default = 0) # 0 = not verified, 1 = verified, -1 = marked as bad # authors # keywords # ... are added through foreignkeys # we will also add an attribute named message, to reference which message should be replied to. This attribute does not need to be saved in the db def to_dict(self): return { "id": self.id, "article_url": self.article_url, "title": self.title, "summary": self.summary, "source_name": self.source_name, "language": self.language, "file_name": self.file_name, "save_path": self.save_path, "fname_nas": self.fname_nas, "archive_url": self.archive_url, "pub_date": self.pub_date.strftime("%Y-%m-%d"), "download_date": self.download_date.strftime("%Y-%m-%d"), "sent": self.sent, "comment": self.comment, "related": [r.related_file_name for r in self.related], "authors": [a.author for a in self.authors] } def set_related(self, related): for r in related: if len(r) > 255: raise Exception("Related file name too long for POSTGRES") ArticleRelated.create( article = self, related_file_name = r ) def file_status(self): if not self.file_name: logger.error(f"Article {self} has no filename!") return False, {"reply_text": "Download failed, no file was saved.", "file_path": None} file_path_abs = self.save_path + self.file_name if not os.path.exists(file_path_abs): logger.error(f"Article {self} has a filename, but the file does not exist at that location!") return False, {"reply_text": "Can't find file. Either the download failed or the file was moved.", "file_path": None} return True, {} class ArticleAuthor(DownloadBaseModel): article = ForeignKeyField(ArticleDownload, backref='authors') author = CharField() class ArticleRelated(DownloadBaseModel): # Related files, such as the full text of a paper, audio files, etc. article = ForeignKeyField(ArticleDownload, backref='related') related_file_name = TextField(default = '') def set_db(download_db_object): download_db.initialize(download_db_object) with download_db: # create tables (does nothing if they exist already) download_db.create_tables([ArticleDownload, ArticleAuthor, ArticleRelated])