remoll/coss_archiving
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Bug fixes, makefile for launch

Browse Source
This commit is contained in:
Remy Moll
2022-12-09 11:19:45 +01:00
parent 24b3bc3b51
commit 647944d23c
25 changed files with 321 additions and 300 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
news_fetch/utils_storage/models.py
View File

@@ -8,8 +8,7 @@ import configuration
import datetime
from . import helpers
config = configuration.main_config["DOWNLOADS"]
slack_config = configuration.main_config["SLACK"]
downloads_config = configuration.config["downloads"]
FILE_SIZE_THRESHOLD = 15 * 1024 * 1024 # 15MB
@@ -34,7 +33,8 @@ class ArticleDownload(DownloadBaseModel):
def is_title_bad(self): # add incrementally
return "PUR-Abo" in self.title \
or "Redirecting" in self.title \
or "Error while running fetch" in self.title
or "Error while running fetch" in self.title \
or self.title == ""
summary = TextField(default = '')
source_name = CharField(default = '')
@@ -44,14 +44,14 @@ class ArticleDownload(DownloadBaseModel):
file_name = TextField(default = '')
@property
def save_path(self):
return f"{config['local_storage_path']}/{self.download_date.year}/{self.download_date.strftime('%B')}/"
return f"{downloads_config['local_storage_path']}/{self.download_date.year}/{self.download_date.strftime('%B')}/"
@property
def fname_nas(self, file_name=""):
if self.download_date:
if file_name:
return f"NAS: {config['remote_storage_path']}/{self.download_date.year}/{self.download_date.strftime('%B')}/{file_name}"
return f"NAS: {downloads_config['remote_storage_path']}/{self.download_date.year}/{self.download_date.strftime('%B')}/{file_name}"
else: # return the self. name
return f"NAS: {config['remote_storage_path']}/{self.download_date.year}/{self.download_date.strftime('%B')}/{self.file_name}"
return f"NAS: {downloads_config['remote_storage_path']}/{self.download_date.year}/{self.download_date.strftime('%B')}/{self.file_name}"
else:
return None
@property
@@ -102,18 +102,22 @@ class ArticleDownload(DownloadBaseModel):
answer_files = []
# displays the summary in a blockquote
status = self.file_status
if status == 1: # file_name was empty
return None # there has been an error do not send any message
elif status == 2: # no file found at specified location
answer_text += f"*{self.title}*\n{summary}\nFilename: {self.file_name}"
elif status == 3: # file found but deemed too big
location = f"File not sent directly. Location on NAS:\n`{self.fname_nas}`"
answer_text += f"*{self.title}*\n{summary}\n{location}"
else: # everything nominal
try:
self.ensure_file_present()
answer_text += f"*{self.title}*\n{summary}"
answer_files.append(self.save_path + self.file_name)
except Exception as e:
msg = e.args[0]
logger.error(f"Article {self} has file-issues: {msg}")
if "file too big" in msg:
location = f"File too big to send directly. Location on NAS:\n`{self.fname_nas}`"
answer_text += f"*{self.title}*\n{summary}\n{location}"
else: # file not found, or filename not set
raise e
# reraise the exception, so that the caller can handle it
# then the related files
if self.related:
rel_text = "Related files on NAS:"
@@ -144,19 +148,14 @@ class ArticleDownload(DownloadBaseModel):
related_file_name = r
)
@property
def file_status(self):
"""0 = file exists, 1 = no file name!, 2 = file does not exit,3 = file exists but is too large"""
def ensure_file_present(self):
if not self.file_name:
logger.error(f"Article {self} has no filename!")
return 2
raise Exception("no filename")
file_path_abs = self.save_path + self.file_name
if not os.path.exists(file_path_abs):
logger.error(f"Article {self} has a filename, but the file does not exist at that location!")
return 2
raise Exception("file not found")
if (os.path.splitext(file_path_abs)[1] != ".pdf") or (os.path.getsize(file_path_abs) > FILE_SIZE_THRESHOLD):
logger.warning(f"Article {self} has a file that exceeds the file size limit.")
return 3
raise Exception("file too big")