Better structure

app/runner.py

@@ -7,7 +7,7 @@ logger = logging.getLogger(__name__)
 
 from utils_mail import runner as mail_runner
 from utils_slack import runner as slack_runner
-from utils.workers import CompressWorker, DownloadWorker, FetchWorker, UploadWorker
+from utils_worker.workers import CompressWorker, DownloadWorker, FetchWorker, UploadWorker
 
 
 class ArticleWatcher:
@@ -174,7 +174,8 @@ if __name__ == "__main__":
         logger.info(f"Launching upload to archive for {len(urls)} urls.")
         coordinator.manual_processing(urls, [UploadWorker()])
     elif "check" in sys.argv:
-        logger.info("Not implemented yet.")
+        from utils_check import runner as check_runner
+        check_runner.verify_unchecked()
     else: # launch with full action
         kwargs = {
             "worker_download" : DownloadWorker(),

app/utils_check/runner.py

@@ -0,0 +1,285 @@
+from rich.console import Console
+from rich.table import Table
+from rich.columns import Columns
+from rich.rule import Rule
+console = Console()
+hline = Rule(style="white")
+
+import os
+import subprocess
+from slack_sdk import WebClient
+import configuration
+models = configuration.models
+
+u_options = {
+    "ENTER" : "Accept PDF as is. It gets marked as verified",
+    "D" : "set languange to DE and set verified",
+    "E" : "set languange to EN and set verified",
+    "O" : "set other language (prompted)",
+    "R" : "set related files (prompted multiple times)",
+    "B" : "reject and move to folder BAD",
+    "L" : "leave file as is, do not send reaction"
+}
+
+
+bot_client = WebClient(
+    token = configuration.parsed["SLACK"]["auth_token"]
+)
+
+
+
+
+
+def file_overview(file_url: str, file_attributes: list, options: dict) -> None:
+    """Prints a neat overview of the current article"""
+    file_table = Table(
+        title = file_url,
+        row_styles = ["white", "bright_black"],
+        min_width = 150
+    )
+
+    file_table.add_column("Attribute", justify = "right", no_wrap = True)
+    file_table.add_column("Value set by auto_news")
+    file_table.add_column("Status", justify = "right")
+    for attr in file_attributes:
+        file_table.add_row(attr["name"], attr["value"], attr["status"])
+
+    
+    option_key = "\n".join([f"[[bold]{k}[/bold]]" for k in options.keys()])
+    option_action = "\n".join([f"[italic]{k}[/italic]" for k in options.values()])
+    columns = Columns([option_key, option_action])
+
+    console.print(file_table)
+    console.print("Your options:")
+    console.print(columns)
+
+
+def send_reaction_to_slack_thread(article, reaction):
+    """Sends the verification status as a reaction to the associated slack thread. This will significantly decrease load times of the bot"""
+    messages = models.Message.select().where(models.Message.text.contains(article.article_url))
+    # TODO rewrite this shit
+    if len(messages) > 5:
+        print("Found more than 5 messages. Aborting reactions...")
+        return
+    for m in messages:
+        if not m.has_single_url:
+            print("Found thread but won't send reaction because thread has multiple urls")
+            pass
+        else:
+            ts = m.slack_ts
+            bot_client.reactions_add(
+                channel=configuration.parsed["SLACK"]["archive_id"],
+                name=reaction,
+                timestamp=ts
+            )
+            print("Sent reaction to message")
+
+def prompt_language(query):
+    not_set = True
+    while not_set:
+        uin = input("Set language (nation-code, 2 letters) ")
+        if len(uin) != 2:
+            print("Bad code, try again")
+        else:
+            not_set = False
+            query.language = uin
+            query.save()
+
+
+def prompt_related(query):
+    file_list = []
+    finished = False
+    while not finished:
+        uin = input("Additional file for article? Type '1' to cancel ")
+        if uin == "1":
+            query.set_related(file_list)
+            finished = True
+        else:
+            file_list.append(uin)
+
+
+def prompt_new_fname(query):
+    uin = input("New fname? ")
+    old_fname =  query.file_name
+    query.file_name = uin
+    query.verified = 1
+    if old_fname != "":
+        os.remove(query.save_path + old_fname)
+    query.save()    
+
+
+
+def reject_article(article):
+    article.verified = -1
+    article.save()
+    print("Article marked as bad")
+    # also update the threads to not be monitored anymore
+    send_reaction_to_slack_thread(article, "x")
+
+
+def unreject_article(query):
+    query.verified = 1
+    query.save()
+    # os.rename(badpdf, fname)
+    print("File set to verified")
+
+
+def accept_article(article, last_accepted):
+    article.verified = 1
+    article.save()
+    print("Article accepted as GOOD")
+
+    # also update the threads to not be monitored anymore
+    send_reaction_to_slack_thread(article, "white_check_mark")
+
+    """linked = None
+    try:
+        thread = message_models.Thread.get(id = last_accepted.id + 1)
+        rel = message_models.get_referenced_articles(thread, article_models.ArticleDownload)
+        assert len(rel) == 1 and rel[0] == article
+        linked = thread
+    except: # if the above, naive method (just increment by one), fails, resort to brute search.
+        print("Bruteforcing search")
+        for t in message_models.Thread.select():
+            rel = message_models.get_referenced_articles(t, article_models.ArticleDownload)
+            if len(rel) == 1 and rel[0] == article:
+                linked = t
+                break
+        
+    if linked:
+        linked.initiator_message.is_processed_override = 1
+        linked.initiator_message.save()
+        print("Message overwritten to PROCESSED")
+
+    else:
+        print("No matching thread found")"""
+    return "" # linked
+
+
+
+
+
+
+def verify_unchecked():
+    query = models.ArticleDownload.select().where(models.ArticleDownload.verified == 0).execute()
+    last_linked = None
+
+    for article in query:
+        console.print(hline)
+        core_info = []
+        for e, name in zip([article.save_path, article.file_name, article.title, article.language], ["Save path", "File name", "Title", "Language"]):
+            entry = {
+                "status" : "[red]██[/red]" if (len(e) == 0 or e == -1) else "[green]██[/green]",
+                "value" : e if len(e) != 0 else "not set",
+                "name" : name
+            }
+            core_info.append(entry)
+        
+        try:
+            subprocess.Popen(["evince", f"file://{os.path.join(article.save_path, article.file_name)}"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            # supress evince gtk warnings
+        except Exception as e:
+            print(str(list((-1, e))))
+            continue
+
+        
+
+        file_overview(
+            file_url = article.article_url, 
+            file_attributes=core_info,
+            options = u_options
+        )
+
+
+        proceed = False
+        while not proceed:
+            proceed = False
+            uin = input("Choice ?").lower()
+            if uin == "":
+                last_linked = accept_article(article, last_linked) # last linked accelerates the whole process
+                proceed = True
+            elif uin == "d":
+                article.language = "de"
+                article.verified = 1
+                article.save()
+                proceed = True
+            elif uin == "e":
+                article.language = "en"
+                article.verified = 1
+                article.save()
+                proceed = True
+            elif uin == "o":
+                prompt_language(article)
+            elif uin == "r":
+                prompt_related(article)
+            elif uin == "b":
+                reject_article(article)
+                proceed = True
+            elif uin == "l":
+                # do nothing
+                proceed = True
+            else:
+                print("Invalid input")
+
+
+
+
+# def verify_bad():
+#     b_options = {
+#         "ENTER":"Accept pdf as fixed",
+#         "B": "Keep pdf in BAD.",
+#         "R" : "set related files (prompted multiple times)",
+#         "C" : "Change the saved file-name and set as verified."
+#         }
+#     query = article_models.ArticleDownload.select().where(article_models.ArticleDownload.verified == -1).execute()
+
+#     for q in query:
+#         pdf = q.file_name
+#         save_dir = get_save_path(q)
+#         fname = save_dir + "BAD/" + pdf
+#         try:
+#             subprocess.call(["xdg-open", fname])
+#         except:
+#             print(f"[{testvar}██{testvar}] PDF moved:")
+#             print(fname)
+#             continue
+
+#         status_pdf = f"{testvar}██{testvar}"
+#         if "just a moment" in pdf:
+#             status_pdf = f"{testvar}██{testvar}"
+        
+#         language = q.language
+#         status_language = f"{testvar}██{testvar}"
+#         if len(language) == 0:
+#             status_language = f"{testvar}██{testvar}"
+
+
+#         print_status_options(
+#             status=u_status.format(
+#                 url = q.article_url,
+#                 status_pdf = status_pdf,
+#                 pdf = pdf[:80],
+#                 status_language = status_language,
+#                 language = language
+#             ),
+#             options = b_options)
+
+
+
+#         proceed = False
+#         while not proceed:
+#             proceed = False
+#             uin = input("Choice? ").lower()
+#             if uin == "":
+#                 unreject_article(q)
+#                 proceed = True
+#             elif uin == "b":
+#                 proceed = True
+#             elif uin == "r":
+#                 prompt_related(q)
+#             elif uin == "c":
+#                 prompt_new_fname(q)
+#                 proceed = True
+#             else:
+#                 print("Invalid input")
+

app/utils_worker/compress/runner.py

@@ -31,10 +31,3 @@ def shrink_pdf(article):
         logger.error(f"Could not run the compression! {c.stderr.decode()} - {c.stdout.decode()}")
     
     return article
-
-
-
-
-
-# gs -sDEVICE=pdfwrite -dPDFSETTINGS=/screen -dNOPAUSE -dBATCH -sOutputFile=out.pdf 
-# ; mv -f temp.pdf file.pdf

app/utils_worker/worker_template.py

@@ -29,7 +29,7 @@ class TemplateWorker(Thread):
                 time.sleep(5)
             else:
                 article_watcher = self._article_queue.pop(0)
-                self.logger.info(f"{self.__class__.__name__} is now processing an article")
+                self.logger.info(f"{self.__class__.__name__} is now processing article ({len(self._article_queue)} in queue)")
                 self._handle_article(article_watcher)