working (feature complete) news_fetch

news_fetch/configuration.py

@@ -2,8 +2,8 @@ import os
 import configparser
 import logging
 import time
-import shutil
-from datetime import datetime
+# import shutil
+# from datetime import datetime
 from peewee import SqliteDatabase, PostgresqlDatabase
 from rich.logging import RichHandler
 

news_fetch/runner.py

@@ -1,4 +1,5 @@
 """Main coordination of other util classes. Handles inbound and outbound calls"""
+from time import sleep
 import configuration
 models = configuration.models
 from threading import Thread
@@ -110,7 +111,8 @@ class Dispatcher(Thread):
             logger.error("Dispatcher.incoming_request called with no arguments")
             return
 
-        if is_new or (article.file_name == "" and article.verified == 0):
+        if is_new or (article.file_name == "" and article.verified == 0) \
+            or (not is_new and len(self.workers_in) == 1): # this is for upload
             # check for models that were created but were abandonned. This means they have missing information, most importantly no associated file
             # this overwrites previously set information, but that should not be too important
             ArticleWatcher(
@@ -121,7 +123,6 @@ class Dispatcher(Thread):
 
         else: # manually trigger notification immediatly
             logger.info(f"Found existing article {article}. Now sending")
-            self.article_complete_notifier(article)
 
 
 
@@ -142,6 +143,8 @@ if __name__ == "__main__":
         class PrintWorker:
             def send(self, article):
                 print(f"Uploaded article {article}")
+            def keep_alive(self): # keeps script running, because there is nothing else in the main thread
+                while True: sleep(1)
 
         articles = models.ArticleDownload.select().where(models.ArticleDownload.archive_url == "" or models.ArticleDownload.archive_url == "TODO:UPLOAD").execute()
         logger.info(f"Launching upload to archive for {len(articles)} articles.")
@@ -149,6 +152,9 @@ if __name__ == "__main__":
         dispatcher.workers_in = [{"UploadWorker": UploadWorker()}]
         dispatcher.workers_out = [{"PrintWorker": PrintWorker()}]
         dispatcher.start()
+        for a in articles:
+            dispatcher.incoming_request(article=a)
+        PrintWorker().keep_alive()
 
     else: # launch with full action
         try: