Working, refactored news_fetch, better documentation for launch

news_fetch/utils_worker/worker_template.py

@@ -7,15 +7,13 @@ class TemplateWorker(Thread):
     """Parent class for any subsequent worker of the article-download pipeline. They should all run in parallel, thus the Thread subclassing"""
     logger = logging.getLogger(__name__)
 
-    def __init__(self, *args, **kwargs) -> None:
+    def __init__(self, **kwargs) -> None:
         target = self._queue_processor # will be executed on Worker.start()
-        group = kwargs.get("group", None)
-        name = kwargs.get("name", None)
-
-        super().__init__(group=group, target=target, name=name)
+        self.keep_running = True
+        super().__init__(target=target, daemon=True)
         self._article_queue = []
         self.logger.info(f"Worker thread {self.__class__.__name__} initialized successfully")
-    
+
 
     def process(self, article_watcher):
         self._article_queue.append(article_watcher)#.article_model.article_url)
@@ -23,7 +21,7 @@ class TemplateWorker(Thread):
 
     def _queue_processor(self):
         """This method is launched by thread.run() and idles when self._article_queue is empty. When an external caller appends to the queue it jumps into action"""
-        while True: # PLEASE tell me if I'm missing an obvious better way of doing this!
+        while self.keep_running: # PLEASE tell me if I'm missing an obvious better way of doing this!
             if len(self._article_queue) == 0:
                 time.sleep(5)
             else:
@@ -39,3 +37,10 @@ class TemplateWorker(Thread):
             article = article_watcher.article
             article = action(article) # action updates the article object but does not save the change
             article.save()
+            article_watcher.update(self.__class__.__name__)
+
+
+    def stop(self):
+        self.logger.info(f"Stopping worker {self.__class__.__name__} whith {len(self._article_queue)} articles left in queue")
+        self.keep_running = False
+        self.join()

news_fetch/utils_worker/workers.py

@@ -25,7 +25,7 @@ class DownloadWorker(TemplateWorker):
             action = self.dl_runner
 
         super()._handle_article(article_watcher, action)
-        article_watcher.download_completed = True
+        # article_watcher.download_completed = True
 
 
 
@@ -36,7 +36,7 @@ class FetchWorker(TemplateWorker):
     def _handle_article(self, article_watcher):
         action = get_description # function
         super()._handle_article(article_watcher, action)
-        article_watcher.fetch_completed = True
+        # article_watcher.fetch_completed = True
 
 
 
@@ -52,7 +52,7 @@ class UploadWorker(TemplateWorker):
             return run_upload(*args, **kwargs)
 
         super()._handle_article(article_watcher, action)
-        article_watcher.upload_completed = True
+        # article_watcher.upload_completed = True
 
 
 
@@ -63,4 +63,4 @@ class CompressWorker(TemplateWorker):
     def _handle_article(self, article_watcher):
         action = shrink_pdf
         super()._handle_article(article_watcher, action)
-        article_watcher.compression_completed = True
+        # article_watcher.compression_completed = True