56 lines
1.6 KiB
Python
56 lines
1.6 KiB
Python
from .worker_template import TemplateWorker
|
|
from .download.browser import PDFDownloader
|
|
from .download.youtube import YouTubeDownloader
|
|
from .fetch.runner import get_description
|
|
from .upload.runner import upload_to_archive as run_upload
|
|
|
|
|
|
import time
|
|
import logging
|
|
logger = logging.getLogger(__name__)
|
|
|
|
class DownloadWorker(TemplateWorker):
|
|
def __init__(self) -> None:
|
|
self.dl_runner = PDFDownloader().download
|
|
self.yt_runner = YouTubeDownloader().save_video
|
|
super().__init__()
|
|
|
|
def _handle_article(self, article_watcher):
|
|
article = article_watcher.article
|
|
u = article.article_url
|
|
|
|
if "youtu.be/" in u or "youtube.com/" in u:
|
|
action = self.yt_runner
|
|
else:
|
|
action = self.dl_runner
|
|
|
|
super()._handle_article(article_watcher, action)
|
|
# article_watcher.download_completed = True
|
|
|
|
|
|
|
|
class FetchWorker(TemplateWorker):
|
|
def __init__(self) -> None:
|
|
super().__init__()
|
|
|
|
def _handle_article(self, article_watcher):
|
|
action = get_description # function
|
|
super()._handle_article(article_watcher, action)
|
|
# article_watcher.fetch_completed = True
|
|
|
|
|
|
|
|
class UploadWorker(TemplateWorker):
|
|
def __init__(self) -> None:
|
|
super().__init__()
|
|
|
|
|
|
|
|
def _handle_article(self, article_watcher):
|
|
def action(*args, **kwargs):
|
|
time.sleep(10) # uploads to archive are throttled to 15/minute, but 5s still triggers a blacklisting
|
|
return run_upload(*args, **kwargs)
|
|
|
|
super()._handle_article(article_watcher, action)
|
|
# article_watcher.upload_completed = True
|