minor corrections

2022-06-23 15:05:59 +02:00 · 2022-06-23 15:05:59 +02:00 · 12a7de91ed
commit 12a7de91ed
parent ac9e988af3
13 changed files with 218 additions and 32 deletions
--- a/3
+++ b/3
@ -30,9 +30,10 @@ RUN chown -R autonews:autonews /app
 USER autonews
 RUN export PATH=/home/autonews/.local/bin:$PATH

+COPY requirements.txt /app/requirements.txt
+RUN python3 -m pip install -r /app/requirements.txt

 COPY app /app/auto_news
 WORKDIR /app/auto_news
-RUN python3 -m pip install -r requirements.txt

 ENTRYPOINT ["python3", "runner.py"]
--- a/app/requirements.txt
+++ b/app/requirements.txt
@ -1,9 +0,0 @@
-peewee
-selenium
-youtube-dl
-waybackpy
-slack_bolt # relies on slack_sdk
-newspaper3k
-htmldate
-markdown
-rich
--- a/app/runner.py
+++ b/app/runner.py
@ -31,7 +31,7 @@ class ArticleWatcher:
        # first step: gather metadata
        if self.fetch and self.upload:
            self.fetch.process(self) # this will call the update_status method
-            self.upload.process(self) # idependdent from the rest
+            self.upload.process(self) # idependent from the rest
        else: # the full kwargs were not provided, only do a manual run
            # overwrite update_status() because calls from the workers will result in erros
            self.update_status = lambda completed: logger.info(f"Completed action {completed}")
--- a/app/utils_worker/download/browser.py
+++ b/app/utils_worker/download/browser.py
@ -28,7 +28,7 @@ class PDFDownloader:
        if os.getenv("HEADLESS", "false") == "true":
            options.add_argument('--headless')
        else:
-            self.logger.warning("Opening browser GUI because of 'HEADLESS=true'")
+            self.logger.warning("Opening browser GUI because of 'HEADLESS=false'")

        options.set_preference('print.save_as_pdf.links.enabled', True)
        # Just save if the filetype is pdf already, does not work!
@ -46,7 +46,7 @@ class PDFDownloader:
        #         log_path = f'{config["local_storage_path"]}/geckodriver.log'
        # ))
        self.driver = webdriver.Remote(
-            command_executor = 'http://localhost:4444',
+            command_executor = 'http://geckodriver:4444',
            options = options,
            # can't set log path...
        )
@ -64,13 +64,17 @@ class PDFDownloader:
    def finish(self):
        if self.running:
            self.logger.info("Exiting gecko driver")
-            self.driver.quit()
+            try:
+                self.driver.quit()
+                time.sleep(10)
+            except:
+                self.logger.critical("Connection to the driver broke off")
            self.running = False
        else:
            self.logger.info("Gecko driver not yet running")

    def download(self, article_object):
-        sleep_time = 1
+        sleep_time = 2
        self.autostart()
        url = article_object.article_url

@ -87,7 +91,7 @@ class PDFDownloader:
        # in the mean time, get a page title if required
        if article_object.is_title_bad:
            article_object.title = self.driver.title.replace(".pdf", "")
-            # will be propagated to dst as well
+            # will be propagated to the saved file (dst) as well

        fname = article_object.fname_template
        dst = os.path.join(article_object.save_path, fname)
@ -110,7 +114,7 @@ class PDFDownloader:
        else:
            article_object.file_name = ""
        
-        return article_object  # this change is saved later manually
+        return article_object  # this change is saved later by the external caller


    def get_exisiting_pdf(self, url, dst):
--- a/app/utils_worker/upload/runner.py
+++ b/app/utils_worker/upload/runner.py
@ -1,3 +1,4 @@
+import time
 from waybackpy import WaybackMachineSaveAPI # upload to archive.org
 import logging
 logger = logging.getLogger(__name__)
@ -11,6 +12,8 @@ def upload_to_archive(article_object):
        archive_url = wayback.save()
        # logger.info(f"{url} uploaded to archive successfully")
        article_object.archive_url = archive_url
+        # time.sleep(4) # Archive Uploads rate limited to 15/minute
+
    except Exception as e:
        article_object.archive_url = "Error while uploading: {}".format(e)
        logger.error(f"Error while generating archive url: {e}")
--- a/app/utils_worker/workers.py
+++ b/app/utils_worker/workers.py
@ -44,10 +44,14 @@ class UploadWorker(TemplateWorker):
    def __init__(self) -> None:
        super().__init__()
    
+
+
    def _handle_article(self, article_watcher):
-        action = run_upload # function
+        def action(*args, **kwargs):
+            run_upload(*args, **kwargs)
+            time.sleep(5) # uploads to archive are throttled to 15/minute
+
        super()._handle_article(article_watcher, action)
-        time.sleep(4) # Archive Uploads rate limited to 15/minute
        article_watcher.upload_completed = True


--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@ -7,11 +7,9 @@ services:
    image: auto_news:latest
    volumes:
      - ${CONTAINER_DATA}:/app/file_storage
-      - ${HOSTS_FILE}:/etc/hosts
-
      - ${CODE:-/dev/null}:/code # not set in prod, defaults to /dev/null
+      - ${XSOCK-/dev/null}:${XSOCK-/tmp/sock}
      - ${XAUTHORITY-/dev/null}:/home/auto_news/.Xauthority
-    network_mode: host
    environment:
      - DISPLAY=$DISPLAY
      - TERM=xterm-256color # colored logs
@ -28,10 +26,12 @@ services:
  geckodriver:
    image: selenium/standalone-firefox:101.0
    volumes:
-      - ${XAUTHORITY-/dev/null}:/home/seluser/.Xauthority
+      - ${XSOCK-/dev/null}:${XSOCK-/tmp/sock}
+      - ${XAUTHORITY-/dev/null}:/home/auto_news/.Xauthority
    environment:
      - DISPLAY=$DISPLAY
      - START_VNC=false
      - START_XVFB=false
    user: 1001:1001
-    network_mode: host
+    expose: # exposed to other docker-compose services only
+    - "4444"
--- a/env/check
+++ b/env/check
@ -4,6 +4,7 @@ CONTAINER_DATA=~/Bulk/COSS/Downloads/auto_news.container
 HOSTS_FILE=~/Bulk/COSS/Downloads/auto_news.container/dependencies/hosts

 XAUTHORTIY=$XAUTHORTIY
+XSOCK=/tmp/.X11-unix

 DEBUG=false
 CHECK=true
--- a/env/debug
+++ b/env/debug
@ -5,6 +5,7 @@ HOSTS_FILE=~/Bulk/COSS/Downloads/auto_news.container/dependencies/hosts

 CODE=./
 XAUTHORTIY=$XAUTHORTIY
+XSOCK=/tmp/.X11-unix

 DEBUG=true
 CHECK=false
--- a/misc/exctract_from_mail_backup.py
+++ b/misc/exctract_from_mail_backup.py
@ -19,5 +19,5 @@ for f in all_files:

 print("Saved {} urls".format(len(all_urls)))

-with open("mails_export.json", "w") as f:
+with open("media_mails_export.json", "w") as f:
    json.dump(all_urls, f)  
--- a/misc/gather_media_files.py
+++ b/misc/gather_media_files.py
@ -1,10 +1,14 @@
 import sys
+from webbrowser import get
 sys.path.append("../app")
 import runner
 import logging
 logger = logging.getLogger()
 import json

+from rich.console import Console
+from rich.table import Table
+console = Console()

 logger.info("Overwriting production values for single time media-fetch")
 runner.configuration.models.set_db(
@ -40,13 +44,30 @@ def fetch():


 def show():
-    sel = runner.models.ArticleDownload.select()
-    entries = ["title"]  #, "article_url", "archive_url"]
+
+    t = Table(
+        title = "ArticleDownloads",
+        row_styles = ["white", "bright_black"],
+    )
+    
+    entries = ["title", "article_url", "archive_url", "authors"]

    for e in entries:
-        r = [t.title for t in sel]
-        print(r)
-        # print([t for t in r])
+        t.add_column(e, justify = "right")
+
+    sel = runner.models.ArticleDownload.select()
+
+    for s in sel:
+        c = [getattr(s, e) for e in entries]#
+        c[-1] = str([a.author for a in c[-1]])
+        print(c)
+        t.add_row(*c)

    
+    console.print(t)
+
+
+
+
+# fetch()
 show()
--- a/misc/media_urls.json
+++ b/misc/media_urls.json
@ -1 +1,151 @@
-["https://www.nesta.org.uk/report/digital-democracy-the-tools-transforming-political-engagement/", "https://media.nesta.org.uk/documents/digital_democracy.pdf", "https://context-cdn.washingtonpost.com/notes/prod/default/documents/c3c41863-be9e-4246-9ed9-e43aedd013f9/note/4e677597-f403-4c9b-b838-f5613d79b341", "https://context-cdn.washingtonpost.com/notes/prod/default/documents/6d274110-a84b-4694-96cd-6a902207d2bd/note/733364cf-0afb-412d-a5b4-ab797a8ba154.#page=1", "https://www.judiciary.senate.gov/fisa-investigation", "https://www.state.gov/fact-sheet-activity-at-the-wuhan-institute-of-virology/", "https://www.whitehouse.gov/trump-administration-accomplishments/", "https://www.whitehouse.gov/wp-content/uploads/2021/01/IPS-Final-Declass.pdf", "https://www.finance.senate.gov/imo/media/doc/Oversight,%2012-23-20,%20Memo%20on%20World%20Vision%20Investigation.pdf", "https://justthenews.com/sites/default/files/2020-12/BidenArcher4-13-14.pdf", "https://www.hsgac.senate.gov/imo/media/doc/Johnson-Grassley%20Submission%202020-12-09.pdf", "https://navarroreport.com/", "https://got-freedom.org/wp-content/uploads/2020/12/HAVA-and-Non-Profit-Organization-Report-FINAL-W-Attachments-and-Preface-121420.pdf", "https://www.depernolaw.com/uploads/2/7/0/2/27029178/antrim_michigan_forensics_report_%5B121320%5D_v2_%5Bredacted%5D.pdf", "https://www.hsgac.senate.gov/imo/media/doc/HSGAC_Finance_Report_FINAL.pdf", "https://www.scribd.com/document/487040771/Emails-About-FBI-Receipt-Fusion-GPS-Thumb-Drive", "https://cdn.epoch.cloud/assets/static_assets/Voter-Fraud-Allegations-Infographic-Epoch-Times.jpg", "https://www.hsgac.senate.gov/imo/media/doc/Lync%20and%20text%20messages%20between%20and%20among%20DOJ%20and%20FBI%20employees.pdf", "https://www.hsgac.senate.gov/imo/media/doc/DOJ%20Docs%20Combined.pdf", "https://www.hsgac.senate.gov/imo/media/doc/FBI%20Productions%20Combined%20-%20updated_FINAL.pdf", "https://www.hsgac.senate.gov/imo/media/doc/STATE_combined.pdf", "https://cdn.epoch.cloud/assets/static_assets/epochtimes-infographic-war-on-president-trump.jpg", "https://centipedenation.com/transmissions/miles-guo-dropping-bombs-hunter-biden-sex-tapes-and-other-evidence-of-the-ccps-infiltration-of-the-u-s/", "https://www.finance.senate.gov/imo/media/doc/2020-11-18%20HSGAC%20-%20Finance%20Joint%20Report%20Supplemental.pdf", "https://www.scribd.com/document/479781400/Steele-Spreadsheet-1", "https://www.zerohedge.com/political/jim-comey-ignored-state-department-whistleblower-hillarys-crimes-classified-material", "https://www.judicialwatch.org/wp-content/uploads/2020/10/JW-v-State-Steele-Oct-2020-prod-00968.pdf", "https://justthenews.com/sites/default/files/2020-10/requested%20email.pdf", "https://www.tagblatt.ch/kultur/sommertipps-20-buchempfehlungen-fuer-prominente-wir-haben-die-besten-buecher-fuer-jeden-charakter-zusammengetragen-ld.2159339", "https://www.tagblatt.ch/kultur/sommertipps-20-buchempfehlungen-fuer-prominente-wir-haben-die-besten-buecher-fuer-jeden-charakter-zusammengetragen-ld.2159339", "https://greennetproject.org/en/2018/11/27/prof-dirk-helbing-es-braucht-vor-allem-tolle-ideen-in-die-sich-die-leute-verlieben/", "https://www.nature.com/articles/news.2010.351", "https://www.focus.de/panorama/welt/tid-19265/gastkommentar-nutzt-die-moeglichkeiten-des-computers_aid_534372.html", "http://www.ccss.ethz.ch/Response/index.html", "https://www.hpcwire.com/2011/05/06/simulating_society_at_the_global_scale/", "https://www.say.media/article/la-mort-par-algorithme", "https://www.say.media/article/la-mort-par-algorithme", "https://www.nzz.ch/panorama/wie-kann-eine-massenpanik-verhindert-werden-ld.1614761", "https://www.theglobalist.com/democracy-technology-innovation-society-internet/","https://www.theglobalist.com/capitalism-democracy-technology-surveillance-privacy/","https://www.theglobalist.com/google-artificial-intelligence-big-data-technology-future/","https://www.theglobalist.com/fascism-big-data-artificial-intelligence-surveillance-democracy/","https://www.theglobalist.com/technology-big-data-artificial-intelligence-future-peace-rooms/","https://www.theglobalist.com/technology-society-sustainability-future-humanity/","https://www.theglobalist.com/society-technology-peace-sustainability/","https://www.theglobalist.com/democracy-technology-social-media-artificial-intelligence/","https://www.theglobalist.com/financial-system-reform-economy-internet-of-things-capitalism/","https://www.theglobalist.com/capitalism-society-equality-sustainability-crowd-funding/","https://www.theglobalist.com/united-nations-world-government-peace-sustainability-society/","https://www.theglobalist.com/world-economy-sustainability-environment-society/"]
+[
+    "https://www.swissinfo.ch/ger/wirtschaft/koennen-ki-und-direkte-demokratie-nebeneinander-bestehen-/47542048",
+    "https://www.zeit.de/2011/33/CH-Oekonophysik",
+    "https://ourworld.unu.edu/en/green-idea-self-organizing-traffic-signals",
+    "https://www.youtube.com/watch?v=-FQD4ie9UYA",
+    "https://www.brandeins.de/corporate-services/mck-wissen/mck-wissen-logistik/schwaermen-fuer-das-optimum",
+    "https://www.youtube.com/watch?v=upQM4Xzh8zM",
+    "https://www.youtube.com/watch?v=gAkoprZmW4k",
+    "https://www.youtube.com/watch?v=VMzfDVAWXHI&t=1s",
+    "https://www.youtube.com/watch?v=1SwTiIlkndE",
+    "https://www.informatik-aktuell.de/management-und-recht/digitalisierung/digitale-revolution-und-oekonomie-40-quo-vadis.html",
+    "https://www.youtube.com/watch?v=cSvvH0SBFOw",
+    "https://www.linkedin.com/posts/margit-osterloh-24198a104_pl%C3%A4doyer-gegen-sprechverbote-ugcPost-6925702100450480129-K7Dl?utm_source=linkedin_share&utm_medium=member_desktop_web",
+    "https://www.nebelspalter.ch/plaedoyer-gegen-sprechverbote",
+    "https://falling-walls.com/people/dirk-helbing/",
+    "https://digitalsensemaker.podigee.io/3-2-mit-dirk-helbing",
+    "https://www.blick.ch/wirtschaft/musk-als-hueter-der-redefreiheit-eth-experte-sagt-musks-vorhaben-hat-potenzial-aber-id17437811.html",
+    "https://www.trend.at/standpunkte/mit-verantwortung-zukunft-10082300",
+    "https://www.pantarhei.ch/podcast/",
+    "https://ethz.ch/en/industry/industry/news/data/2022/04/intelligent-traffic-lights-for-optimal-traffic-flow.html",
+    "https://ethz.ch/de/wirtschaft/industry/news/data/2022/04/optimaler-verkehrsfluss-mit-intelligenten-ampeln.html",
+    "https://www.spektrum.de/news/die-verschlungenen-wege-der-menschen/1181815",
+    "https://www.pcwelt.de/a/diktatur-4-0-schoene-neue-digitalisierte-welt,3447005",
+    "https://www.nzz.ch/english/cancel-culture-at-eth-a-professor-receives-death-threats-over-a-lecture-slide-ld.1675322",
+    "https://www.brandeins.de/corporate-services/mck-wissen/mck-wissen-logistik/schwaermen-fuer-das-optimum",
+    "https://www.achgut.com/artikel/ausgestossene_der_woche_prinz_william_als_immaginierter_rassist",
+    "https://www.pinterpolitik.com/in-depth/klaim-big-data-luhut-perlu-diuji/",
+    "https://www.srf.ch/kultur/gesellschaft-religion/eklat-an-der-eth-wenn-ein-angeblicher-schweinevergleich-zur-staatsaffaere-wird",
+    "https://open.spotify.com/episode/6s1icdoplZeNOINvx6ZHTd?si=610a699eba004da2&nd=1",
+    "https://www.nzz.ch/schweiz/shitstorm-an-der-eth-ein-professor-erhaelt-morddrohungen-ld.1673554",
+    "https://www.nzz.ch/schweiz/shitstorm-an-der-eth-ein-professor-erhaelt-morddrohungen-ld.1673554",
+    "https://djmag.com/features/after-astroworld-what-being-done-stop-crowd-crushes-happening-again",
+    "https://prisma-hsg.ch/articles/meine-daten-deine-daten-unsere-daten/",
+    "https://www.srf.ch/audio/focus/zukunftsforscher-dirk-helbing-die-welt-ist-keine-maschine?id=10756661",
+    "https://www.20min.ch/story/roboter-fuer-hunde-machen-wenig-sinn-647302764916",
+    "https://www.wienerzeitung.at/nachrichten/wissen/mensch/942890-Roboter-als-Praesidentschaftskandidaten.html",
+    "https://disruptors.fm/11-building-a-crystal-ball-of-the-world-unseating-capitalism-and-creating-a-new-world-order-with-prof-dirk-helbing/",
+    "https://www.spreaker.com/user/disruptorsfm/11-building-crystal-ball-of-the-world-un",
+    "https://www.youtube.com/watch?v=fRkCMC3zqSQ",
+    "https://arstechnica.com/science/2021/11/what-the-physics-of-crowds-can-tell-us-about-the-tragic-deaths-at-astroworld/",
+    "https://www.fox23.com/news/trending/astroworld-festival-big-crowds-can-flow-like-liquid-with-terrifying-results/37QH6Q4RGFELHGCZSZTBV46STU/",
+    "https://futurism.com/astroworld-theory-deaths-bodies-fluid",
+    "https://www.businessinsider.com/why-people-died-astroworld-crowd-crush-physics-fluid-dynamics-2021-11",
+    "https://theconversation.com/ten-tips-for-surviving-a-crowd-crush-112169",
+    "https://www.limmattalerzeitung.ch/basel/das-wort-zum-tag-kopie-von-4-januar-hypotenuse-schlaegt-kathete-trivia-trampel-pandemie-ld.2233931",
+    "https://magazine.swissinformatics.org/en/whats-wrong-with-ai/",
+    "https://magazine.swissinformatics.org/en/whats-wrong-with-ai/",
+    "https://www.netkwesties.nl/1541/wrr-ai-wordt-de-verbrandingsmotor-van.htm",
+    "https://youtu.be/ptm9zLG2KaE",
+    "https://www.deutschlandfunkkultur.de/die-zukunft-der-demokratie-mehr-teilhabe-von-unten-wagen.976.de.html?dram:article_id=468341",
+    "https://www.springer.com/gp/book/9783642240034",
+    "https://www.springer.com/de/book/9783319908687",
+    "https://technikjournal.de/2017/08/02/ein-plaedoyer-fuer-die-digitale-demokratie/",
+    "https://technikjournal.de/2017/08/02/ein-plaedoyer-fuer-die-digitale-demokratie/",
+    "https://trafo.hypotheses.org/23989",
+    "https://web.archive.org/web/20200609053329/https://www.wiko-berlin.de/institution/projekte-kooperationen/projekte/working-futures/wiko-briefs-working-futures-in-corona-times/the-corona-crisis-reveals-the-struggle-for-a-sustainable-digital-future/",
+    "https://www.wiko-berlin.de/institution/projekte-kooperationen/projekte/working-futures/wiko-briefs-working-futures-in-corona-times/",
+    "https://www.youtube.com/watch?v=gAkoprZmW4k",
+    "https://www.rhein-zeitung.de/region/aus-den-lokalredaktionen/nahe-zeitung_artikel,-peter-flaschels-lebenswerk-hat-die-sozialgeschichte-beeinflusst-_arid,2322161.html",
+    "https://www.blick.ch/wirtschaft/online-boom-ohne-ende-corona-befeuert-die-tech-revolution-id16359910.html",
+    "https://www.nzz.ch/meinung/china-unterwirft-tech-und-social-media-das-geht-auch-europa-an-ld.1643010",
+    "https://www.say.media/article/la-mort-par-algorithme",
+    "https://www.suedostschweiz.ch/aus-dem-leben/2021-08-14/stau-ist-nicht-gleich-stau",
+    "https://www.swissinfo.ch/eng/directdemocracy/political-perspectives_digital-democracy--too-risky--or-the-chance-of-a-generation-/43836222",
+    "https://kow-berlin.com/exhibitions/illusion-einer-menschenmenge",
+    "https://www.springer.com/gp/book/9783642240034",
+    "https://www.springer.com/de/book/9783319908687",
+    "https://www.politik-kommunikation.de/ressorts/artikel/eine-gefaehrliche-machtasymmetrie-1383558602",
+    "https://www.springer.com/gp/book/9783642240034",
+    "https://www.springer.com/de/book/9783319908687",
+    "https://solutions.hamburg/ethik-und-digitalisierung-nicht-voneinander-getrennt-betrachten/",
+    "https://www.springer.com/gp/book/9783642240034",
+    "https://www.springer.com/de/book/9783319908687",
+    "https://avenue.argusdatainsights.ch/Article/AvenueClip?artikelHash=d14d91ec9a8b4cb0b6bb3012c0cefd8b_27F0B19422F1F03723769C18906AA1EE&artikelDateiId=298862327",
+    "https://www.tagblatt.ch/kultur/grosses-ranking-ihre-stimme-hat-gewicht-das-sind-die-50-profiliertesten-intellektuellen-der-schweiz-ld.2182261",
+    "https://reliefweb.int/report/world/building-multisystemic-understanding-societal-resilience-covid-19-pandemic",
+    "https://reliefweb.int/report/world/building-multisystemic-understanding-societal-resilience-covid-19-pandemic",
+    "https://www.events.at/e/wie-wir-in-zukunft-leben-wollen-die-stadt-als-datenfeld",
+    "https://www.events.at/e/wie-wir-in-zukunft-leben-wollen-die-stadt-als-datenfeld",
+    "https://greennetproject.org/en/2018/11/27/prof-dirk-helbing-es-braucht-vor-allem-tolle-ideen-in-die-sich-die-leute-verlieben/",
+    "https://www.hpcwire.com/2011/05/06/simulating_society_at_the_global_scale/",
+    "https://www.technologyreview.com/2010/04/30/204005/europes-plan-to-simulate-the-entire-planet/",
+    "https://komentare.sme.sk/c/22543617/smrt-podla-algoritmu.html",
+    "https://komentare.sme.sk/c/22543617/smrt-podla-algoritmu.html",
+    "https://www.confidencial.com.ni/opinion/muerte-por-algoritmo/",
+    "https://www.nzz.ch/panorama/wie-kann-eine-massenpanik-verhindert-werden-ld.1614761",
+    "https://www.20min.ch/story/roboter-fuer-hunde-machen-wenig-sinn-647302764916",
+    "https://www.wienerzeitung.at/nachrichten/wissen/mensch/942890-Roboter-als-Praesidentschaftskandidaten.html",
+    "https://www.srf.ch/audio/focus/zukunftsforscher-dirk-helbing-die-welt-ist-keine-maschine?id=10756661",
+    "https://disruptors.fm/11-building-a-crystal-ball-of-the-world-unseating-capitalism-and-creating-a-new-world-order-with-prof-dirk-helbing/",
+    "https://www.spreaker.com/user/disruptorsfm/11-building-crystal-ball-of-the-world-un",
+    "https://www.youtube.com/watch?v=fRkCMC3zqSQ",
+    "https://arstechnica.com/science/2021/11/what-the-physics-of-crowds-can-tell-us-about-the-tragic-deaths-at-astroworld/",
+    "https://www.fox23.com/news/trending/astroworld-festival-big-crowds-can-flow-like-liquid-with-terrifying-results/37QH6Q4RGFELHGCZSZTBV46STU/",
+    "https://futurism.com/astroworld-theory-deaths-bodies-fluid",
+    "https://www.businessinsider.com/why-people-died-astroworld-crowd-crush-physics-fluid-dynamics-2021-11",
+    "https://theconversation.com/ten-tips-for-surviving-a-crowd-crush-112169",
+    "https://www.limmattalerzeitung.ch/basel/das-wort-zum-tag-kopie-von-4-januar-hypotenuse-schlaegt-kathete-trivia-trampel-pandemie-ld.2233931",
+    "https://www.pantarhei.ch/podcast/",
+    "https://www.focus.it/scienza/scienze/folla-fisica-modelli-simulazioni",
+    "https://www.focus.it/scienza/scienze/folla-fisica-modelli-simulazioni",
+    "https://www.netkwesties.nl/1541/wrr-ai-wordt-de-verbrandingsmotor-van.htm",
+    "https://www.transformationbeats.com/de/transformation/digitale-gesellschaft/",
+    "https://www.transformationbeats.com/de/transformation/digitale-gesellschaft/",
+    "https://www.suedkurier.de/ueberregional/wirtschaft/Wie-uns-der-Staat-heimlich-erzieht-sogar-auf-dem-Klo;art416,8763904",
+    "https://www.suedkurier.de/ueberregional/wirtschaft/Wie-uns-der-Staat-heimlich-erzieht-sogar-auf-dem-Klo;art416,8763904",
+    "https://www.deutschlandfunkkultur.de/die-zukunft-der-demokratie-mehr-teilhabe-von-unten-wagen.976.de.html?dram:article_id=468341",
+    "https://www.springer.com/gp/book/9783642240034",
+    "https://www.springer.com/de/book/9783319908687",
+    "https://trafo.hypotheses.org/23989",
+    "https://web.archive.org/web/20200609053329/https://www.wiko-berlin.de/institution/projekte-kooperationen/projekte/working-futures/wiko-briefs-working-futures-in-corona-times/the-corona-crisis-reveals-the-struggle-for-a-sustainable-digital-future/",
+    "https://www.wiko-berlin.de/institution/projekte-kooperationen/projekte/working-futures/wiko-briefs-working-futures-in-corona-times/",
+    "https://www.youtube.com/watch?v=gAkoprZmW4k",
+    "https://futurium.de/de/gespraech/ranga-yogeshwar-1/ranga-yogeshwar-dirk-helbing-mit-musik-von-till-broenner",
+    "https://www.springer.com/gp/book/9783642240034",
+    "https://www.springer.com/de/book/9783319908687",
+    "https://idw-online.de/en/news113518",
+    "https://blmplus.de/die-digitalcharta-ist-erst-der-anfang-ein-szenario-von-dirk-helbing/",
+    "https://www.risiko-dialog.ch/big-nudging-vom-computer-gelenkt-aber-wohin/",
+    "https://idw-online.de/de/news13986",
+    "https://www.uni-stuttgart.de/presse/archiv/uni-kurier/uk84_85/forschung/fw66.html",
+    "https://www.infosperber.ch/medien/trends/rankings-oft-unbrauchbar-so-oder-so-aber-immer-schadlich/",
+    "https://www.infosperber.ch/medien/trends/rankings-oft-unbrauchbar-so-oder-so-aber-immer-schadlich/",
+    "https://www.nzz.ch/meinung/china-unterwirft-tech-und-social-media-das-geht-auch-europa-an-ld.1643010",
+    "https://www.suedostschweiz.ch/aus-dem-leben/2021-08-14/stau-ist-nicht-gleich-stau",
+    "https://www.swissinfo.ch/eng/directdemocracy/political-perspectives_digital-democracy--too-risky--or-the-chance-of-a-generation-/43836222",
+    "https://werteundwandel.de/inhalte/d2030-in-aufbruchstimmung-fuer-eine-lebenswerte-zukunft/",
+    "https://www.springer.com/gp/book/9783642240034",
+    "https://www.springer.com/de/book/9783319908687",
+    "https://www.youtube.com/watch?v=n9e77iYZPEY",
+    "https://greennetproject.org/en/2018/11/27/prof-dirk-helbing-es-braucht-vor-allem-tolle-ideen-in-die-sich-die-leute-verlieben/",
+    "https://www.hpcwire.com/2011/05/06/simulating_society_at_the_global_scale/",
+    "https://www.say.media/article/la-mort-par-algorithme",
+    "https://www.confidencial.com.ni/opinion/muerte-por-algoritmo/",
+    "https://www.nzz.ch/panorama/wie-kann-eine-massenpanik-verhindert-werden-ld.1614761",
+    "https://www.nesta.org.uk/report/digital-democracy-the-tools-transforming-political-engagement/",
+    "https://www.nature.com/articles/news.2010.351",
+    "https://www.focus.de/panorama/welt/tid-19265/gastkommentar-nutzt-die-moeglichkeiten-des-computers_aid_534372.html",
+    "https://www.theglobalist.com/democracy-technology-innovation-society-internet/",
+    "https://www.theglobalist.com/capitalism-democracy-technology-surveillance-privacy/",
+    "https://www.theglobalist.com/google-artificial-intelligence-big-data-technology-future/",
+    "https://www.theglobalist.com/fascism-big-data-artificial-intelligence-surveillance-democracy/",
+    "https://www.theglobalist.com/technology-big-data-artificial-intelligence-future-peace-rooms/",
+    "https://www.theglobalist.com/technology-society-sustainability-future-humanity/",
+    "https://www.theglobalist.com/society-technology-peace-sustainability/",
+    "https://www.theglobalist.com/democracy-technology-social-media-artificial-intelligence/",
+    "https://www.theglobalist.com/financial-system-reform-economy-internet-of-things-capitalism/",
+    "https://www.theglobalist.com/capitalism-society-equality-sustainability-crowd-funding/",
+    "https://www.theglobalist.com/united-nations-world-government-peace-sustainability-society/",
+    "https://www.theglobalist.com/world-economy-sustainability-environment-society/"
+]
--- a/requirements.txt
+++ b/requirements.txt
@ -0,0 +1,10 @@
+peewee
+selenium
+# youtube-dl
+git+https://github.com/ytdl-org/youtube-dl.git #non-release version with faster downloads
+waybackpy
+slack_bolt # relies on slack_sdk
+newspaper3k
+htmldate
+markdown
+rich