diff --git a/Dockerfile b/Dockerfile index b3091c8..0ba9459 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,9 +30,10 @@ RUN chown -R autonews:autonews /app USER autonews RUN export PATH=/home/autonews/.local/bin:$PATH +COPY requirements.txt /app/requirements.txt +RUN python3 -m pip install -r /app/requirements.txt COPY app /app/auto_news WORKDIR /app/auto_news -RUN python3 -m pip install -r requirements.txt ENTRYPOINT ["python3", "runner.py"] diff --git a/app/requirements.txt b/app/requirements.txt deleted file mode 100644 index 5347fd9..0000000 --- a/app/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -peewee -selenium -youtube-dl -waybackpy -slack_bolt # relies on slack_sdk -newspaper3k -htmldate -markdown -rich diff --git a/app/runner.py b/app/runner.py index bb0db7e..818b9ad 100644 --- a/app/runner.py +++ b/app/runner.py @@ -31,7 +31,7 @@ class ArticleWatcher: # first step: gather metadata if self.fetch and self.upload: self.fetch.process(self) # this will call the update_status method - self.upload.process(self) # idependdent from the rest + self.upload.process(self) # idependent from the rest else: # the full kwargs were not provided, only do a manual run # overwrite update_status() because calls from the workers will result in erros self.update_status = lambda completed: logger.info(f"Completed action {completed}") diff --git a/app/utils_worker/download/browser.py b/app/utils_worker/download/browser.py index 1b14ab0..0c05f56 100644 --- a/app/utils_worker/download/browser.py +++ b/app/utils_worker/download/browser.py @@ -28,7 +28,7 @@ class PDFDownloader: if os.getenv("HEADLESS", "false") == "true": options.add_argument('--headless') else: - self.logger.warning("Opening browser GUI because of 'HEADLESS=true'") + self.logger.warning("Opening browser GUI because of 'HEADLESS=false'") options.set_preference('print.save_as_pdf.links.enabled', True) # Just save if the filetype is pdf already, does not work! @@ -46,7 +46,7 @@ class PDFDownloader: # log_path = f'{config["local_storage_path"]}/geckodriver.log' # )) self.driver = webdriver.Remote( - command_executor = 'http://localhost:4444', + command_executor = 'http://geckodriver:4444', options = options, # can't set log path... ) @@ -64,13 +64,17 @@ class PDFDownloader: def finish(self): if self.running: self.logger.info("Exiting gecko driver") - self.driver.quit() + try: + self.driver.quit() + time.sleep(10) + except: + self.logger.critical("Connection to the driver broke off") self.running = False else: self.logger.info("Gecko driver not yet running") def download(self, article_object): - sleep_time = 1 + sleep_time = 2 self.autostart() url = article_object.article_url @@ -87,7 +91,7 @@ class PDFDownloader: # in the mean time, get a page title if required if article_object.is_title_bad: article_object.title = self.driver.title.replace(".pdf", "") - # will be propagated to dst as well + # will be propagated to the saved file (dst) as well fname = article_object.fname_template dst = os.path.join(article_object.save_path, fname) @@ -110,7 +114,7 @@ class PDFDownloader: else: article_object.file_name = "" - return article_object # this change is saved later manually + return article_object # this change is saved later by the external caller def get_exisiting_pdf(self, url, dst): diff --git a/app/utils_worker/upload/runner.py b/app/utils_worker/upload/runner.py index 5542d16..5714bce 100644 --- a/app/utils_worker/upload/runner.py +++ b/app/utils_worker/upload/runner.py @@ -1,3 +1,4 @@ +import time from waybackpy import WaybackMachineSaveAPI # upload to archive.org import logging logger = logging.getLogger(__name__) @@ -11,6 +12,8 @@ def upload_to_archive(article_object): archive_url = wayback.save() # logger.info(f"{url} uploaded to archive successfully") article_object.archive_url = archive_url + # time.sleep(4) # Archive Uploads rate limited to 15/minute + except Exception as e: article_object.archive_url = "Error while uploading: {}".format(e) logger.error(f"Error while generating archive url: {e}") diff --git a/app/utils_worker/workers.py b/app/utils_worker/workers.py index 657fae3..8d46707 100644 --- a/app/utils_worker/workers.py +++ b/app/utils_worker/workers.py @@ -43,11 +43,15 @@ class FetchWorker(TemplateWorker): class UploadWorker(TemplateWorker): def __init__(self) -> None: super().__init__() + + def _handle_article(self, article_watcher): - action = run_upload # function + def action(*args, **kwargs): + run_upload(*args, **kwargs) + time.sleep(5) # uploads to archive are throttled to 15/minute + super()._handle_article(article_watcher, action) - time.sleep(4) # Archive Uploads rate limited to 15/minute article_watcher.upload_completed = True diff --git a/docker-compose.yaml b/docker-compose.yaml index 8416d79..5679a3a 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -7,11 +7,9 @@ services: image: auto_news:latest volumes: - ${CONTAINER_DATA}:/app/file_storage - - ${HOSTS_FILE}:/etc/hosts - - ${CODE:-/dev/null}:/code # not set in prod, defaults to /dev/null + - ${XSOCK-/dev/null}:${XSOCK-/tmp/sock} - ${XAUTHORITY-/dev/null}:/home/auto_news/.Xauthority - network_mode: host environment: - DISPLAY=$DISPLAY - TERM=xterm-256color # colored logs @@ -28,10 +26,12 @@ services: geckodriver: image: selenium/standalone-firefox:101.0 volumes: - - ${XAUTHORITY-/dev/null}:/home/seluser/.Xauthority + - ${XSOCK-/dev/null}:${XSOCK-/tmp/sock} + - ${XAUTHORITY-/dev/null}:/home/auto_news/.Xauthority environment: - DISPLAY=$DISPLAY - START_VNC=false - START_XVFB=false user: 1001:1001 - network_mode: host \ No newline at end of file + expose: # exposed to other docker-compose services only + - "4444" \ No newline at end of file diff --git a/env/check b/env/check index 7b2b86e..3a40348 100644 --- a/env/check +++ b/env/check @@ -4,6 +4,7 @@ CONTAINER_DATA=~/Bulk/COSS/Downloads/auto_news.container HOSTS_FILE=~/Bulk/COSS/Downloads/auto_news.container/dependencies/hosts XAUTHORTIY=$XAUTHORTIY +XSOCK=/tmp/.X11-unix DEBUG=false CHECK=true diff --git a/env/debug b/env/debug index 041187c..2c2ab99 100644 --- a/env/debug +++ b/env/debug @@ -5,6 +5,7 @@ HOSTS_FILE=~/Bulk/COSS/Downloads/auto_news.container/dependencies/hosts CODE=./ XAUTHORTIY=$XAUTHORTIY +XSOCK=/tmp/.X11-unix DEBUG=true CHECK=false diff --git a/misc/exctract_from_mail_backup.py b/misc/exctract_from_mail_backup.py index b0dbf07..a00dc63 100644 --- a/misc/exctract_from_mail_backup.py +++ b/misc/exctract_from_mail_backup.py @@ -19,5 +19,5 @@ for f in all_files: print("Saved {} urls".format(len(all_urls))) -with open("mails_export.json", "w") as f: +with open("media_mails_export.json", "w") as f: json.dump(all_urls, f) \ No newline at end of file diff --git a/misc/gather_media_files.py b/misc/gather_media_files.py index 1059b6d..3b376ad 100644 --- a/misc/gather_media_files.py +++ b/misc/gather_media_files.py @@ -1,10 +1,14 @@ import sys +from webbrowser import get sys.path.append("../app") import runner import logging logger = logging.getLogger() import json +from rich.console import Console +from rich.table import Table +console = Console() logger.info("Overwriting production values for single time media-fetch") runner.configuration.models.set_db( @@ -40,13 +44,30 @@ def fetch(): def show(): - sel = runner.models.ArticleDownload.select() - entries = ["title"] #, "article_url", "archive_url"] + + t = Table( + title = "ArticleDownloads", + row_styles = ["white", "bright_black"], + ) + + entries = ["title", "article_url", "archive_url", "authors"] for e in entries: - r = [t.title for t in sel] - print(r) - # print([t for t in r]) + t.add_column(e, justify = "right") + + sel = runner.models.ArticleDownload.select() + + for s in sel: + c = [getattr(s, e) for e in entries]# + c[-1] = str([a.author for a in c[-1]]) + print(c) + t.add_row(*c) + + + console.print(t) -show() + + +# fetch() +show() \ No newline at end of file diff --git a/misc/media_urls.json b/misc/media_urls.json index 584cf21..158c9df 100644 --- a/misc/media_urls.json +++ b/misc/media_urls.json @@ -1 +1,151 @@ -["https://www.nesta.org.uk/report/digital-democracy-the-tools-transforming-political-engagement/", "https://media.nesta.org.uk/documents/digital_democracy.pdf", "https://context-cdn.washingtonpost.com/notes/prod/default/documents/c3c41863-be9e-4246-9ed9-e43aedd013f9/note/4e677597-f403-4c9b-b838-f5613d79b341", "https://context-cdn.washingtonpost.com/notes/prod/default/documents/6d274110-a84b-4694-96cd-6a902207d2bd/note/733364cf-0afb-412d-a5b4-ab797a8ba154.#page=1", "https://www.judiciary.senate.gov/fisa-investigation", "https://www.state.gov/fact-sheet-activity-at-the-wuhan-institute-of-virology/", "https://www.whitehouse.gov/trump-administration-accomplishments/", "https://www.whitehouse.gov/wp-content/uploads/2021/01/IPS-Final-Declass.pdf", "https://www.finance.senate.gov/imo/media/doc/Oversight,%2012-23-20,%20Memo%20on%20World%20Vision%20Investigation.pdf", "https://justthenews.com/sites/default/files/2020-12/BidenArcher4-13-14.pdf", "https://www.hsgac.senate.gov/imo/media/doc/Johnson-Grassley%20Submission%202020-12-09.pdf", "https://navarroreport.com/", "https://got-freedom.org/wp-content/uploads/2020/12/HAVA-and-Non-Profit-Organization-Report-FINAL-W-Attachments-and-Preface-121420.pdf", "https://www.depernolaw.com/uploads/2/7/0/2/27029178/antrim_michigan_forensics_report_%5B121320%5D_v2_%5Bredacted%5D.pdf", "https://www.hsgac.senate.gov/imo/media/doc/HSGAC_Finance_Report_FINAL.pdf", "https://www.scribd.com/document/487040771/Emails-About-FBI-Receipt-Fusion-GPS-Thumb-Drive", "https://cdn.epoch.cloud/assets/static_assets/Voter-Fraud-Allegations-Infographic-Epoch-Times.jpg", "https://www.hsgac.senate.gov/imo/media/doc/Lync%20and%20text%20messages%20between%20and%20among%20DOJ%20and%20FBI%20employees.pdf", "https://www.hsgac.senate.gov/imo/media/doc/DOJ%20Docs%20Combined.pdf", "https://www.hsgac.senate.gov/imo/media/doc/FBI%20Productions%20Combined%20-%20updated_FINAL.pdf", "https://www.hsgac.senate.gov/imo/media/doc/STATE_combined.pdf", "https://cdn.epoch.cloud/assets/static_assets/epochtimes-infographic-war-on-president-trump.jpg", "https://centipedenation.com/transmissions/miles-guo-dropping-bombs-hunter-biden-sex-tapes-and-other-evidence-of-the-ccps-infiltration-of-the-u-s/", "https://www.finance.senate.gov/imo/media/doc/2020-11-18%20HSGAC%20-%20Finance%20Joint%20Report%20Supplemental.pdf", "https://www.scribd.com/document/479781400/Steele-Spreadsheet-1", "https://www.zerohedge.com/political/jim-comey-ignored-state-department-whistleblower-hillarys-crimes-classified-material", "https://www.judicialwatch.org/wp-content/uploads/2020/10/JW-v-State-Steele-Oct-2020-prod-00968.pdf", "https://justthenews.com/sites/default/files/2020-10/requested%20email.pdf", "https://www.tagblatt.ch/kultur/sommertipps-20-buchempfehlungen-fuer-prominente-wir-haben-die-besten-buecher-fuer-jeden-charakter-zusammengetragen-ld.2159339", "https://www.tagblatt.ch/kultur/sommertipps-20-buchempfehlungen-fuer-prominente-wir-haben-die-besten-buecher-fuer-jeden-charakter-zusammengetragen-ld.2159339", "https://greennetproject.org/en/2018/11/27/prof-dirk-helbing-es-braucht-vor-allem-tolle-ideen-in-die-sich-die-leute-verlieben/", "https://www.nature.com/articles/news.2010.351", "https://www.focus.de/panorama/welt/tid-19265/gastkommentar-nutzt-die-moeglichkeiten-des-computers_aid_534372.html", "http://www.ccss.ethz.ch/Response/index.html", "https://www.hpcwire.com/2011/05/06/simulating_society_at_the_global_scale/", "https://www.say.media/article/la-mort-par-algorithme", "https://www.say.media/article/la-mort-par-algorithme", "https://www.nzz.ch/panorama/wie-kann-eine-massenpanik-verhindert-werden-ld.1614761", "https://www.theglobalist.com/democracy-technology-innovation-society-internet/","https://www.theglobalist.com/capitalism-democracy-technology-surveillance-privacy/","https://www.theglobalist.com/google-artificial-intelligence-big-data-technology-future/","https://www.theglobalist.com/fascism-big-data-artificial-intelligence-surveillance-democracy/","https://www.theglobalist.com/technology-big-data-artificial-intelligence-future-peace-rooms/","https://www.theglobalist.com/technology-society-sustainability-future-humanity/","https://www.theglobalist.com/society-technology-peace-sustainability/","https://www.theglobalist.com/democracy-technology-social-media-artificial-intelligence/","https://www.theglobalist.com/financial-system-reform-economy-internet-of-things-capitalism/","https://www.theglobalist.com/capitalism-society-equality-sustainability-crowd-funding/","https://www.theglobalist.com/united-nations-world-government-peace-sustainability-society/","https://www.theglobalist.com/world-economy-sustainability-environment-society/"] +[ + "https://www.swissinfo.ch/ger/wirtschaft/koennen-ki-und-direkte-demokratie-nebeneinander-bestehen-/47542048", + "https://www.zeit.de/2011/33/CH-Oekonophysik", + "https://ourworld.unu.edu/en/green-idea-self-organizing-traffic-signals", + "https://www.youtube.com/watch?v=-FQD4ie9UYA", + "https://www.brandeins.de/corporate-services/mck-wissen/mck-wissen-logistik/schwaermen-fuer-das-optimum", + "https://www.youtube.com/watch?v=upQM4Xzh8zM", + "https://www.youtube.com/watch?v=gAkoprZmW4k", + "https://www.youtube.com/watch?v=VMzfDVAWXHI&t=1s", + "https://www.youtube.com/watch?v=1SwTiIlkndE", + "https://www.informatik-aktuell.de/management-und-recht/digitalisierung/digitale-revolution-und-oekonomie-40-quo-vadis.html", + "https://www.youtube.com/watch?v=cSvvH0SBFOw", + "https://www.linkedin.com/posts/margit-osterloh-24198a104_pl%C3%A4doyer-gegen-sprechverbote-ugcPost-6925702100450480129-K7Dl?utm_source=linkedin_share&utm_medium=member_desktop_web", + "https://www.nebelspalter.ch/plaedoyer-gegen-sprechverbote", + "https://falling-walls.com/people/dirk-helbing/", + "https://digitalsensemaker.podigee.io/3-2-mit-dirk-helbing", + "https://www.blick.ch/wirtschaft/musk-als-hueter-der-redefreiheit-eth-experte-sagt-musks-vorhaben-hat-potenzial-aber-id17437811.html", + "https://www.trend.at/standpunkte/mit-verantwortung-zukunft-10082300", + "https://www.pantarhei.ch/podcast/", + "https://ethz.ch/en/industry/industry/news/data/2022/04/intelligent-traffic-lights-for-optimal-traffic-flow.html", + "https://ethz.ch/de/wirtschaft/industry/news/data/2022/04/optimaler-verkehrsfluss-mit-intelligenten-ampeln.html", + "https://www.spektrum.de/news/die-verschlungenen-wege-der-menschen/1181815", + "https://www.pcwelt.de/a/diktatur-4-0-schoene-neue-digitalisierte-welt,3447005", + "https://www.nzz.ch/english/cancel-culture-at-eth-a-professor-receives-death-threats-over-a-lecture-slide-ld.1675322", + "https://www.brandeins.de/corporate-services/mck-wissen/mck-wissen-logistik/schwaermen-fuer-das-optimum", + "https://www.achgut.com/artikel/ausgestossene_der_woche_prinz_william_als_immaginierter_rassist", + "https://www.pinterpolitik.com/in-depth/klaim-big-data-luhut-perlu-diuji/", + "https://www.srf.ch/kultur/gesellschaft-religion/eklat-an-der-eth-wenn-ein-angeblicher-schweinevergleich-zur-staatsaffaere-wird", + "https://open.spotify.com/episode/6s1icdoplZeNOINvx6ZHTd?si=610a699eba004da2&nd=1", + "https://www.nzz.ch/schweiz/shitstorm-an-der-eth-ein-professor-erhaelt-morddrohungen-ld.1673554", + "https://www.nzz.ch/schweiz/shitstorm-an-der-eth-ein-professor-erhaelt-morddrohungen-ld.1673554", + "https://djmag.com/features/after-astroworld-what-being-done-stop-crowd-crushes-happening-again", + "https://prisma-hsg.ch/articles/meine-daten-deine-daten-unsere-daten/", + "https://www.srf.ch/audio/focus/zukunftsforscher-dirk-helbing-die-welt-ist-keine-maschine?id=10756661", + "https://www.20min.ch/story/roboter-fuer-hunde-machen-wenig-sinn-647302764916", + "https://www.wienerzeitung.at/nachrichten/wissen/mensch/942890-Roboter-als-Praesidentschaftskandidaten.html", + "https://disruptors.fm/11-building-a-crystal-ball-of-the-world-unseating-capitalism-and-creating-a-new-world-order-with-prof-dirk-helbing/", + "https://www.spreaker.com/user/disruptorsfm/11-building-crystal-ball-of-the-world-un", + "https://www.youtube.com/watch?v=fRkCMC3zqSQ", + "https://arstechnica.com/science/2021/11/what-the-physics-of-crowds-can-tell-us-about-the-tragic-deaths-at-astroworld/", + "https://www.fox23.com/news/trending/astroworld-festival-big-crowds-can-flow-like-liquid-with-terrifying-results/37QH6Q4RGFELHGCZSZTBV46STU/", + "https://futurism.com/astroworld-theory-deaths-bodies-fluid", + "https://www.businessinsider.com/why-people-died-astroworld-crowd-crush-physics-fluid-dynamics-2021-11", + "https://theconversation.com/ten-tips-for-surviving-a-crowd-crush-112169", + "https://www.limmattalerzeitung.ch/basel/das-wort-zum-tag-kopie-von-4-januar-hypotenuse-schlaegt-kathete-trivia-trampel-pandemie-ld.2233931", + "https://magazine.swissinformatics.org/en/whats-wrong-with-ai/", + "https://magazine.swissinformatics.org/en/whats-wrong-with-ai/", + "https://www.netkwesties.nl/1541/wrr-ai-wordt-de-verbrandingsmotor-van.htm", + "https://youtu.be/ptm9zLG2KaE", + "https://www.deutschlandfunkkultur.de/die-zukunft-der-demokratie-mehr-teilhabe-von-unten-wagen.976.de.html?dram:article_id=468341", + "https://www.springer.com/gp/book/9783642240034", + "https://www.springer.com/de/book/9783319908687", + "https://technikjournal.de/2017/08/02/ein-plaedoyer-fuer-die-digitale-demokratie/", + "https://technikjournal.de/2017/08/02/ein-plaedoyer-fuer-die-digitale-demokratie/", + "https://trafo.hypotheses.org/23989", + "https://web.archive.org/web/20200609053329/https://www.wiko-berlin.de/institution/projekte-kooperationen/projekte/working-futures/wiko-briefs-working-futures-in-corona-times/the-corona-crisis-reveals-the-struggle-for-a-sustainable-digital-future/", + "https://www.wiko-berlin.de/institution/projekte-kooperationen/projekte/working-futures/wiko-briefs-working-futures-in-corona-times/", + "https://www.youtube.com/watch?v=gAkoprZmW4k", + "https://www.rhein-zeitung.de/region/aus-den-lokalredaktionen/nahe-zeitung_artikel,-peter-flaschels-lebenswerk-hat-die-sozialgeschichte-beeinflusst-_arid,2322161.html", + "https://www.blick.ch/wirtschaft/online-boom-ohne-ende-corona-befeuert-die-tech-revolution-id16359910.html", + "https://www.nzz.ch/meinung/china-unterwirft-tech-und-social-media-das-geht-auch-europa-an-ld.1643010", + "https://www.say.media/article/la-mort-par-algorithme", + "https://www.suedostschweiz.ch/aus-dem-leben/2021-08-14/stau-ist-nicht-gleich-stau", + "https://www.swissinfo.ch/eng/directdemocracy/political-perspectives_digital-democracy--too-risky--or-the-chance-of-a-generation-/43836222", + "https://kow-berlin.com/exhibitions/illusion-einer-menschenmenge", + "https://www.springer.com/gp/book/9783642240034", + "https://www.springer.com/de/book/9783319908687", + "https://www.politik-kommunikation.de/ressorts/artikel/eine-gefaehrliche-machtasymmetrie-1383558602", + "https://www.springer.com/gp/book/9783642240034", + "https://www.springer.com/de/book/9783319908687", + "https://solutions.hamburg/ethik-und-digitalisierung-nicht-voneinander-getrennt-betrachten/", + "https://www.springer.com/gp/book/9783642240034", + "https://www.springer.com/de/book/9783319908687", + "https://avenue.argusdatainsights.ch/Article/AvenueClip?artikelHash=d14d91ec9a8b4cb0b6bb3012c0cefd8b_27F0B19422F1F03723769C18906AA1EE&artikelDateiId=298862327", + "https://www.tagblatt.ch/kultur/grosses-ranking-ihre-stimme-hat-gewicht-das-sind-die-50-profiliertesten-intellektuellen-der-schweiz-ld.2182261", + "https://reliefweb.int/report/world/building-multisystemic-understanding-societal-resilience-covid-19-pandemic", + "https://reliefweb.int/report/world/building-multisystemic-understanding-societal-resilience-covid-19-pandemic", + "https://www.events.at/e/wie-wir-in-zukunft-leben-wollen-die-stadt-als-datenfeld", + "https://www.events.at/e/wie-wir-in-zukunft-leben-wollen-die-stadt-als-datenfeld", + "https://greennetproject.org/en/2018/11/27/prof-dirk-helbing-es-braucht-vor-allem-tolle-ideen-in-die-sich-die-leute-verlieben/", + "https://www.hpcwire.com/2011/05/06/simulating_society_at_the_global_scale/", + "https://www.technologyreview.com/2010/04/30/204005/europes-plan-to-simulate-the-entire-planet/", + "https://komentare.sme.sk/c/22543617/smrt-podla-algoritmu.html", + "https://komentare.sme.sk/c/22543617/smrt-podla-algoritmu.html", + "https://www.confidencial.com.ni/opinion/muerte-por-algoritmo/", + "https://www.nzz.ch/panorama/wie-kann-eine-massenpanik-verhindert-werden-ld.1614761", + "https://www.20min.ch/story/roboter-fuer-hunde-machen-wenig-sinn-647302764916", + "https://www.wienerzeitung.at/nachrichten/wissen/mensch/942890-Roboter-als-Praesidentschaftskandidaten.html", + "https://www.srf.ch/audio/focus/zukunftsforscher-dirk-helbing-die-welt-ist-keine-maschine?id=10756661", + "https://disruptors.fm/11-building-a-crystal-ball-of-the-world-unseating-capitalism-and-creating-a-new-world-order-with-prof-dirk-helbing/", + "https://www.spreaker.com/user/disruptorsfm/11-building-crystal-ball-of-the-world-un", + "https://www.youtube.com/watch?v=fRkCMC3zqSQ", + "https://arstechnica.com/science/2021/11/what-the-physics-of-crowds-can-tell-us-about-the-tragic-deaths-at-astroworld/", + "https://www.fox23.com/news/trending/astroworld-festival-big-crowds-can-flow-like-liquid-with-terrifying-results/37QH6Q4RGFELHGCZSZTBV46STU/", + "https://futurism.com/astroworld-theory-deaths-bodies-fluid", + "https://www.businessinsider.com/why-people-died-astroworld-crowd-crush-physics-fluid-dynamics-2021-11", + "https://theconversation.com/ten-tips-for-surviving-a-crowd-crush-112169", + "https://www.limmattalerzeitung.ch/basel/das-wort-zum-tag-kopie-von-4-januar-hypotenuse-schlaegt-kathete-trivia-trampel-pandemie-ld.2233931", + "https://www.pantarhei.ch/podcast/", + "https://www.focus.it/scienza/scienze/folla-fisica-modelli-simulazioni", + "https://www.focus.it/scienza/scienze/folla-fisica-modelli-simulazioni", + "https://www.netkwesties.nl/1541/wrr-ai-wordt-de-verbrandingsmotor-van.htm", + "https://www.transformationbeats.com/de/transformation/digitale-gesellschaft/", + "https://www.transformationbeats.com/de/transformation/digitale-gesellschaft/", + "https://www.suedkurier.de/ueberregional/wirtschaft/Wie-uns-der-Staat-heimlich-erzieht-sogar-auf-dem-Klo;art416,8763904", + "https://www.suedkurier.de/ueberregional/wirtschaft/Wie-uns-der-Staat-heimlich-erzieht-sogar-auf-dem-Klo;art416,8763904", + "https://www.deutschlandfunkkultur.de/die-zukunft-der-demokratie-mehr-teilhabe-von-unten-wagen.976.de.html?dram:article_id=468341", + "https://www.springer.com/gp/book/9783642240034", + "https://www.springer.com/de/book/9783319908687", + "https://trafo.hypotheses.org/23989", + "https://web.archive.org/web/20200609053329/https://www.wiko-berlin.de/institution/projekte-kooperationen/projekte/working-futures/wiko-briefs-working-futures-in-corona-times/the-corona-crisis-reveals-the-struggle-for-a-sustainable-digital-future/", + "https://www.wiko-berlin.de/institution/projekte-kooperationen/projekte/working-futures/wiko-briefs-working-futures-in-corona-times/", + "https://www.youtube.com/watch?v=gAkoprZmW4k", + "https://futurium.de/de/gespraech/ranga-yogeshwar-1/ranga-yogeshwar-dirk-helbing-mit-musik-von-till-broenner", + "https://www.springer.com/gp/book/9783642240034", + "https://www.springer.com/de/book/9783319908687", + "https://idw-online.de/en/news113518", + "https://blmplus.de/die-digitalcharta-ist-erst-der-anfang-ein-szenario-von-dirk-helbing/", + "https://www.risiko-dialog.ch/big-nudging-vom-computer-gelenkt-aber-wohin/", + "https://idw-online.de/de/news13986", + "https://www.uni-stuttgart.de/presse/archiv/uni-kurier/uk84_85/forschung/fw66.html", + "https://www.infosperber.ch/medien/trends/rankings-oft-unbrauchbar-so-oder-so-aber-immer-schadlich/", + "https://www.infosperber.ch/medien/trends/rankings-oft-unbrauchbar-so-oder-so-aber-immer-schadlich/", + "https://www.nzz.ch/meinung/china-unterwirft-tech-und-social-media-das-geht-auch-europa-an-ld.1643010", + "https://www.suedostschweiz.ch/aus-dem-leben/2021-08-14/stau-ist-nicht-gleich-stau", + "https://www.swissinfo.ch/eng/directdemocracy/political-perspectives_digital-democracy--too-risky--or-the-chance-of-a-generation-/43836222", + "https://werteundwandel.de/inhalte/d2030-in-aufbruchstimmung-fuer-eine-lebenswerte-zukunft/", + "https://www.springer.com/gp/book/9783642240034", + "https://www.springer.com/de/book/9783319908687", + "https://www.youtube.com/watch?v=n9e77iYZPEY", + "https://greennetproject.org/en/2018/11/27/prof-dirk-helbing-es-braucht-vor-allem-tolle-ideen-in-die-sich-die-leute-verlieben/", + "https://www.hpcwire.com/2011/05/06/simulating_society_at_the_global_scale/", + "https://www.say.media/article/la-mort-par-algorithme", + "https://www.confidencial.com.ni/opinion/muerte-por-algoritmo/", + "https://www.nzz.ch/panorama/wie-kann-eine-massenpanik-verhindert-werden-ld.1614761", + "https://www.nesta.org.uk/report/digital-democracy-the-tools-transforming-political-engagement/", + "https://www.nature.com/articles/news.2010.351", + "https://www.focus.de/panorama/welt/tid-19265/gastkommentar-nutzt-die-moeglichkeiten-des-computers_aid_534372.html", + "https://www.theglobalist.com/democracy-technology-innovation-society-internet/", + "https://www.theglobalist.com/capitalism-democracy-technology-surveillance-privacy/", + "https://www.theglobalist.com/google-artificial-intelligence-big-data-technology-future/", + "https://www.theglobalist.com/fascism-big-data-artificial-intelligence-surveillance-democracy/", + "https://www.theglobalist.com/technology-big-data-artificial-intelligence-future-peace-rooms/", + "https://www.theglobalist.com/technology-society-sustainability-future-humanity/", + "https://www.theglobalist.com/society-technology-peace-sustainability/", + "https://www.theglobalist.com/democracy-technology-social-media-artificial-intelligence/", + "https://www.theglobalist.com/financial-system-reform-economy-internet-of-things-capitalism/", + "https://www.theglobalist.com/capitalism-society-equality-sustainability-crowd-funding/", + "https://www.theglobalist.com/united-nations-world-government-peace-sustainability-society/", + "https://www.theglobalist.com/world-economy-sustainability-environment-society/" +] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..99fd694 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +peewee +selenium +# youtube-dl +git+https://github.com/ytdl-org/youtube-dl.git #non-release version with faster downloads +waybackpy +slack_bolt # relies on slack_sdk +newspaper3k +htmldate +markdown +rich