remoll/coss_archiving
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

minor corrections

Browse Source
This commit is contained in:
Remy Moll
2022-06-23 15:05:59 +02:00
parent ac9e988af3
commit 12a7de91ed
13 changed files with 218 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
Dockerfile
View File

@@ -30,9 +30,10 @@ RUN chown -R autonews:autonews /app
USER autonews USER autonews
RUN export PATH=/home/autonews/.local/bin:$PATH RUN export PATH=/home/autonews/.local/bin:$PATH
COPY requirements.txt /app/requirements.txt
RUN python3 -m pip install -r /app/requirements.txt
COPY app /app/auto_news COPY app /app/auto_news
WORKDIR /app/auto_news WORKDIR /app/auto_news
RUN python3 -m pip install -r requirements.txt
ENTRYPOINT ["python3", "runner.py"] ENTRYPOINT ["python3", "runner.py"]

9
app/requirements.txt
View File

@@ -1,9 +0,0 @@
peewee
selenium
youtube-dl
waybackpy
slack_bolt # relies on slack_sdk
newspaper3k
htmldate
markdown
rich

2
app/runner.py
View File

@@ -31,7 +31,7 @@ class ArticleWatcher:
# first step: gather metadata # first step: gather metadata
if self.fetch and self.upload: if self.fetch and self.upload:
self.fetch.process(self) # this will call the update_status method self.fetch.process(self) # this will call the update_status method
self.upload.process(self) # idependdent from the rest self.upload.process(self) # idependent from the rest
else: # the full kwargs were not provided, only do a manual run else: # the full kwargs were not provided, only do a manual run
# overwrite update_status() because calls from the workers will result in erros # overwrite update_status() because calls from the workers will result in erros
self.update_status = lambda completed: logger.info(f"Completed action {completed}") self.update_status = lambda completed: logger.info(f"Completed action {completed}")

14
app/utils_worker/download/browser.py
View File

@@ -28,7 +28,7 @@ class PDFDownloader:
if os.getenv("HEADLESS", "false") == "true": if os.getenv("HEADLESS", "false") == "true":
options.add_argument('--headless') options.add_argument('--headless')
else: else:
self.logger.warning("Opening browser GUI because of 'HEADLESS=true'") self.logger.warning("Opening browser GUI because of 'HEADLESS=false'")
options.set_preference('print.save_as_pdf.links.enabled', True) options.set_preference('print.save_as_pdf.links.enabled', True)
# Just save if the filetype is pdf already, does not work! # Just save if the filetype is pdf already, does not work!
@@ -46,7 +46,7 @@ class PDFDownloader:
# log_path = f'{config["local_storage_path"]}/geckodriver.log' # log_path = f'{config["local_storage_path"]}/geckodriver.log'
# )) # ))
self.driver = webdriver.Remote( self.driver = webdriver.Remote(
command_executor = 'http://localhost:4444', command_executor = 'http://geckodriver:4444',
options = options, options = options,
# can't set log path... # can't set log path...
) )
@@ -64,13 +64,17 @@ class PDFDownloader:
def finish(self): def finish(self):
if self.running: if self.running:
self.logger.info("Exiting gecko driver") self.logger.info("Exiting gecko driver")
try:
self.driver.quit() self.driver.quit()
time.sleep(10)
except:
self.logger.critical("Connection to the driver broke off")
self.running = False self.running = False
else: else:
self.logger.info("Gecko driver not yet running") self.logger.info("Gecko driver not yet running")
def download(self, article_object): def download(self, article_object):
sleep_time = 1 sleep_time = 2
self.autostart() self.autostart()
url = article_object.article_url url = article_object.article_url
@@ -87,7 +91,7 @@ class PDFDownloader:
# in the mean time, get a page title if required # in the mean time, get a page title if required
if article_object.is_title_bad: if article_object.is_title_bad:
article_object.title = self.driver.title.replace(".pdf", "") article_object.title = self.driver.title.replace(".pdf", "")
# will be propagated to dst as well # will be propagated to the saved file (dst) as well
fname = article_object.fname_template fname = article_object.fname_template
dst = os.path.join(article_object.save_path, fname) dst = os.path.join(article_object.save_path, fname)
@@ -110,7 +114,7 @@ class PDFDownloader:
else: else:
article_object.file_name = "" article_object.file_name = ""
return article_object # this change is saved later manually return article_object # this change is saved later by the external caller
def get_exisiting_pdf(self, url, dst): def get_exisiting_pdf(self, url, dst):

3
app/utils_worker/upload/runner.py
View File

@@ -1,3 +1,4 @@
import time
from waybackpy import WaybackMachineSaveAPI # upload to archive.org from waybackpy import WaybackMachineSaveAPI # upload to archive.org
import logging import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -11,6 +12,8 @@ def upload_to_archive(article_object):
archive_url = wayback.save() archive_url = wayback.save()
# logger.info(f"{url} uploaded to archive successfully") # logger.info(f"{url} uploaded to archive successfully")
article_object.archive_url = archive_url article_object.archive_url = archive_url
# time.sleep(4) # Archive Uploads rate limited to 15/minute
except Exception as e: except Exception as e:
article_object.archive_url = "Error while uploading: {}".format(e) article_object.archive_url = "Error while uploading: {}".format(e)
logger.error(f"Error while generating archive url: {e}") logger.error(f"Error while generating archive url: {e}")

8
app/utils_worker/workers.py
View File

@@ -44,10 +44,14 @@ class UploadWorker(TemplateWorker):
def __init__(self) -> None: def __init__(self) -> None:
super().__init__() super().__init__()
def _handle_article(self, article_watcher): def _handle_article(self, article_watcher):
action = run_upload # function def action(*args, **kwargs):
run_upload(*args, **kwargs)
time.sleep(5) # uploads to archive are throttled to 15/minute
super()._handle_article(article_watcher, action) super()._handle_article(article_watcher, action)
time.sleep(4) # Archive Uploads rate limited to 15/minute
article_watcher.upload_completed = True article_watcher.upload_completed = True

10
docker-compose.yaml
View File

@@ -7,11 +7,9 @@ services:
image: auto_news:latest image: auto_news:latest
volumes: volumes:
- ${CONTAINER_DATA}:/app/file_storage - ${CONTAINER_DATA}:/app/file_storage
- ${HOSTS_FILE}:/etc/hosts
- ${CODE:-/dev/null}:/code # not set in prod, defaults to /dev/null - ${CODE:-/dev/null}:/code # not set in prod, defaults to /dev/null
- ${XSOCK-/dev/null}:${XSOCK-/tmp/sock}
- ${XAUTHORITY-/dev/null}:/home/auto_news/.Xauthority - ${XAUTHORITY-/dev/null}:/home/auto_news/.Xauthority
network_mode: host
environment: environment:
- DISPLAY=$DISPLAY - DISPLAY=$DISPLAY
- TERM=xterm-256color # colored logs - TERM=xterm-256color # colored logs
@@ -28,10 +26,12 @@ services:
geckodriver: geckodriver:
image: selenium/standalone-firefox:101.0 image: selenium/standalone-firefox:101.0
volumes: volumes:
- ${XAUTHORITY-/dev/null}:/home/seluser/.Xauthority - ${XSOCK-/dev/null}:${XSOCK-/tmp/sock}
- ${XAUTHORITY-/dev/null}:/home/auto_news/.Xauthority
environment: environment:
- DISPLAY=$DISPLAY - DISPLAY=$DISPLAY
- START_VNC=false - START_VNC=false
- START_XVFB=false - START_XVFB=false
user: 1001:1001 user: 1001:1001
network_mode: host expose: # exposed to other docker-compose services only
- "4444"

1
env/check vendored
View File

@@ -4,6 +4,7 @@ CONTAINER_DATA=~/Bulk/COSS/Downloads/auto_news.container
HOSTS_FILE=~/Bulk/COSS/Downloads/auto_news.container/dependencies/hosts HOSTS_FILE=~/Bulk/COSS/Downloads/auto_news.container/dependencies/hosts
XAUTHORTIY=$XAUTHORTIY XAUTHORTIY=$XAUTHORTIY
XSOCK=/tmp/.X11-unix
DEBUG=false DEBUG=false
CHECK=true CHECK=true

1
env/debug vendored
View File

@@ -5,6 +5,7 @@ HOSTS_FILE=~/Bulk/COSS/Downloads/auto_news.container/dependencies/hosts
CODE=./ CODE=./
XAUTHORTIY=$XAUTHORTIY XAUTHORTIY=$XAUTHORTIY
XSOCK=/tmp/.X11-unix
DEBUG=true DEBUG=true
CHECK=false CHECK=false

2
misc/exctract_from_mail_backup.py
View File

@@ -19,5 +19,5 @@ for f in all_files:
print("Saved {} urls".format(len(all_urls))) print("Saved {} urls".format(len(all_urls)))
with open("mails_export.json", "w") as f: with open("media_mails_export.json", "w") as f:
json.dump(all_urls, f) json.dump(all_urls, f)

31
misc/gather_media_files.py
View File

@@ -1,10 +1,14 @@
import sys import sys
from webbrowser import get
sys.path.append("../app") sys.path.append("../app")
import runner import runner
import logging import logging
logger = logging.getLogger() logger = logging.getLogger()
import json import json
from rich.console import Console
from rich.table import Table
console = Console()
logger.info("Overwriting production values for single time media-fetch") logger.info("Overwriting production values for single time media-fetch")
runner.configuration.models.set_db( runner.configuration.models.set_db(
@@ -40,13 +44,30 @@ def fetch():
def show(): def show():
sel = runner.models.ArticleDownload.select()
entries = ["title"] #, "article_url", "archive_url"] t = Table(
title = "ArticleDownloads",
row_styles = ["white", "bright_black"],
)
entries = ["title", "article_url", "archive_url", "authors"]
for e in entries: for e in entries:
r = [t.title for t in sel] t.add_column(e, justify = "right")
print(r)
# print([t for t in r]) sel = runner.models.ArticleDownload.select()
for s in sel:
c = [getattr(s, e) for e in entries]#
c[-1] = str([a.author for a in c[-1]])
print(c)
t.add_row(*c)
console.print(t)
# fetch()
show() show()

152
misc/media_urls.json
View File

@@ -1 +1,151 @@
["https://www.nesta.org.uk/report/digital-democracy-the-tools-transforming-political-engagement/", "https://media.nesta.org.uk/documents/digital_democracy.pdf", "https://context-cdn.washingtonpost.com/notes/prod/default/documents/c3c41863-be9e-4246-9ed9-e43aedd013f9/note/4e677597-f403-4c9b-b838-f5613d79b341", "https://context-cdn.washingtonpost.com/notes/prod/default/documents/6d274110-a84b-4694-96cd-6a902207d2bd/note/733364cf-0afb-412d-a5b4-ab797a8ba154.#page=1", "https://www.judiciary.senate.gov/fisa-investigation", "https://www.state.gov/fact-sheet-activity-at-the-wuhan-institute-of-virology/", "https://www.whitehouse.gov/trump-administration-accomplishments/", "https://www.whitehouse.gov/wp-content/uploads/2021/01/IPS-Final-Declass.pdf", "https://www.finance.senate.gov/imo/media/doc/Oversight,%2012-23-20,%20Memo%20on%20World%20Vision%20Investigation.pdf", "https://justthenews.com/sites/default/files/2020-12/BidenArcher4-13-14.pdf", "https://www.hsgac.senate.gov/imo/media/doc/Johnson-Grassley%20Submission%202020-12-09.pdf", "https://navarroreport.com/", "https://got-freedom.org/wp-content/uploads/2020/12/HAVA-and-Non-Profit-Organization-Report-FINAL-W-Attachments-and-Preface-121420.pdf", "https://www.depernolaw.com/uploads/2/7/0/2/27029178/antrim_michigan_forensics_report_%5B121320%5D_v2_%5Bredacted%5D.pdf", "https://www.hsgac.senate.gov/imo/media/doc/HSGAC_Finance_Report_FINAL.pdf", "https://www.scribd.com/document/487040771/Emails-About-FBI-Receipt-Fusion-GPS-Thumb-Drive", "https://cdn.epoch.cloud/assets/static_assets/Voter-Fraud-Allegations-Infographic-Epoch-Times.jpg", "https://www.hsgac.senate.gov/imo/media/doc/Lync%20and%20text%20messages%20between%20and%20among%20DOJ%20and%20FBI%20employees.pdf", "https://www.hsgac.senate.gov/imo/media/doc/DOJ%20Docs%20Combined.pdf", "https://www.hsgac.senate.gov/imo/media/doc/FBI%20Productions%20Combined%20-%20updated_FINAL.pdf", "https://www.hsgac.senate.gov/imo/media/doc/STATE_combined.pdf", "https://cdn.epoch.cloud/assets/static_assets/epochtimes-infographic-war-on-president-trump.jpg", "https://centipedenation.com/transmissions/miles-guo-dropping-bombs-hunter-biden-sex-tapes-and-other-evidence-of-the-ccps-infiltration-of-the-u-s/", "https://www.finance.senate.gov/imo/media/doc/2020-11-18%20HSGAC%20-%20Finance%20Joint%20Report%20Supplemental.pdf", "https://www.scribd.com/document/479781400/Steele-Spreadsheet-1", "https://www.zerohedge.com/political/jim-comey-ignored-state-department-whistleblower-hillarys-crimes-classified-material", "https://www.judicialwatch.org/wp-content/uploads/2020/10/JW-v-State-Steele-Oct-2020-prod-00968.pdf", "https://justthenews.com/sites/default/files/2020-10/requested%20email.pdf", "https://www.tagblatt.ch/kultur/sommertipps-20-buchempfehlungen-fuer-prominente-wir-haben-die-besten-buecher-fuer-jeden-charakter-zusammengetragen-ld.2159339", "https://www.tagblatt.ch/kultur/sommertipps-20-buchempfehlungen-fuer-prominente-wir-haben-die-besten-buecher-fuer-jeden-charakter-zusammengetragen-ld.2159339", "https://greennetproject.org/en/2018/11/27/prof-dirk-helbing-es-braucht-vor-allem-tolle-ideen-in-die-sich-die-leute-verlieben/", "https://www.nature.com/articles/news.2010.351", "https://www.focus.de/panorama/welt/tid-19265/gastkommentar-nutzt-die-moeglichkeiten-des-computers_aid_534372.html", "http://www.ccss.ethz.ch/Response/index.html", "https://www.hpcwire.com/2011/05/06/simulating_society_at_the_global_scale/", "https://www.say.media/article/la-mort-par-algorithme", "https://www.say.media/article/la-mort-par-algorithme", "https://www.nzz.ch/panorama/wie-kann-eine-massenpanik-verhindert-werden-ld.1614761", "https://www.theglobalist.com/democracy-technology-innovation-society-internet/","https://www.theglobalist.com/capitalism-democracy-technology-surveillance-privacy/","https://www.theglobalist.com/google-artificial-intelligence-big-data-technology-future/","https://www.theglobalist.com/fascism-big-data-artificial-intelligence-surveillance-democracy/","https://www.theglobalist.com/technology-big-data-artificial-intelligence-future-peace-rooms/","https://www.theglobalist.com/technology-society-sustainability-future-humanity/","https://www.theglobalist.com/society-technology-peace-sustainability/","https://www.theglobalist.com/democracy-technology-social-media-artificial-intelligence/","https://www.theglobalist.com/financial-system-reform-economy-internet-of-things-capitalism/","https://www.theglobalist.com/capitalism-society-equality-sustainability-crowd-funding/","https://www.theglobalist.com/united-nations-world-government-peace-sustainability-society/","https://www.theglobalist.com/world-economy-sustainability-environment-society/"] [
"https://www.swissinfo.ch/ger/wirtschaft/koennen-ki-und-direkte-demokratie-nebeneinander-bestehen-/47542048",
"https://www.zeit.de/2011/33/CH-Oekonophysik",
"https://ourworld.unu.edu/en/green-idea-self-organizing-traffic-signals",
"https://www.youtube.com/watch?v=-FQD4ie9UYA",
"https://www.brandeins.de/corporate-services/mck-wissen/mck-wissen-logistik/schwaermen-fuer-das-optimum",
"https://www.youtube.com/watch?v=upQM4Xzh8zM",
"https://www.youtube.com/watch?v=gAkoprZmW4k",
"https://www.youtube.com/watch?v=VMzfDVAWXHI&t=1s",
"https://www.youtube.com/watch?v=1SwTiIlkndE",
"https://www.informatik-aktuell.de/management-und-recht/digitalisierung/digitale-revolution-und-oekonomie-40-quo-vadis.html",
"https://www.youtube.com/watch?v=cSvvH0SBFOw",
"https://www.linkedin.com/posts/margit-osterloh-24198a104_pl%C3%A4doyer-gegen-sprechverbote-ugcPost-6925702100450480129-K7Dl?utm_source=linkedin_share&utm_medium=member_desktop_web",
"https://www.nebelspalter.ch/plaedoyer-gegen-sprechverbote",
"https://falling-walls.com/people/dirk-helbing/",
"https://digitalsensemaker.podigee.io/3-2-mit-dirk-helbing",
"https://www.blick.ch/wirtschaft/musk-als-hueter-der-redefreiheit-eth-experte-sagt-musks-vorhaben-hat-potenzial-aber-id17437811.html",
"https://www.trend.at/standpunkte/mit-verantwortung-zukunft-10082300",
"https://www.pantarhei.ch/podcast/",
"https://ethz.ch/en/industry/industry/news/data/2022/04/intelligent-traffic-lights-for-optimal-traffic-flow.html",
"https://ethz.ch/de/wirtschaft/industry/news/data/2022/04/optimaler-verkehrsfluss-mit-intelligenten-ampeln.html",
"https://www.spektrum.de/news/die-verschlungenen-wege-der-menschen/1181815",
"https://www.pcwelt.de/a/diktatur-4-0-schoene-neue-digitalisierte-welt,3447005",
"https://www.nzz.ch/english/cancel-culture-at-eth-a-professor-receives-death-threats-over-a-lecture-slide-ld.1675322",
"https://www.brandeins.de/corporate-services/mck-wissen/mck-wissen-logistik/schwaermen-fuer-das-optimum",
"https://www.achgut.com/artikel/ausgestossene_der_woche_prinz_william_als_immaginierter_rassist",
"https://www.pinterpolitik.com/in-depth/klaim-big-data-luhut-perlu-diuji/",
"https://www.srf.ch/kultur/gesellschaft-religion/eklat-an-der-eth-wenn-ein-angeblicher-schweinevergleich-zur-staatsaffaere-wird",
"https://open.spotify.com/episode/6s1icdoplZeNOINvx6ZHTd?si=610a699eba004da2&nd=1",
"https://www.nzz.ch/schweiz/shitstorm-an-der-eth-ein-professor-erhaelt-morddrohungen-ld.1673554",
"https://www.nzz.ch/schweiz/shitstorm-an-der-eth-ein-professor-erhaelt-morddrohungen-ld.1673554",
"https://djmag.com/features/after-astroworld-what-being-done-stop-crowd-crushes-happening-again",
"https://prisma-hsg.ch/articles/meine-daten-deine-daten-unsere-daten/",
"https://www.srf.ch/audio/focus/zukunftsforscher-dirk-helbing-die-welt-ist-keine-maschine?id=10756661",
"https://www.20min.ch/story/roboter-fuer-hunde-machen-wenig-sinn-647302764916",
"https://www.wienerzeitung.at/nachrichten/wissen/mensch/942890-Roboter-als-Praesidentschaftskandidaten.html",
"https://disruptors.fm/11-building-a-crystal-ball-of-the-world-unseating-capitalism-and-creating-a-new-world-order-with-prof-dirk-helbing/",
"https://www.spreaker.com/user/disruptorsfm/11-building-crystal-ball-of-the-world-un",
"https://www.youtube.com/watch?v=fRkCMC3zqSQ",
"https://arstechnica.com/science/2021/11/what-the-physics-of-crowds-can-tell-us-about-the-tragic-deaths-at-astroworld/",
"https://www.fox23.com/news/trending/astroworld-festival-big-crowds-can-flow-like-liquid-with-terrifying-results/37QH6Q4RGFELHGCZSZTBV46STU/",
"https://futurism.com/astroworld-theory-deaths-bodies-fluid",
"https://www.businessinsider.com/why-people-died-astroworld-crowd-crush-physics-fluid-dynamics-2021-11",
"https://theconversation.com/ten-tips-for-surviving-a-crowd-crush-112169",
"https://www.limmattalerzeitung.ch/basel/das-wort-zum-tag-kopie-von-4-januar-hypotenuse-schlaegt-kathete-trivia-trampel-pandemie-ld.2233931",
"https://magazine.swissinformatics.org/en/whats-wrong-with-ai/",
"https://magazine.swissinformatics.org/en/whats-wrong-with-ai/",
"https://www.netkwesties.nl/1541/wrr-ai-wordt-de-verbrandingsmotor-van.htm",
"https://youtu.be/ptm9zLG2KaE",
"https://www.deutschlandfunkkultur.de/die-zukunft-der-demokratie-mehr-teilhabe-von-unten-wagen.976.de.html?dram:article_id=468341",
"https://www.springer.com/gp/book/9783642240034",
"https://www.springer.com/de/book/9783319908687",
"https://technikjournal.de/2017/08/02/ein-plaedoyer-fuer-die-digitale-demokratie/",
"https://technikjournal.de/2017/08/02/ein-plaedoyer-fuer-die-digitale-demokratie/",
"https://trafo.hypotheses.org/23989",
"https://web.archive.org/web/20200609053329/https://www.wiko-berlin.de/institution/projekte-kooperationen/projekte/working-futures/wiko-briefs-working-futures-in-corona-times/the-corona-crisis-reveals-the-struggle-for-a-sustainable-digital-future/",
"https://www.wiko-berlin.de/institution/projekte-kooperationen/projekte/working-futures/wiko-briefs-working-futures-in-corona-times/",
"https://www.youtube.com/watch?v=gAkoprZmW4k",
"https://www.rhein-zeitung.de/region/aus-den-lokalredaktionen/nahe-zeitung_artikel,-peter-flaschels-lebenswerk-hat-die-sozialgeschichte-beeinflusst-_arid,2322161.html",
"https://www.blick.ch/wirtschaft/online-boom-ohne-ende-corona-befeuert-die-tech-revolution-id16359910.html",
"https://www.nzz.ch/meinung/china-unterwirft-tech-und-social-media-das-geht-auch-europa-an-ld.1643010",
"https://www.say.media/article/la-mort-par-algorithme",
"https://www.suedostschweiz.ch/aus-dem-leben/2021-08-14/stau-ist-nicht-gleich-stau",
"https://www.swissinfo.ch/eng/directdemocracy/political-perspectives_digital-democracy--too-risky--or-the-chance-of-a-generation-/43836222",
"https://kow-berlin.com/exhibitions/illusion-einer-menschenmenge",
"https://www.springer.com/gp/book/9783642240034",
"https://www.springer.com/de/book/9783319908687",
"https://www.politik-kommunikation.de/ressorts/artikel/eine-gefaehrliche-machtasymmetrie-1383558602",
"https://www.springer.com/gp/book/9783642240034",
"https://www.springer.com/de/book/9783319908687",
"https://solutions.hamburg/ethik-und-digitalisierung-nicht-voneinander-getrennt-betrachten/",
"https://www.springer.com/gp/book/9783642240034",
"https://www.springer.com/de/book/9783319908687",
"https://avenue.argusdatainsights.ch/Article/AvenueClip?artikelHash=d14d91ec9a8b4cb0b6bb3012c0cefd8b_27F0B19422F1F03723769C18906AA1EE&artikelDateiId=298862327",
"https://www.tagblatt.ch/kultur/grosses-ranking-ihre-stimme-hat-gewicht-das-sind-die-50-profiliertesten-intellektuellen-der-schweiz-ld.2182261",
"https://reliefweb.int/report/world/building-multisystemic-understanding-societal-resilience-covid-19-pandemic",
"https://reliefweb.int/report/world/building-multisystemic-understanding-societal-resilience-covid-19-pandemic",
"https://www.events.at/e/wie-wir-in-zukunft-leben-wollen-die-stadt-als-datenfeld",
"https://www.events.at/e/wie-wir-in-zukunft-leben-wollen-die-stadt-als-datenfeld",
"https://greennetproject.org/en/2018/11/27/prof-dirk-helbing-es-braucht-vor-allem-tolle-ideen-in-die-sich-die-leute-verlieben/",
"https://www.hpcwire.com/2011/05/06/simulating_society_at_the_global_scale/",
"https://www.technologyreview.com/2010/04/30/204005/europes-plan-to-simulate-the-entire-planet/",
"https://komentare.sme.sk/c/22543617/smrt-podla-algoritmu.html",
"https://komentare.sme.sk/c/22543617/smrt-podla-algoritmu.html",
"https://www.confidencial.com.ni/opinion/muerte-por-algoritmo/",
"https://www.nzz.ch/panorama/wie-kann-eine-massenpanik-verhindert-werden-ld.1614761",
"https://www.20min.ch/story/roboter-fuer-hunde-machen-wenig-sinn-647302764916",
"https://www.wienerzeitung.at/nachrichten/wissen/mensch/942890-Roboter-als-Praesidentschaftskandidaten.html",
"https://www.srf.ch/audio/focus/zukunftsforscher-dirk-helbing-die-welt-ist-keine-maschine?id=10756661",
"https://disruptors.fm/11-building-a-crystal-ball-of-the-world-unseating-capitalism-and-creating-a-new-world-order-with-prof-dirk-helbing/",
"https://www.spreaker.com/user/disruptorsfm/11-building-crystal-ball-of-the-world-un",
"https://www.youtube.com/watch?v=fRkCMC3zqSQ",
"https://arstechnica.com/science/2021/11/what-the-physics-of-crowds-can-tell-us-about-the-tragic-deaths-at-astroworld/",
"https://www.fox23.com/news/trending/astroworld-festival-big-crowds-can-flow-like-liquid-with-terrifying-results/37QH6Q4RGFELHGCZSZTBV46STU/",
"https://futurism.com/astroworld-theory-deaths-bodies-fluid",
"https://www.businessinsider.com/why-people-died-astroworld-crowd-crush-physics-fluid-dynamics-2021-11",
"https://theconversation.com/ten-tips-for-surviving-a-crowd-crush-112169",
"https://www.limmattalerzeitung.ch/basel/das-wort-zum-tag-kopie-von-4-januar-hypotenuse-schlaegt-kathete-trivia-trampel-pandemie-ld.2233931",
"https://www.pantarhei.ch/podcast/",
"https://www.focus.it/scienza/scienze/folla-fisica-modelli-simulazioni",
"https://www.focus.it/scienza/scienze/folla-fisica-modelli-simulazioni",
"https://www.netkwesties.nl/1541/wrr-ai-wordt-de-verbrandingsmotor-van.htm",
"https://www.transformationbeats.com/de/transformation/digitale-gesellschaft/",
"https://www.transformationbeats.com/de/transformation/digitale-gesellschaft/",
"https://www.suedkurier.de/ueberregional/wirtschaft/Wie-uns-der-Staat-heimlich-erzieht-sogar-auf-dem-Klo;art416,8763904",
"https://www.suedkurier.de/ueberregional/wirtschaft/Wie-uns-der-Staat-heimlich-erzieht-sogar-auf-dem-Klo;art416,8763904",
"https://www.deutschlandfunkkultur.de/die-zukunft-der-demokratie-mehr-teilhabe-von-unten-wagen.976.de.html?dram:article_id=468341",
"https://www.springer.com/gp/book/9783642240034",
"https://www.springer.com/de/book/9783319908687",
"https://trafo.hypotheses.org/23989",
"https://web.archive.org/web/20200609053329/https://www.wiko-berlin.de/institution/projekte-kooperationen/projekte/working-futures/wiko-briefs-working-futures-in-corona-times/the-corona-crisis-reveals-the-struggle-for-a-sustainable-digital-future/",
"https://www.wiko-berlin.de/institution/projekte-kooperationen/projekte/working-futures/wiko-briefs-working-futures-in-corona-times/",
"https://www.youtube.com/watch?v=gAkoprZmW4k",
"https://futurium.de/de/gespraech/ranga-yogeshwar-1/ranga-yogeshwar-dirk-helbing-mit-musik-von-till-broenner",
"https://www.springer.com/gp/book/9783642240034",
"https://www.springer.com/de/book/9783319908687",
"https://idw-online.de/en/news113518",
"https://blmplus.de/die-digitalcharta-ist-erst-der-anfang-ein-szenario-von-dirk-helbing/",
"https://www.risiko-dialog.ch/big-nudging-vom-computer-gelenkt-aber-wohin/",
"https://idw-online.de/de/news13986",
"https://www.uni-stuttgart.de/presse/archiv/uni-kurier/uk84_85/forschung/fw66.html",
"https://www.infosperber.ch/medien/trends/rankings-oft-unbrauchbar-so-oder-so-aber-immer-schadlich/",
"https://www.infosperber.ch/medien/trends/rankings-oft-unbrauchbar-so-oder-so-aber-immer-schadlich/",
"https://www.nzz.ch/meinung/china-unterwirft-tech-und-social-media-das-geht-auch-europa-an-ld.1643010",
"https://www.suedostschweiz.ch/aus-dem-leben/2021-08-14/stau-ist-nicht-gleich-stau",
"https://www.swissinfo.ch/eng/directdemocracy/political-perspectives_digital-democracy--too-risky--or-the-chance-of-a-generation-/43836222",
"https://werteundwandel.de/inhalte/d2030-in-aufbruchstimmung-fuer-eine-lebenswerte-zukunft/",
"https://www.springer.com/gp/book/9783642240034",
"https://www.springer.com/de/book/9783319908687",
"https://www.youtube.com/watch?v=n9e77iYZPEY",
"https://greennetproject.org/en/2018/11/27/prof-dirk-helbing-es-braucht-vor-allem-tolle-ideen-in-die-sich-die-leute-verlieben/",
"https://www.hpcwire.com/2011/05/06/simulating_society_at_the_global_scale/",
"https://www.say.media/article/la-mort-par-algorithme",
"https://www.confidencial.com.ni/opinion/muerte-por-algoritmo/",
"https://www.nzz.ch/panorama/wie-kann-eine-massenpanik-verhindert-werden-ld.1614761",
"https://www.nesta.org.uk/report/digital-democracy-the-tools-transforming-political-engagement/",
"https://www.nature.com/articles/news.2010.351",
"https://www.focus.de/panorama/welt/tid-19265/gastkommentar-nutzt-die-moeglichkeiten-des-computers_aid_534372.html",
"https://www.theglobalist.com/democracy-technology-innovation-society-internet/",
"https://www.theglobalist.com/capitalism-democracy-technology-surveillance-privacy/",
"https://www.theglobalist.com/google-artificial-intelligence-big-data-technology-future/",
"https://www.theglobalist.com/fascism-big-data-artificial-intelligence-surveillance-democracy/",
"https://www.theglobalist.com/technology-big-data-artificial-intelligence-future-peace-rooms/",
"https://www.theglobalist.com/technology-society-sustainability-future-humanity/",
"https://www.theglobalist.com/society-technology-peace-sustainability/",
"https://www.theglobalist.com/democracy-technology-social-media-artificial-intelligence/",
"https://www.theglobalist.com/financial-system-reform-economy-internet-of-things-capitalism/",
"https://www.theglobalist.com/capitalism-society-equality-sustainability-crowd-funding/",
"https://www.theglobalist.com/united-nations-world-government-peace-sustainability-society/",
"https://www.theglobalist.com/world-economy-sustainability-environment-society/"
]

10
requirements.txt Normal file
View File

@@ -0,0 +1,10 @@
peewee
selenium
# youtube-dl
git+https://github.com/ytdl-org/youtube-dl.git #non-release version with faster downloads
waybackpy
slack_bolt # relies on slack_sdk
newspaper3k
htmldate
markdown
rich