remoll/coss_archiving
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed browser profile bug, line breaks and exceptions in news_check

Browse Source
This commit is contained in:
Remy Moll
2022-09-26 15:25:55 +02:00
parent db161e50c8
commit 9349b046d2
12 changed files with 150 additions and 319 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
misc/sample_config/news_fetch.config.ini
View File

@@ -26,4 +26,6 @@ local_storage_path: /app/containerdata/files
debug_storage_path: /app/containerdata/debug/
default_download_path: /app/containerdata/tmp
remote_storage_path: /helbing_support/Files RM/Archiving
browser_profile_path: /user_data/news_fetch.profile
browser_profile_path: /app/containerdata/dependencies/news_fetch.profile
# please keep this exact name
browser_print_delay: 5

45
misc/youtube_batch.py
View File

@@ -4,24 +4,27 @@ import time
urls = [
"https://www.youtube.com/watch?v=R4h_yiDIuQE",
"https://www.youtube.com/watch?v=-G8ZI1Jq8xA",
"https://www.youtube.com/watch?v=8eYBcASQIQI",
"https://www.thingiverse.com/thing:5463267",
"https://www.youtube.com/watch?v=cJoUSHJcV4E&t=0s",
"https://www.youtube.com/watch?v=UbBYZZBREBA&t=0s",
"https://www.youtube.com/watch?v=bQQn_vET4ys",
"https://www.youtube.com/watch?v=6FqNctiO06E",
"https://www.youtube.com/watch?v=ImnuJgj8XJo",
"https://www.youtube.com/watch?v=4QZQtSqaC34",
"https://www.youtube.com/watch?v=cW4qIjPMGkQ",
"https://www.youtube.com/watch?v=QWsUGpKfP8A",
"https://www.youtube.com/watch?v=a0PwEwLG9No",
"https://www.youtube.com/watch?v=Hd3lnWVIIpo",
"https://www.youtube.com/watch?v=JNtdAp-BdzI",
"https://en.wikipedia.org/wiki/Viktor_Schauberger",
"https://de.wikipedia.org/wiki/Viktor_Schauberger",
"https://id2020.org",
"https://www.weforum.org/platforms/the-centre-for-cybersecurity",
"https://www.unhcr.org/blogs/wp-content/uploads/sites/48/2018/04/fs.pdf",
"https://en.wikipedia.org/wiki/Social_Credit_System",
"https://en.wikipedia.org/wiki/Customer_lifetime_value",
"https://www.weforum.org/reports/the-internet-of-bodies-is-here-tackling-new-challenges-of-technology-governance",
"https://www.un.org/en/about-us/universal-declaration-of-human-rights",
"https://www.biometricupdate.com/201909/id2020-and-partners-launch-program-to-provide-digital-id-with-vaccines",
"https://www.wired.com/2008/06/pb-theory/",
"https://www.medtechdive.com/news/fda-warns-of-false-positives-with-bd-coronavirus-diagnostic/581115/",
"https://www.bbc.com/news/world-middle-east-52579475",
"https://www.timesofisrael.com/over-12000-mistakenly-quarantined-by-phone-tracking-health-ministry-admits/",
"https://www.delftdesignforvalues.nl",
"https://www.theglobalist.com/technology-big-data-artificial-intelligence-future-peace-rooms/",
"https://link.springer.com/chapter/10.1007/978-3-319-90869-4_17",
"https://www.youtube.com/watch?v=_KhAsJRk2lo",
"https://www.bloomberg.org/environment/supporting-sustainable-cities/american-cities-climate-challenge/",
"https://climatecitycup.org",
]
def post_download_hook(ret_code):
# print(ret_code)
if ret_code['status'] == 'finished':
@@ -45,10 +48,12 @@ def save_video(url):
print(f"Youtube download crashed: {e}")
# for url in urls:
# save_video(url)
# for i, url in enumerate(urls):
# print(f"Downloading video {i+1} / {len(urls)}")
# save_video(url)
for url in urls:
for i, url in enumerate(urls):
print(f"Saving url {i+1} / {len(urls)}")
user_agent = "Mozilla/5.0 (Windows NT 5.1; rv:40.0) Gecko/20100101 Firefox/40.0" # needed?
wayback = WaybackMachineSaveAPI(url, user_agent)
archive_url = wayback.save()