import time import keys import slack_sdk from slack_sdk.errors import SlackApiError from peewee import SqliteDatabase from persistence import message_models # from bot_utils import messages # Constant values... MESSAGES_DB = "/app/containerdata/messages.db" BOT_ID = "U02MR1R8UJH" ARCHIVE_ID = "C02MM7YG1V4" DEBUG_ID = "C02NM2H9J5Q" client = slack_sdk.WebClient(token=keys.OAUTH_TOKEN) message_models.set_db(SqliteDatabase(MESSAGES_DB)) def message_dict_to_model(message): if message["type"] == "message": thread_ts = message["thread_ts"] if "thread_ts" in message else message["ts"] uid = message.get("user", "BAD USER") user, _ = message_models.User.get_or_create(user_id = uid) thread, _ = message_models.Thread.get_or_create(thread_ts = thread_ts) m, new = message_models.Message.get_or_create( user = user, thread = thread, ts = message["ts"], channel_id = ARCHIVE_ID, text = message["text"] ) print("Saved (text) {} (new={})".format(m, new)) for f in message.get("files", []): #default: [] m.file_type = f["filetype"] m.perma_link = f["url_private_download"] m.save() print("Saved permalink {} to {} (possibly overwriting)".format(f["name"], m)) if new: return m else: return None else: print("What should I do of {}".format(message)) return None def check_all_past_messages(): last_ts = 0 result = client.conversations_history( channel=ARCHIVE_ID, oldest=last_ts ) new_messages = result.get("messages", []) # fetches 100 messages by default new_fetches = [] for m in new_messages: new_fetches.append(message_dict_to_model(m)) # print(result) refetch = result.get("has_more", False) print(f"Refetching : {refetch}") while refetch: # we have not actually fetched them all try: result = client.conversations_history( channel = ARCHIVE_ID, cursor = result["response_metadata"]["next_cursor"], oldest = last_ts ) # refetches in batches of 100 messages refetch = result.get("has_more", False) new_messages = result.get("messages", []) for m in new_messages: new_fetches.append(message_dict_to_model(m)) except SlackApiError: # Most likely a rate-limit print("Error while fetching channel messages. (likely rate limit) Retrying in {} seconds...".format(30)) time.sleep(30) refetch = True check_all_past_messages()