88 lines
2.6 KiB
Python
88 lines
2.6 KiB
Python
import time
|
|
import keys
|
|
import slack_sdk
|
|
from slack_sdk.errors import SlackApiError
|
|
from peewee import SqliteDatabase
|
|
|
|
from persistence import message_models
|
|
# from bot_utils import messages
|
|
|
|
|
|
|
|
# Constant values...
|
|
MESSAGES_DB = "/app/containerdata/messages.db"
|
|
|
|
BOT_ID = "U02MR1R8UJH"
|
|
ARCHIVE_ID = "C02MM7YG1V4"
|
|
DEBUG_ID = "C02NM2H9J5Q"
|
|
|
|
|
|
|
|
client = slack_sdk.WebClient(token=keys.OAUTH_TOKEN)
|
|
|
|
message_models.set_db(SqliteDatabase(MESSAGES_DB))
|
|
|
|
|
|
def message_dict_to_model(message):
|
|
if message["type"] == "message":
|
|
thread_ts = message["thread_ts"] if "thread_ts" in message else message["ts"]
|
|
uid = message.get("user", "BAD USER")
|
|
user, _ = message_models.User.get_or_create(user_id = uid)
|
|
thread, _ = message_models.Thread.get_or_create(thread_ts = thread_ts)
|
|
m, new = message_models.Message.get_or_create(
|
|
user = user,
|
|
thread = thread,
|
|
ts = message["ts"],
|
|
channel_id = ARCHIVE_ID,
|
|
text = message["text"]
|
|
)
|
|
print("Saved (text) {} (new={})".format(m, new))
|
|
|
|
for f in message.get("files", []): #default: []
|
|
m.file_type = f["filetype"]
|
|
m.perma_link = f["url_private_download"]
|
|
m.save()
|
|
print("Saved permalink {} to {} (possibly overwriting)".format(f["name"], m))
|
|
if new:
|
|
return m
|
|
else:
|
|
return None
|
|
else:
|
|
print("What should I do of {}".format(message))
|
|
return None
|
|
|
|
|
|
def check_all_past_messages():
|
|
last_ts = 0
|
|
|
|
result = client.conversations_history(
|
|
channel=ARCHIVE_ID,
|
|
oldest=last_ts
|
|
)
|
|
|
|
new_messages = result.get("messages", []) # fetches 100 messages by default
|
|
|
|
new_fetches = []
|
|
for m in new_messages:
|
|
new_fetches.append(message_dict_to_model(m))
|
|
# print(result)
|
|
refetch = result.get("has_more", False)
|
|
print(f"Refetching : {refetch}")
|
|
while refetch: # we have not actually fetched them all
|
|
try:
|
|
result = client.conversations_history(
|
|
channel = ARCHIVE_ID,
|
|
cursor = result["response_metadata"]["next_cursor"],
|
|
oldest = last_ts
|
|
) # refetches in batches of 100 messages
|
|
refetch = result.get("has_more", False)
|
|
new_messages = result.get("messages", [])
|
|
for m in new_messages:
|
|
new_fetches.append(message_dict_to_model(m))
|
|
except SlackApiError: # Most likely a rate-limit
|
|
print("Error while fetching channel messages. (likely rate limit) Retrying in {} seconds...".format(30))
|
|
time.sleep(30)
|
|
refetch = True
|
|
|
|
|
|
check_all_past_messages() |