qbi/FediFetcher
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

hook up host software lookup to user/post fetching

Browse source 
add depth fetching for misskey posts (works in firefish and foundkey)
This commit is contained in:
Toad King 2023-07-29 22:30:25 -05:00
parent c92d4e1c2c
commit 80ee1387f7
1 changed files with 187 additions and 134 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

307
find_posts.py
View file

@ -67,17 +67,17 @@ def get_favourites(server, access_token, max):
"Authorization": f"Bearer {access_token}", "Authorization": f"Bearer {access_token}",
}) })
def add_user_posts(server, access_token, followings, know_followings, all_known_users, seen_urls): def add_user_posts(server, access_token, followings, known_followings, all_known_users, seen_urls, seen_hosts):
for user in followings: for user in followings:
if user['acct'] not in all_known_users and not user['url'].startswith(f"https://{server}/"): if user['acct'] not in all_known_users and not user['url'].startswith(f"https://{server}/"):
posts = get_user_posts(user, know_followings, server) posts = get_user_posts(user, known_followings, server, seen_hosts)
if(posts != None): if(posts != None):
count = 0 count = 0
failed = 0 failed = 0
for post in posts: for post in posts:
if post.get('reblog') is None and post.get('renoteId') is None and post.get('url') is not None and post.get('url') not in seen_urls: if post.get('reblog') is None and post.get('renoteId') is None and post.get('url') is not None and post.get('url') not in seen_urls:
added = add_post_with_context(post, server, access_token, seen_urls) added = add_post_with_context(post, server, access_token, seen_urls, seen_hosts)
if added is True: if added is True:
seen_urls.add(post['url']) seen_urls.add(post['url'])
count += 1 count += 1
@ -85,10 +85,10 @@ def add_user_posts(server, access_token, followings, know_followings, all_known_
failed += 1 failed += 1
log(f"Added {count} posts for user {user['acct']} with {failed} errors") log(f"Added {count} posts for user {user['acct']} with {failed} errors")
if failed == 0: if failed == 0:
know_followings.add(user['acct']) known_followings.add(user['acct'])
all_known_users.add(user['acct']) all_known_users.add(user['acct'])
def add_post_with_context(post, server, access_token, seen_urls): def add_post_with_context(post, server, access_token, seen_urls, seen_hosts):
added = add_context_url(post['url'], server, access_token) added = add_context_url(post['url'], server, access_token)
if added is True: if added is True:
seen_urls.add(post['url']) seen_urls.add(post['url'])
@ -97,27 +97,72 @@ def add_post_with_context(post, server, access_token, seen_urls):
parsed = parse_url(post['url'], parsed_urls) parsed = parse_url(post['url'], parsed_urls)
if parsed == None: if parsed == None:
return True return True
known_context_urls = get_all_known_context_urls(server, [post],parsed_urls) known_context_urls = get_all_known_context_urls(server, [post],parsed_urls, seen_hosts)
add_context_urls(server, access_token, known_context_urls, seen_urls) add_context_urls(server, access_token, known_context_urls, seen_urls)
return True return True
return False return False
def get_user_posts(user, know_followings, server): def get_user_posts(user, known_followings, server, seen_hosts):
parsed_url = parse_user_url(user['url']) parsed_url = parse_user_url(user['url'])
if parsed_url == None: if parsed_url == None:
# We are adding it as 'known' anyway, because we won't be able to fix this. # We are adding it as 'known' anyway, because we won't be able to fix this.
know_followings.add(user['acct']) known_followings.add(user['acct'])
return None return None
if(parsed_url[0] == server): if(parsed_url[0] == server):
log(f"{user['acct']} is a local user. Skip") log(f"{user['acct']} is a local user. Skip")
know_followings.add(user['acct']) known_followings.add(user['acct'])
return None return None
if re.match(r"^https:\/\/[^\/]+\/c\/", user['url']):
post_server = get_server_info(parsed_url[0], seen_hosts)
if post_server is None:
log(f'server {parsed_url[0]} not found for post')
return None
if post_server['mastodonApiSupport']:
return get_user_posts_mastodon(parsed_url[1], post_server['webserver'])
if post_server['lemmyApiSupport']:
return get_user_posts_lemmy(parsed_url[1], user['url'], post_server['webserver'])
if post_server['misskeyApiSupport']:
return get_user_posts_misskey(parsed_url[1], post_server['webserver'])
log(f'server api unknown for {post_server["webserver"]}, cannot fetch user posts')
return None
def get_user_posts_mastodon(userName, webserver):
try: try:
url = f"https://{parsed_url[0]}/api/v3/post/list?community_name={parsed_url[1]}&sort=New&limit=50" user_id = get_user_id(webserver, userName)
except Exception as ex:
log(f"Error getting user ID for user {user['acct']}: {ex}")
return None
try:
url = f"https://{webserver}/api/v1/accounts/{user_id}/statuses?limit=40"
response = get(url)
if(response.status_code == 200):
return response.json()
elif response.status_code == 404:
raise Exception(
f"User {user['acct']} was not found on server {webserver}"
)
else:
raise Exception(
f"Error getting URL {url}. Status code: {response.status_code}"
)
except Exception as ex:
log(f"Error getting posts for user {user['acct']}: {ex}")
return None
def get_user_posts_lemmy(userName, userUrl, webserver):
# community
if re.match(r"^https:\/\/[^\/]+\/c\/", userUrl):
try:
url = f"https://{webserver}/api/v3/post/list?community_name={userName}&sort=New&limit=50"
response = get(url) response = get(url)
if(response.status_code == 200): if(response.status_code == 200):
@ -127,12 +172,13 @@ def get_user_posts(user, know_followings, server):
return posts return posts
except Exception as ex: except Exception as ex:
log(f"Error getting community posts for community {parsed_url[1]}: {ex}") log(f"Error getting community posts for community {userName}: {ex}")
return None return None
if re.match(r"^https:\/\/[^\/]+\/u\/", user['url']): # user
if re.match(r"^https:\/\/[^\/]+\/u\/", userUrl):
try: try:
url = f"https://{parsed_url[0]}/api/v3/user?username={parsed_url[1]}&sort=New&limit=50" url = f"https://{webserver}/api/v3/user?username={userName}&sort=New&limit=50"
response = get(url) response = get(url)
if(response.status_code == 200): if(response.status_code == 200):
@ -144,84 +190,51 @@ def get_user_posts(user, know_followings, server):
return all_posts return all_posts
except Exception as ex: except Exception as ex:
log(f"Error getting user posts for user {parsed_url[1]}: {ex}") log(f"Error getting user posts for user {userName}: {ex}")
return None return None
try: def get_user_posts_misskey(userName, webserver):
user_id = get_user_id(parsed_url[0], parsed_url[1])
except Exception as ex:
log(f"Error getting user ID for user {user['acct']}: {ex}")
## HACK HACK HACK: in the future actually try to detect the correct APIs to use at runtime
log('trying misskey')
return get_user_posts_misskey(user, know_followings, parsed_url[0])
try:
url = f"https://{parsed_url[0]}/api/v1/accounts/{user_id}/statuses?limit=40"
response = get(url)
if(response.status_code == 200):
return response.json()
elif response.status_code == 404:
raise Exception(
f"User {user['acct']} was not found on server {parsed_url[0]}"
)
else:
raise Exception(
f"Error getting URL {url}. Status code: {response.status_code}"
)
except Exception as ex:
log(f"Error getting posts for user {user['acct']}: {ex}")
return None
def get_user_posts_misskey(user, know_followings, server):
# query user info via search api # query user info via search api
# we could filter by host but there's no way to limit that to just the main host on firefish currently # we could filter by host but there's no way to limit that to just the main host on firefish currently
# on misskey it works if you supply '.' as the host but firefish does not # on misskey it works if you supply '.' as the host but firefish does not
url = f'https://{server}/api/users/search-by-username-and-host' userId = None
try: try:
resp = post(url, { 'username': user['username'] }) url = f'https://{webserver}/api/users/search-by-username-and-host'
except Exception as ex: resp = post(url, { 'username': userName })
log(f"Error finding user {user['username']} from {server}. Exception: {ex}")
return None
if resp.status_code == 200: if resp.status_code == 200:
try:
res = resp.json() res = resp.json()
for user in res: for user in res:
if user['host'] is None: if user['host'] is None:
userId = user['id'] userId = user['id']
break break
if userId is None:
raise Exception('user not found on server in search')
except Exception as ex:
log(f"Error finding user {user['username']} from {server}. Exception: {ex}")
return None
else: else:
log(f"Error finding user {user['username']} from {server}. Status Code: {resp.status_code}") log(f"Error finding user {userName} from {webserver}. Status Code: {resp.status_code}")
return None
except Exception as ex:
log(f"Error finding user {userName} from {webserver}. Exception: {ex}")
return None return None
url = f'https://{server}/api/users/notes' if userId is None:
try: log(f'Error finding user {userName} from {webserver}: user not found on server in search')
resp = post(url, { 'userId': userId, 'limit': 40 })
except Exception as ex:
log(f"Error getting posts by user {user['username']} from {server}. Exception: {ex}")
return None return None
try:
url = f'https://{webserver}/api/users/notes'
resp = post(url, { 'userId': userId, 'limit': 40 })
if resp.status_code == 200: if resp.status_code == 200:
try:
notes = resp.json() notes = resp.json()
for note in notes: for note in notes:
if note.get('url') is None: if note.get('url') is None:
# add this to make it look like Mastodon status objects # add this to make it look like Mastodon status objects
note.update({ 'url': f"https://{server}/notes/{note['id']}" }) note.update({ 'url': f"https://{webserver}/notes/{note['id']}" })
return notes return notes
except Exception as ex:
log(f"Error getting posts by user {user['username']} from {server}. Exception: {ex}")
return None
else: else:
log(f"Error getting posts by user {user['username']} from {server}. Status Code: {resp.status_code}") log(f"Error getting posts by user {userName} from {webserver}. Status Code: {resp.status_code}")
return None
except Exception as ex:
log(f"Error getting posts by user {userName} from {webserver}. Exception: {ex}")
return None return None
@ -443,7 +456,7 @@ def get_reply_toots(user_id, server, access_token, seen_urls, reply_since):
) )
def get_all_known_context_urls(server, reply_toots, parsed_urls): def get_all_known_context_urls(server, reply_toots, parsed_urls, seen_hosts):
"""get the context toots of the given toots from their original server""" """get the context toots of the given toots from their original server"""
known_context_urls = set() known_context_urls = set()
@ -451,7 +464,7 @@ def get_all_known_context_urls(server, reply_toots, parsed_urls):
if toot_has_parseable_url(toot, parsed_urls): if toot_has_parseable_url(toot, parsed_urls):
url = toot["url"] if toot["reblog"] is None else toot["reblog"]["url"] url = toot["url"] if toot["reblog"] is None else toot["reblog"]["url"]
parsed_url = parse_url(url, parsed_urls) parsed_url = parse_url(url, parsed_urls)
context = get_toot_context(parsed_url[0], parsed_url[1], url) context = get_toot_context(parsed_url[0], parsed_url[1], url, seen_hosts)
if context is not None: if context is not None:
for item in context: for item in context:
known_context_urls.add(item) known_context_urls.add(item)
@ -675,26 +688,37 @@ def get_redirect_url(url):
return None return None
def get_all_context_urls(server, replied_toot_ids): def get_all_context_urls(server, replied_toot_ids, seen_hosts):
"""get the URLs of the context toots of the given toots""" """get the URLs of the context toots of the given toots"""
return filter( return filter(
lambda url: not url.startswith(f"https://{server}/"), lambda url: not url.startswith(f"https://{server}/"),
itertools.chain.from_iterable( itertools.chain.from_iterable(
get_toot_context(server, toot_id, url) get_toot_context(server, toot_id, url, seen_hosts)
for (url, (server, toot_id)) in replied_toot_ids for (url, (server, toot_id)) in replied_toot_ids
), ),
) )
def get_toot_context(server, toot_id, toot_url): def get_toot_context(server, toot_id, toot_url, seen_hosts):
"""get the URLs of the context toots of the given toot""" """get the URLs of the context toots of the given toot"""
if toot_url.find("/comment/") != -1:
return get_comment_context(server, toot_id, toot_url) post_server = get_server_info(server, seen_hosts)
if toot_url.find("/post/") != -1: if post_server is None:
return get_comments_urls(server, toot_id, toot_url) log(f'server {server} not found for post')
if toot_url.find("/notes/") != -1: return []
return get_misskey_urls(server, toot_id, toot_url)
url = f"https://{server}/api/v1/statuses/{toot_id}/context" if post_server['mastodonApiSupport']:
return get_mastodon_urls(post_server['webserver'], toot_id, toot_url)
if post_server['lemmyApiSupport']:
return get_lemmy_urls(post_server['webserver'], toot_id, toot_url)
if post_server['misskeyApiSupport']:
return get_misskey_urls(post_server['webserver'], toot_id, toot_url)
log(f'unknown server api for {server}')
return []
def get_mastodon_urls(webserver, toot_id, toot_url):
url = f"https://{webserver}/api/v1/statuses/{toot_id}/context"
try: try:
resp = get(url) resp = get(url)
except Exception as ex: except Exception as ex:
@ -713,16 +737,25 @@ def get_toot_context(server, toot_id, toot_url):
reset = datetime.strptime(resp.headers['x-ratelimit-reset'], '%Y-%m-%dT%H:%M:%S.%fZ') reset = datetime.strptime(resp.headers['x-ratelimit-reset'], '%Y-%m-%dT%H:%M:%S.%fZ')
log(f"Rate Limit hit when getting context for {toot_url}. Waiting to retry at {resp.headers['x-ratelimit-reset']}") log(f"Rate Limit hit when getting context for {toot_url}. Waiting to retry at {resp.headers['x-ratelimit-reset']}")
time.sleep((reset - datetime.now()).total_seconds() + 1) time.sleep((reset - datetime.now()).total_seconds() + 1)
return get_toot_context(server, toot_id, toot_url) return get_toot_context(server, toot_id, toot_url, seen_hosts)
log( log(
f"Error getting context for toot {toot_url}. Status code: {resp.status_code}" f"Error getting context for toot {toot_url}. Status code: {resp.status_code}"
) )
return [] return []
def get_comment_context(server, toot_id, toot_url): def get_lemmy_urls(webserver, toot_id, toot_url):
if toot_url.find("/comment/") != -1:
return get_lemmy_comment_context(webserver, toot_id, toot_url)
if toot_url.find("/post/") != -1:
return get_lemmy_comments_urls(webserver, toot_id, toot_url)
else:
log(f'unknown lemmy url type {toot_url}')
return []
def get_lemmy_comment_context(webserver, toot_id, toot_url):
"""get the URLs of the context toots of the given toot""" """get the URLs of the context toots of the given toot"""
comment = f"https://{server}/api/v3/comment?id={toot_id}" comment = f"https://{webserver}/api/v3/comment?id={toot_id}"
try: try:
resp = get(comment) resp = get(comment)
except Exception as ex: except Exception as ex:
@ -733,7 +766,7 @@ def get_comment_context(server, toot_id, toot_url):
try: try:
res = resp.json() res = resp.json()
post_id = res['comment_view']['comment']['post_id'] post_id = res['comment_view']['comment']['post_id']
return get_comments_urls(server, post_id, toot_url) return get_lemmy_comments_urls(webserver, post_id, toot_url)
except Exception as ex: except Exception as ex:
log(f"Error parsing context for comment {toot_url}. Exception: {ex}") log(f"Error parsing context for comment {toot_url}. Exception: {ex}")
return [] return []
@ -741,12 +774,12 @@ def get_comment_context(server, toot_id, toot_url):
reset = datetime.strptime(resp.headers['x-ratelimit-reset'], '%Y-%m-%dT%H:%M:%S.%fZ') reset = datetime.strptime(resp.headers['x-ratelimit-reset'], '%Y-%m-%dT%H:%M:%S.%fZ')
log(f"Rate Limit hit when getting context for {toot_url}. Waiting to retry at {resp.headers['x-ratelimit-reset']}") log(f"Rate Limit hit when getting context for {toot_url}. Waiting to retry at {resp.headers['x-ratelimit-reset']}")
time.sleep((reset - datetime.now()).total_seconds() + 1) time.sleep((reset - datetime.now()).total_seconds() + 1)
return get_comment_context(server, toot_id, toot_url) return get_lemmy_comment_context(webserver, toot_id, toot_url)
def get_comments_urls(server, post_id, toot_url): def get_lemmy_comments_urls(webserver, post_id, toot_url):
"""get the URLs of the comments of the given post""" """get the URLs of the comments of the given post"""
urls = [] urls = []
url = f"https://{server}/api/v3/post?id={post_id}" url = f"https://{webserver}/api/v3/post?id={post_id}"
try: try:
resp = get(url) resp = get(url)
except Exception as ex: except Exception as ex:
@ -762,7 +795,7 @@ def get_comments_urls(server, post_id, toot_url):
except Exception as ex: except Exception as ex:
log(f"Error parsing post {post_id} from {toot_url}. Exception: {ex}") log(f"Error parsing post {post_id} from {toot_url}. Exception: {ex}")
url = f"https://{server}/api/v3/comment/list?post_id={post_id}&sort=New&limit=50" url = f"https://{webserver}/api/v3/comment/list?post_id={post_id}&sort=New&limit=50"
try: try:
resp = get(url) resp = get(url)
except Exception as ex: except Exception as ex:
@ -782,17 +815,18 @@ def get_comments_urls(server, post_id, toot_url):
reset = datetime.strptime(resp.headers['x-ratelimit-reset'], '%Y-%m-%dT%H:%M:%S.%fZ') reset = datetime.strptime(resp.headers['x-ratelimit-reset'], '%Y-%m-%dT%H:%M:%S.%fZ')
log(f"Rate Limit hit when getting comments for {toot_url}. Waiting to retry at {resp.headers['x-ratelimit-reset']}") log(f"Rate Limit hit when getting comments for {toot_url}. Waiting to retry at {resp.headers['x-ratelimit-reset']}")
time.sleep((reset - datetime.now()).total_seconds() + 1) time.sleep((reset - datetime.now()).total_seconds() + 1)
return get_comments_urls(server, post_id, toot_url) return get_lemmy_comments_urls(webserver, post_id, toot_url)
log(f"Error getting comments for post {toot_url}. Status code: {resp.status_code}") log(f"Error getting comments for post {toot_url}. Status code: {resp.status_code}")
return [] return []
def get_misskey_urls(server, post_id, toot_url): def get_misskey_urls(webserver, post_id, toot_url):
"""get the URLs of the comments of a given misskey post""" """get the URLs of the comments of a given misskey post"""
urls = [] urls = []
url = f"https://{server}/api/notes/children" url = f"https://{webserver}/api/notes/children"
try: try:
resp = post(url, { 'noteId': post_id }) resp = post(url, { 'noteId': post_id, 'limit': 100, 'depth': 12 })
except Exception as ex: except Exception as ex:
log(f"Error getting post {post_id} from {toot_url}. Exception: {ex}") log(f"Error getting post {post_id} from {toot_url}. Exception: {ex}")
return [] return []
@ -801,16 +835,16 @@ def get_misskey_urls(server, post_id, toot_url):
try: try:
res = resp.json() res = resp.json()
log(f"Got children for misskey post {toot_url}") log(f"Got children for misskey post {toot_url}")
list_of_urls = [f'https://{server}/notes/{comment_info["id"]}' for comment_info in res] list_of_urls = [f'https://{webserver}/notes/{comment_info["id"]}' for comment_info in res]
urls.extend(list_of_urls) urls.extend(list_of_urls)
except Exception as ex: except Exception as ex:
log(f"Error parsing post {post_id} from {toot_url}. Exception: {ex}") log(f"Error parsing post {post_id} from {toot_url}. Exception: {ex}")
else: else:
log(f"Error getting post {post_id} from {toot_url}. Status Code: {resp.status_code}") log(f"Error getting post {post_id} from {toot_url}. Status Code: {resp.status_code}")
url = f"https://{server}/api/notes/conversation" url = f"https://{webserver}/api/notes/conversation"
try: try:
resp = post(url, { 'noteId': post_id }) resp = post(url, { 'noteId': post_id, 'limit': 100 })
except Exception as ex: except Exception as ex:
log(f"Error getting post {post_id} from {toot_url}. Exception: {ex}") log(f"Error getting post {post_id} from {toot_url}. Exception: {ex}")
return [] return []
@ -819,7 +853,7 @@ def get_misskey_urls(server, post_id, toot_url):
try: try:
res = resp.json() res = resp.json()
log(f"Got conversation for misskey post {toot_url}") log(f"Got conversation for misskey post {toot_url}")
list_of_urls = [f'https://{server}/notes/{comment_info["id"]}' for comment_info in res] list_of_urls = [f'https://{webserver}/notes/{comment_info["id"]}' for comment_info in res]
urls.extend(list_of_urls) urls.extend(list_of_urls)
except Exception as ex: except Exception as ex:
log(f"Error parsing post {post_id} from {toot_url}. Exception: {ex}") log(f"Error parsing post {post_id} from {toot_url}. Exception: {ex}")
@ -1037,7 +1071,7 @@ def get_server_from_host_meta(server):
if resp.status_code == 200: if resp.status_code == 200:
try: try:
hostMeta = ET.fromstring(resp.text) hostMeta = ET.fromstring(resp.text)
lrdd = hostMeta.find(".//{http://docs.oasis-open.org/ns/xri/xrd-1.0}Link[@rel='lrdd']") lrdd = hostMeta.find('.//{http://docs.oasis-open.org/ns/xri/xrd-1.0}Link[@rel="lrdd"]')
url = lrdd.get('template') url = lrdd.get('template')
match = re.match( match = re.match(
r"https://(?P<server>[^/]+)/", url r"https://(?P<server>[^/]+)/", url
@ -1045,7 +1079,7 @@ def get_server_from_host_meta(server):
if match is not None: if match is not None:
return match.group("server") return match.group("server")
else: else:
raise Exception('server not found in lrdd') raise Exception(f'server not found in lrdd for {server}')
return None return None
except Exception as ex: except Exception as ex:
log(f'Error parsing host meta for {server}. Exception: {ex}') log(f'Error parsing host meta for {server}. Exception: {ex}')
@ -1054,7 +1088,7 @@ def get_server_from_host_meta(server):
log(f'Error getting host meta for {server}. Status Code: {resp.status_code}') log(f'Error getting host meta for {server}. Status Code: {resp.status_code}')
return None return None
def get_nodeinfo(server, host_meta_fallback = False): def get_nodeinfo(server, seen_hosts, host_meta_fallback = False):
url = f'https://{server}/.well-known/nodeinfo' url = f'https://{server}/.well-known/nodeinfo'
try: try:
resp = get(url, timeout = 30) resp = get(url, timeout = 30)
@ -1069,7 +1103,11 @@ def get_nodeinfo(server, host_meta_fallback = False):
log(f'nodeinfo for {server} not found, checking host-meta') log(f'nodeinfo for {server} not found, checking host-meta')
new_server = get_server_from_host_meta(server) new_server = get_server_from_host_meta(server)
if new_server is not None: if new_server is not None:
return get_nodeinfo(new_server, True) if new_server == server:
log(f'host-meta for {server} did not get a new server.')
return None
else:
return get_nodeinfo(new_server, seen_hosts, True)
else: else:
return None return None
@ -1085,23 +1123,29 @@ def get_nodeinfo(server, host_meta_fallback = False):
break break
except Exception as ex: except Exception as ex:
log(f'error getting server {server} info from well-known node info. Exception: {ex}') log(f'error getting server {server} info from well-known node info. Exception: {ex}')
return None
else: else:
log(f'Error getting well-known host node info for {server}. Status Code: {resp.status_code}') log(f'Error getting well-known host node info for {server}. Status Code: {resp.status_code}')
return None return None
if nodeLoc is None: if nodeLoc is None:
log(f'could not find link to node info in well-known nodeinfo of {server}') log(f'could not find link to node info in well-known nodeinfo of {server}')
return None
# regrab server from nodeLoc, again in the case of different display and web domains # regrab server from nodeLoc, again in the case of different display and web domains
try:
match = re.match( match = re.match(
r"https://(?P<server>[^/]+)/", nodeLoc r"https://(?P<server>[^/]+)/", nodeLoc
) )
server = match.group('server') if match is None:
except Exception as ex: log(f"Error getting web server name from {server}.")
log(f"Error getting web server name from {server}. Exception: {ex}")
return None return None
server = match.group('server')
# return early if the web domain has been seen previously (in cases with host-meta lookups)
if server in seen_hosts:
return seen_hosts[server]
try: try:
resp = get(nodeLoc, timeout = 30) resp = get(nodeLoc, timeout = 30)
except Exception as ex: except Exception as ex:
@ -1118,40 +1162,46 @@ def get_nodeinfo(server, host_meta_fallback = False):
'webserver': server, 'webserver': server,
'software': nodeInfo['software']['name'], 'software': nodeInfo['software']['name'],
'version': nodeInfo['software']['version'], 'version': nodeInfo['software']['version'],
'rawnodeinfo': nodeInfo,
} }
except Exception as ex: except Exception as ex:
log(f'error getting server {server} info from nodeinfo. Exception: {ex}') log(f'error getting server {server} info from nodeinfo. Exception: {ex}')
return None
else: else:
log(f'Error getting host node info for {server}. Status Code: {resp.status_code}') log(f'Error getting host node info for {server}. Status Code: {resp.status_code}')
return None return None
return None
def get_server_info(server, seen_hosts): def get_server_info(server, seen_hosts):
if server in seen_hosts: if server in seen_hosts:
return seen_hosts[server] return seen_hosts[server]
nodeinfo = get_nodeinfo(server) nodeinfo = get_nodeinfo(server, seen_hosts)
if nodeinfo is None: if nodeinfo is None:
seen_hosts[server] = None seen_hosts[server] = None
else: else:
if nodeinfo['software'] in ['misskey', 'calckey', 'firefish']: set_server_apis(nodeinfo)
nodeinfo['supportsMisskeyApi'] = True
else:
nodeinfo['supportsMisskeyApi'] = False
seen_hosts[server] = nodeinfo seen_hosts[server] = nodeinfo
if server is not nodeinfo['webserver']: if server is not nodeinfo['webserver']:
seen_hosts[nodeinfo['webserver']] = nodeinfo seen_hosts[nodeinfo['webserver']] = nodeinfo
return nodeinfo return nodeinfo
# arguments = argparser.parse_args() def set_server_apis(server):
# seen_hosts = {} # support for new server software should be added here
# test = get_server_info('mastodon.social', seen_hosts) software_apis = {
# log(f'test: {test}') 'mastodonApiSupport': ['mastodon', 'pleroma', 'akkoma', 'pixelfed', 'gotosocial', 'hometown'],
# log(f'seen_hosts: {seen_hosts}') 'misskeyApiSupport': ['misskey', 'calckey', 'firefish', 'foundkey'],
# exit() 'lemmyApiSupport': ['lemmy']
}
for api, softwareList in software_apis.items():
server[api] = server['software'] in softwareList
# search `features` list in metadata if available
if 'metadata' in server['rawnodeinfo'] and 'features' in server['rawnodeinfo']['metadata'] and type(server['rawnodeinfo']['metadata']['features']) is list:
features = server['rawnodeinfo']['metadata']['features']
if 'mastodon_api' in features:
server['mastodonApiSupport'] = True
if __name__ == "__main__": if __name__ == "__main__":
start = datetime.now() start = datetime.now()
@ -1271,6 +1321,9 @@ if __name__ == "__main__":
all_known_users = OrderedSet(list(known_followings) + list(recently_checked_users)) all_known_users = OrderedSet(list(known_followings) + list(recently_checked_users))
# NOTE: explicitly not cached in a file so we get server version upgrades or migrations to new software
seen_hosts = {}
if(isinstance(arguments.access_token, str)): if(isinstance(arguments.access_token, str)):
setattr(arguments, 'access_token', [arguments.access_token]) setattr(arguments, 'access_token', [arguments.access_token])
@ -1283,19 +1336,19 @@ if __name__ == "__main__":
reply_toots = get_all_reply_toots( reply_toots = get_all_reply_toots(
arguments.server, user_ids, token, seen_urls, arguments.reply_interval_in_hours arguments.server, user_ids, token, seen_urls, arguments.reply_interval_in_hours
) )
known_context_urls = get_all_known_context_urls(arguments.server, reply_toots,parsed_urls) known_context_urls = get_all_known_context_urls(arguments.server, reply_toots,parsed_urls, seen_hosts)
seen_urls.update(known_context_urls) seen_urls.update(known_context_urls)
replied_toot_ids = get_all_replied_toot_server_ids( replied_toot_ids = get_all_replied_toot_server_ids(
arguments.server, reply_toots, replied_toot_server_ids, parsed_urls arguments.server, reply_toots, replied_toot_server_ids, parsed_urls
) )
context_urls = get_all_context_urls(arguments.server, replied_toot_ids) context_urls = get_all_context_urls(arguments.server, replied_toot_ids, seen_hosts)
add_context_urls(arguments.server, token, context_urls, seen_urls) add_context_urls(arguments.server, token, context_urls, seen_urls)
if arguments.home_timeline_length > 0: if arguments.home_timeline_length > 0:
"""Do the same with any toots on the key owner's home timeline """ """Do the same with any toots on the key owner's home timeline """
timeline_toots = get_timeline(arguments.server, token, arguments.home_timeline_length) timeline_toots = get_timeline(arguments.server, token, arguments.home_timeline_length)
known_context_urls = get_all_known_context_urls(arguments.server, timeline_toots,parsed_urls) known_context_urls = get_all_known_context_urls(arguments.server, timeline_toots,parsed_urls, seen_hosts)
add_context_urls(arguments.server, token, known_context_urls, seen_urls) add_context_urls(arguments.server, token, known_context_urls, seen_urls)
# Backfill any post authors, and any mentioned users # Backfill any post authors, and any mentioned users
@ -1317,40 +1370,40 @@ if __name__ == "__main__":
if user not in mentioned_users and user['acct'] not in all_known_users: if user not in mentioned_users and user['acct'] not in all_known_users:
mentioned_users.append(user) mentioned_users.append(user)
add_user_posts(arguments.server, token, filter_known_users(mentioned_users, all_known_users), recently_checked_users, all_known_users, seen_urls) add_user_posts(arguments.server, token, filter_known_users(mentioned_users, all_known_users), recently_checked_users, all_known_users, seen_urls, seen_hosts)
if arguments.max_followings > 0: if arguments.max_followings > 0:
log(f"Getting posts from last {arguments.max_followings} followings") log(f"Getting posts from last {arguments.max_followings} followings")
user_id = get_user_id(arguments.server, arguments.user, token) user_id = get_user_id(arguments.server, arguments.user, token)
followings = get_new_followings(arguments.server, user_id, arguments.max_followings, all_known_users) followings = get_new_followings(arguments.server, user_id, arguments.max_followings, all_known_users)
add_user_posts(arguments.server, token, followings, known_followings, all_known_users, seen_urls) add_user_posts(arguments.server, token, followings, known_followings, all_known_users, seen_urls, seen_hosts)
if arguments.max_followers > 0: if arguments.max_followers > 0:
log(f"Getting posts from last {arguments.max_followers} followers") log(f"Getting posts from last {arguments.max_followers} followers")
user_id = get_user_id(arguments.server, arguments.user, token) user_id = get_user_id(arguments.server, arguments.user, token)
followers = get_new_followers(arguments.server, user_id, arguments.max_followers, all_known_users) followers = get_new_followers(arguments.server, user_id, arguments.max_followers, all_known_users)
add_user_posts(arguments.server, token, followers, recently_checked_users, all_known_users, seen_urls) add_user_posts(arguments.server, token, followers, recently_checked_users, all_known_users, seen_urls, seen_hosts)
if arguments.max_follow_requests > 0: if arguments.max_follow_requests > 0:
log(f"Getting posts from last {arguments.max_follow_requests} follow requests") log(f"Getting posts from last {arguments.max_follow_requests} follow requests")
follow_requests = get_new_follow_requests(arguments.server, token, arguments.max_follow_requests, all_known_users) follow_requests = get_new_follow_requests(arguments.server, token, arguments.max_follow_requests, all_known_users)
add_user_posts(arguments.server, token, follow_requests, recently_checked_users, all_known_users, seen_urls) add_user_posts(arguments.server, token, follow_requests, recently_checked_users, all_known_users, seen_urls, seen_hosts)
if arguments.from_notifications > 0: if arguments.from_notifications > 0:
log(f"Getting notifications for last {arguments.from_notifications} hours") log(f"Getting notifications for last {arguments.from_notifications} hours")
notification_users = get_notification_users(arguments.server, token, all_known_users, arguments.from_notifications) notification_users = get_notification_users(arguments.server, token, all_known_users, arguments.from_notifications)
add_user_posts(arguments.server, token, notification_users, recently_checked_users, all_known_users, seen_urls) add_user_posts(arguments.server, token, notification_users, recently_checked_users, all_known_users, seen_urls, seen_hosts)
if arguments.max_bookmarks > 0: if arguments.max_bookmarks > 0:
log(f"Pulling replies to the last {arguments.max_bookmarks} bookmarks") log(f"Pulling replies to the last {arguments.max_bookmarks} bookmarks")
bookmarks = get_bookmarks(arguments.server, token, arguments.max_bookmarks) bookmarks = get_bookmarks(arguments.server, token, arguments.max_bookmarks)
known_context_urls = get_all_known_context_urls(arguments.server, bookmarks,parsed_urls) known_context_urls = get_all_known_context_urls(arguments.server, bookmarks,parsed_urls, seen_hosts)
add_context_urls(arguments.server, token, known_context_urls, seen_urls) add_context_urls(arguments.server, token, known_context_urls, seen_urls)
if arguments.max_favourites > 0: if arguments.max_favourites > 0:
log(f"Pulling replies to the last {arguments.max_favourites} favourites") log(f"Pulling replies to the last {arguments.max_favourites} favourites")
favourites = get_favourites(arguments.server, token, arguments.max_favourites) favourites = get_favourites(arguments.server, token, arguments.max_favourites)
known_context_urls = get_all_known_context_urls(arguments.server, favourites,parsed_urls) known_context_urls = get_all_known_context_urls(arguments.server, favourites,parsed_urls, seen_hosts)
add_context_urls(arguments.server, token, known_context_urls, seen_urls) add_context_urls(arguments.server, token, known_context_urls, seen_urls)
with open(KNOWN_FOLLOWINGS_FILE, "w", encoding="utf-8") as f: with open(KNOWN_FOLLOWINGS_FILE, "w", encoding="utf-8") as f: