qbi/FediFetcher
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Use file based locking when running script locally

Browse source 
Fixes #18
This commit is contained in:
nanos 2023-03-20 21:14:08 +00:00
parent 5be011b8d9
commit 62989f4041
1 changed files with 52 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

87
find_posts.py
View file

@ -683,47 +683,64 @@ class OrderedSet:
if __name__ == "__main__": if __name__ == "__main__":
SEEN_URLS_FILE = "artifacts/seen_urls" LOCK_FILE = 'artifacts/lock.lock'
REPLIED_TOOT_SERVER_IDS_FILE = "artifacts/replied_toot_server_ids"
KNOWN_FOLLOWINGS_FILE = "artifacts/known_followings" if( os.path.exists(LOCK_FILE)):
log(f"Lock file exists at {LOCK_FILE}")
sys.exit(1)
with open(LOCK_FILE, "w", encoding="utf-8") as f:
f.write(f"{datetime.now()}")
try:
SEEN_URLS_FILE = "artifacts/seen_urls"
REPLIED_TOOT_SERVER_IDS_FILE = "artifacts/replied_toot_server_ids"
KNOWN_FOLLOWINGS_FILE = "artifacts/known_followings"
SEEN_URLS = OrderedSet([]) SEEN_URLS = OrderedSet([])
if os.path.exists(SEEN_URLS_FILE): if os.path.exists(SEEN_URLS_FILE):
with open(SEEN_URLS_FILE, "r", encoding="utf-8") as f: with open(SEEN_URLS_FILE, "r", encoding="utf-8") as f:
SEEN_URLS = OrderedSet(f.read().splitlines()) SEEN_URLS = OrderedSet(f.read().splitlines())
REPLIED_TOOT_SERVER_IDS = {} REPLIED_TOOT_SERVER_IDS = {}
if os.path.exists(REPLIED_TOOT_SERVER_IDS_FILE): if os.path.exists(REPLIED_TOOT_SERVER_IDS_FILE):
with open(REPLIED_TOOT_SERVER_IDS_FILE, "r", encoding="utf-8") as f: with open(REPLIED_TOOT_SERVER_IDS_FILE, "r", encoding="utf-8") as f:
REPLIED_TOOT_SERVER_IDS = json.load(f) REPLIED_TOOT_SERVER_IDS = json.load(f)
KNOWN_FOLLOWINGS = OrderedSet([]) KNOWN_FOLLOWINGS = OrderedSet([])
if os.path.exists(KNOWN_FOLLOWINGS_FILE): if os.path.exists(KNOWN_FOLLOWINGS_FILE):
with open(KNOWN_FOLLOWINGS_FILE, "r", encoding="utf-8") as f: with open(KNOWN_FOLLOWINGS_FILE, "r", encoding="utf-8") as f:
KNOWN_FOLLOWINGS = OrderedSet(f.read().splitlines()) KNOWN_FOLLOWINGS = OrderedSet(f.read().splitlines())
arguments = argparser.parse_args() arguments = argparser.parse_args()
pull_context( pull_context(
arguments.server, arguments.server,
arguments.access_token, arguments.access_token,
SEEN_URLS, SEEN_URLS,
REPLIED_TOOT_SERVER_IDS, REPLIED_TOOT_SERVER_IDS,
arguments.reply_interval_in_hours, arguments.reply_interval_in_hours,
arguments.home_timeline_length, arguments.home_timeline_length,
arguments.max_followings, arguments.max_followings,
arguments.user, arguments.user,
KNOWN_FOLLOWINGS, KNOWN_FOLLOWINGS,
arguments.max_followers, arguments.max_followers,
arguments.max_follow_requests arguments.max_follow_requests
) )
with open(KNOWN_FOLLOWINGS_FILE, "w", encoding="utf-8") as f: with open(KNOWN_FOLLOWINGS_FILE, "w", encoding="utf-8") as f:
f.write("\n".join(list(KNOWN_FOLLOWINGS)[-10000:])) f.write("\n".join(list(KNOWN_FOLLOWINGS)[-10000:]))
with open(SEEN_URLS_FILE, "w", encoding="utf-8") as f: with open(SEEN_URLS_FILE, "w", encoding="utf-8") as f:
f.write("\n".join(list(SEEN_URLS)[-10000:])) f.write("\n".join(list(SEEN_URLS)[-10000:]))
with open(REPLIED_TOOT_SERVER_IDS_FILE, "w", encoding="utf-8") as f: with open(REPLIED_TOOT_SERVER_IDS_FILE, "w", encoding="utf-8") as f:
json.dump(dict(list(REPLIED_TOOT_SERVER_IDS.items())[-10000:]), f) json.dump(dict(list(REPLIED_TOOT_SERVER_IDS.items())[-10000:]), f)
os.remove(LOCK_FILE)
except Exception as ex:
os.remove(LOCK_FILE)
raise