From a624f1ae58fdbde6e12ce64aaa0201948487cf4c Mon Sep 17 00:00:00 2001 From: Michael Thomas Date: Fri, 17 Mar 2023 08:26:38 +0000 Subject: [PATCH] Allow us to make the HTTP timeout configurable --- .github/workflows/get_context.yml | 2 +- README.md | 1 + find_posts.py | 6 +++++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/get_context.yml b/.github/workflows/get_context.yml index 7e82f59..2ebf795 100644 --- a/.github/workflows/get_context.yml +++ b/.github/workflows/get_context.yml @@ -32,7 +32,7 @@ jobs: path: artifacts - name: Get Directory structure run: ls -lR - - run: python find_posts.py --access-token=${{ secrets.ACCESS_TOKEN }} --server=${{ vars.MASTODON_SERVER }} --reply-interval-in-hours=${{ vars.REPLY_INTERVAL_IN_HOURS || 0 }} --home-timeline-length=${{ vars.HOME_TIMELINE_LENGTH || 0 }} --max-followings=${{ vars.MAX_FOLLOWINGS || 0 }} --user=${{ vars.USER }} --max-followers=${{ vars.MAX_FOLLOWERS || 0 }} + - run: python find_posts.py --access-token=${{ secrets.ACCESS_TOKEN }} --server=${{ vars.MASTODON_SERVER }} --reply-interval-in-hours=${{ vars.REPLY_INTERVAL_IN_HOURS || 0 }} --home-timeline-length=${{ vars.HOME_TIMELINE_LENGTH || 0 }} --max-followings=${{ vars.MAX_FOLLOWINGS || 0 }} --user=${{ vars.USER }} --max-followers=${{ vars.MAX_FOLLOWERS || 0 }} --http-timeout=${{ vars.HTTP_TIMEOUT || 0 }} - name: Upload artifacts uses: actions/upload-artifact@v3 with: diff --git a/README.md b/README.md index 11d4c3d..4909358 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,7 @@ Please see below for a list of configuration options. | `MAX_FOLLOWINGS` | `--max-followings` | No | Provide to backfill profiles for your most recent followings. Determines how many of your last followings you want to backfill. (An integer number, e.g. `80`. Ensure you also provide `USER`). | `MAX_FOLLOWERS` | `--max-followers` | No | Provide to backfill profiles for your most recent followings. Determines how many of your last followers you want to backfill. (An integer number, e.g. `80`. Ensure you also provide `USER`). | `USER` | `--user` | See Notes | Required together with `MAX_FOLLOWERS` or `MAX_FOLLOWINGS`: The username of the user whose followers or followings you want to backfill (e.g. `michael` for the user `@michael@thms.uk`). +| `HTTP_TIMEOUT` | `--http-timeout` | No | The timeout for any HTTP requests to the Mastodon API in seconds. Defaults to `5`. ## Acknowledgments diff --git a/find_posts.py b/find_posts.py index 2fd55e6..6fe44c0 100644 --- a/find_posts.py +++ b/find_posts.py @@ -20,6 +20,7 @@ parser.add_argument('--home-timeline-length', required = False, type=int, defaul parser.add_argument('--user', required = False, default='', help="Use together with --max-followings or --max-followers to tell us which user's followings/followers we should backfill") parser.add_argument('--max-followings', required = False, type=int, default=0, help="Backfill posts for new accounts followed by --user. We'll backfill at most this many followings' posts") parser.add_argument('--max-followers', required = False, type=int, default=0, help="Backfill posts for new accounts following --user. We'll backfill at most this many followers' posts") +parser.add_argument('--http-timeout', required = False, type=int, default=5, help="The timeout for any HTTP requests to your own, or other instances.") def pull_context( server, @@ -587,11 +588,14 @@ def add_context_url(url, server, access_token): ) return False -def get(url, headers = {}, timeout = 5, max_tries = 5): +def get(url, headers = {}, timeout = 0, max_tries = 5): """A simple wrapper to make a get request while providing our user agent, and respecting rate limits""" h = headers.copy() if 'User-Agent' not in h: h['User-Agent'] = 'mastodon_get_replies (https://go.thms.uk/mgr)' + + if timeout == 0: + timeout = arguments.http_timeout response = requests.get( url, headers= h, timeout=timeout) if response.status_code == 429: