Support for fetching lists

README.md

@@ -5,13 +5,15 @@ This GitHub repository provides a simple script that can pull missing posts into
 1. It can pull missing remote replies to posts that are already on your server into your server. Specifically, it can
    1. fetch missing replies to posts that users on your instance have already replied to,
    2. fetch missing replies to the most recent posts in your home timeline,
-   3. fetch missing replies to your bookmarks.
-   4. fetch missing replies to your favourites.
+   3. fetch missing replies to your bookmarks,
+   4. fetch missing replies to your favourites,
+   5. fetch missing replies to the most recent posts in your lists.
 2. It can also backfill profiles on your instance. In particular it can
    1. fetch missing posts from users that have recently appeared in your notifications,
-   1. fetch missing posts from users that you have recently followed,
-   2. fetch missing posts form users that have recently followed you,
-   3. fetch missing posts form users that have recently sent you a follow request.
+   2. fetch missing posts from users that you have recently followed,
+   3. fetch missing posts from users that have recently followed you,
+   4. fetch missing posts from users that have recently sent you a follow request,
+   5. fetch missing posts from users that have recently been added to your lists.
 
 Each part of this script is fully configurable, and you can completely disable parts that you are not interested in.
 
@@ -142,6 +144,9 @@ Option | Required? | Notes |
 | `reply-interval-in-hours` | No | Provide to fetch remote replies to posts that have received replies from users on your own instance. Determines how far back in time we'll go to find posts that have received replies. You must be administrator on your instance to use this option, and this option is not supported on Pleroma / Akkoma and its forks. Recommend value: `0` (disabled). Requires an access token with `admin:read:accounts`.
 |`backfill-with-context` | No | Set to `0` to disable fetching remote replies while backfilling profiles. This is enabled by default, but you can disable it, if it's too slow for you.
 |`backfill-mentioned-users` | No | Set to `0` to disable backfilling any mentioned users when fetching the home timeline. This is enabled by default, but you can disable it, if it's too slow for you.
+| `from-lists`| No | Set to `1` to fetch missing replies from your lists. This is disabled by default.
+| `max-list-length` | No | Determines how many posts we'll fetch replies for in each list. Default value: `200`. This will be ignored, unless you also provide `from-lists = 1`. Set to `0` if you only want to backfill profiles in lists. |
+| `max-list-accounts` | No | Determines how users we'll backfill for in each list. Default value: `10`. This will be ignored, unless you also provide `from-lists = 1`. Set to `0` if you only want to fetch replies in lists. |
 | `remember-users-for-hours` | No | How long between back-filling attempts for non-followed accounts? Defaults to `168`, i.e. one week.
 | `remember-hosts-for-days` | No | How long should FediFetcher cache host info for? Defaults to `30`.
 | `http-timeout` | No | The timeout for any HTTP requests to the Mastodon API in seconds. Defaults to `5`.
@@ -174,6 +179,8 @@ This is only supported when running FediFetcher as cron job, or container. Multi
    - `read:favourites`
  - If you are supplying `from-notifications` you must additionally enable this scope:
    - `read:notifications`
+ - If you are supplying `from-lists` you must additionally enable this scope:
+   - `read:lists`
 
 ## Acknowledgments
 

find_posts.py

@@ -48,6 +48,9 @@ argparser.add_argument('--state-dir', required = False, default="artifacts", hel
 argparser.add_argument('--on-done', required = False, default=None, help="Provide a url that will be pinged when processing has completed. You can use this for 'dead man switch' monitoring of your task")
 argparser.add_argument('--on-start', required = False, default=None, help="Provide a url that will be pinged when processing is starting. You can use this for 'dead man switch' monitoring of your task")
 argparser.add_argument('--on-fail', required = False, default=None, help="Provide a url that will be pinged when processing has failed. You can use this for 'dead man switch' monitoring of your task")
+argparser.add_argument('--from-lists', required=False, type=bool, default=False, help="Set to 1 to backfill lists")
+argparser.add_argument('--max-list-length', required=False, type=int, default=100, help="Determine how many posts in your list to get context for")
+argparser.add_argument('--max-list-accounts', required=False, type=int, default=10, help="Determine how many users to backfill in your list")
 
 def get_notification_users(server, access_token, known_users, max_age):
     since = datetime.now(datetime.now().astimezone().tzinfo) - timedelta(hours=max_age)
@@ -331,7 +334,6 @@ def get_user_id(server, user = None, access_token = None):
             f"Error getting URL {url}. Status code: {response.status_code}"
         )
 
-
 def get_timeline(server, access_token, max):
     """Get all post in the user's home timeline"""
 
@@ -1331,6 +1333,33 @@ def set_server_apis(server):
 
     server['last_checked'] = datetime.now()
 
+def get_user_lists(server, token):
+    return get_paginated_mastodon(f"https://{server}/api/v1/lists", 99, {
+        "Authorization": f"Bearer {token}",
+    })
+
+def get_list_timeline(server, list, token, max):
+    """Get all post in the user's home timeline"""
+
+    url = f"https://{server}/api/v1/timelines/list/{list['id']}"
+
+    posts = get_paginated_mastodon(url, max, {
+        "Authorization": f"Bearer {token}",
+    })
+
+    logger.info(f"Found {len(posts)} toots in list {list['title']}")
+
+    return posts
+
+def get_list_users(server, list, token, max):
+    url = f"https://{server}/api/v1/lists/{list['id']}/accounts"
+    accounts = get_paginated_mastodon(url, max, {
+        "Authorization": f"Bearer {token}",
+    })
+    logger.info(f"Found {len(accounts)} accounts in list {list['title']}")
+    return accounts
+
+
 if __name__ == "__main__":
     start = datetime.now()
 
@@ -1497,6 +1526,21 @@ if __name__ == "__main__":
 
         for token in arguments.access_token:
 
+            if arguments.from_lists:
+                """Pull replies from lists"""
+                lists = get_user_lists(arguments.server, token)
+                for list in lists:
+                    # Fill context from list
+                    if arguments.max_list_length > 0:
+                        timeline_toots = get_list_timeline(arguments.server, list, token, arguments.max_list_length)
+                        known_context_urls = get_all_known_context_urls(arguments.server, timeline_toots,parsed_urls, seen_hosts)
+                        add_context_urls(arguments.server, token, known_context_urls, seen_urls)
+
+                    # Backfill profiles from list
+                    if arguments.max_list_accounts:
+                        accounts = get_list_users(arguments.server, list, token, arguments.max_list_accounts)
+                        add_user_posts(arguments.server, token, accounts, recently_checked_users, all_known_users, seen_urls, seen_hosts)
+
             if arguments.reply_interval_in_hours > 0:
                 """pull the context toots of toots user replied to, from their
                 original server, and add them to the local server."""