itsulu/FediFetcher
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #164 from AndrewKvalheim/trivial

Browse source 
Remove unused code, correct spelling
This commit is contained in:
Michael 2024-09-03 16:16:11 +01:00 committed by GitHub
commit d6066f34c8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 65 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
find_posts.py
View file

@ -1,7 +1,6 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
from datetime import datetime, timedelta from datetime import datetime, timedelta
import string
from dateutil import parser from dateutil import parser
import itertools import itertools
import json import json
@ -53,7 +52,7 @@ argparser.add_argument('--max-list-length', required=False, type=int, default=10
argparser.add_argument('--max-list-accounts', required=False, type=int, default=10, help="Determines how many accounts we'll backfill for in each list. This will be ignored, unless you also provide `from-lists = 1`. Set to `0` if you only want to fetch replies in lists.") argparser.add_argument('--max-list-accounts', required=False, type=int, default=10, help="Determines how many accounts we'll backfill for in each list. This will be ignored, unless you also provide `from-lists = 1`. Set to `0` if you only want to fetch replies in lists.")
argparser.add_argument('--log-level', required=False, default="DEBUG", help="Severity of events to log (DEBUG|INFO|WARNING|ERROR|CRITICAL)") argparser.add_argument('--log-level', required=False, default="DEBUG", help="Severity of events to log (DEBUG|INFO|WARNING|ERROR|CRITICAL)")
argparser.add_argument('--log-format', required=False, type=str, default="%(asctime)s: %(message)s",help="Specify the log format") argparser.add_argument('--log-format', required=False, type=str, default="%(asctime)s: %(message)s",help="Specify the log format")
argparser.add_argument('--instance-blocklist', required=False, type=str, default="",help="A comma-seperated array of instances that FediFetcher should never try to connect to") argparser.add_argument('--instance-blocklist', required=False, type=str, default="",help="A comma-separated array of instances that FediFetcher should never try to connect to")
def get_notification_users(server, access_token, known_users, max_age): def get_notification_users(server, access_token, known_users, max_age):
since = datetime.now(datetime.now().astimezone().tzinfo) - timedelta(hours=max_age) since = datetime.now(datetime.now().astimezone().tzinfo) - timedelta(hours=max_age)
@ -1110,7 +1109,7 @@ def get_robots_from_url(robots_url):
with open(get_robots_txt_cache_path(robots_url), "w", encoding="utf-8") as f: with open(get_robots_txt_cache_path(robots_url), "w", encoding="utf-8") as f:
f.write(robotsTxt) f.write(robotsTxt)
except Exception as ex: except Exception:
robotsTxt = True robotsTxt = True
ROBOTS_TXT[robots_url] = robotsTxt ROBOTS_TXT[robots_url] = robotsTxt
@ -1516,7 +1515,7 @@ if __name__ == "__main__":
if tokens := [token for envvar, token in os.environ.items() if envvar.lower().startswith("ff_access_token")]: if tokens := [token for envvar, token in os.environ.items() if envvar.lower().startswith("ff_access_token")]:
arguments.access_token = tokens arguments.access_token = tokens
logger.info(f"Starting FediFetcher") logger.info("Starting FediFetcher")
if(arguments.server == None or arguments.access_token == None): if(arguments.server == None or arguments.access_token == None):
logger.critical("You must supply at least a server name and an access token") logger.critical("You must supply at least a server name and an access token")
@ -1547,7 +1546,7 @@ if __name__ == "__main__":
if (datetime.now() - lock_time).total_seconds() >= arguments.lock_hours * 60 * 60: if (datetime.now() - lock_time).total_seconds() >= arguments.lock_hours * 60 * 60:
os.remove(LOCK_FILE) os.remove(LOCK_FILE)
logger.debug(f"Lock file has expired. Removed lock file.") logger.debug("Lock file has expired. Removed lock file.")
else: else:
logger.critical(f"Lock file age is {datetime.now() - lock_time} - below --lock-hours={arguments.lock_hours} provided.") logger.critical(f"Lock file age is {datetime.now() - lock_time} - below --lock-hours={arguments.lock_hours} provided.")
if(arguments.on_fail != None and arguments.on_fail != ''): if(arguments.on_fail != None and arguments.on_fail != ''):
@ -1558,7 +1557,7 @@ if __name__ == "__main__":
sys.exit(1) sys.exit(1)
except Exception: except Exception:
logger.critical(f"Cannot read logfile age - aborting.") logger.critical("Cannot read logfile age - aborting.")
if(arguments.on_fail != None and arguments.on_fail != ''): if(arguments.on_fail != None and arguments.on_fail != ''):
try: try:
get(f"{arguments.on_fail}?rid={runId}", ignore_robots_txt = True) get(f"{arguments.on_fail}?rid={runId}", ignore_robots_txt = True)
@ -1692,7 +1691,7 @@ if __name__ == "__main__":
if arguments.home_timeline_length > 0: if arguments.home_timeline_length > 0:
"""Do the same with any toots on the key owner's home timeline """ """Do the same with any toots on the key owner's home timeline """
logger.info(f"Getting context for home timeline") logger.info("Getting context for home timeline")
timeline_toots = get_timeline(arguments.server, token, arguments.home_timeline_length) timeline_toots = get_timeline(arguments.server, token, arguments.home_timeline_length)
fetch_timeline_context(timeline_toots, token, parsed_urls, seen_hosts, seen_urls, all_known_users, recently_checked_users) fetch_timeline_context(timeline_toots, token, parsed_urls, seen_hosts, seen_urls, all_known_users, recently_checked_users)
@ -1758,7 +1757,7 @@ if __name__ == "__main__":
logger.info(f"Processing finished in {datetime.now() - start}.") logger.info(f"Processing finished in {datetime.now() - start}.")
except Exception as ex: except Exception:
os.remove(LOCK_FILE) os.remove(LOCK_FILE)
logger.error(f"Job failed after {datetime.now() - start}.") logger.error(f"Job failed after {datetime.now() - start}.")
if(arguments.on_fail != None and arguments.on_fail != ''): if(arguments.on_fail != None and arguments.on_fail != ''):