itsulu/FediFetcher
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

ensure callbacks aren't blocked by robtos

Browse source
This commit is contained in:
nanos 2024-06-25 10:38:47 +01:00
parent 1b4c135f8f
commit 885b84d598
1 changed files with 7 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
find_posts.py
View file

@ -1016,13 +1016,13 @@ def can_fetch(user_agent, url):
def user_agent():
return f"FediFetcher/{VERSION}; +{arguments.server} (https://go.thms.uk/ff)"
def get(url, headers = {}, timeout = 0, max_tries = 5):
def get(url, headers = {}, timeout = 0, max_tries = 5, ignore_robots_txt = False):
"""A simple wrapper to make a get request while providing our user agent, and respecting rate limits"""
h = headers.copy()
if 'User-Agent' not in h:
h['User-Agent'] = user_agent()
if not can_fetch(h['User-Agent'], url):
if not ignore_robots_txt and not can_fetch(h['User-Agent'], url):
raise Exception(f"Querying {url} prohibited by robots.txt")
if timeout == 0:
@ -1334,7 +1334,7 @@ if __name__ == "__main__":
if(arguments.on_start != None and arguments.on_start != ''):
try:
get(f"{arguments.on_start}?rid={runId}")
get(f"{arguments.on_start}?rid={runId}", ignore_robots_txt = True)
except Exception as ex:
logger.error(f"Error getting callback url: {ex}")
@ -1356,7 +1356,7 @@ if __name__ == "__main__":
logger.critical(f"Lock file age is {datetime.now() - lock_time} - below --lock-hours={arguments.lock_hours} provided.")
if(arguments.on_fail != None and arguments.on_fail != ''):
try:
get(f"{arguments.on_fail}?rid={runId}")
get(f"{arguments.on_fail}?rid={runId}", ignore_robots_txt = True)
except Exception as ex:
logger.error(f"Error getting callback url: {ex}")
sys.exit(1)
@ -1365,7 +1365,7 @@ if __name__ == "__main__":
logger.critical(f"Cannot read logfile age - aborting.")
if(arguments.on_fail != None and arguments.on_fail != ''):
try:
get(f"{arguments.on_fail}?rid={runId}")
get(f"{arguments.on_fail}?rid={runId}", ignore_robots_txt = True)
except Exception as ex:
logger.error(f"Error getting callback url: {ex}")
sys.exit(1)
@ -1549,7 +1549,7 @@ if __name__ == "__main__":
if(arguments.on_done != None and arguments.on_done != ''):
try:
get(f"{arguments.on_done}?rid={runId}")
get(f"{arguments.on_done}?rid={runId}", ignore_robots_txt = True)
except Exception as ex:
logger.error(f"Error getting callback url: {ex}")
@ -1560,7 +1560,7 @@ if __name__ == "__main__":
logger.error(f"Job failed after {datetime.now() - start}.")
if(arguments.on_fail != None and arguments.on_fail != ''):
try:
get(f"{arguments.on_fail}?rid={runId}")
get(f"{arguments.on_fail}?rid={runId}", ignore_robots_txt = True)
except Exception as ex:
logger.error(f"Error getting callback url: {ex}")
raise