itsulu/FediFetcher
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #129 from nanos/cache-robots

Browse source 
Cache robots.txt for each run of the script, to reduce load on the server
This commit is contained in:
Michael 2024-06-25 16:25:26 +01:00 committed by GitHub
commit dec718db76
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 19 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
find_posts.py
View file

@ -1009,14 +1009,23 @@ def can_fetch(user_agent, url):
parsed_uri = urlparse(url)
robots = '{uri.scheme}://{uri.netloc}/robots.txt'.format(uri=parsed_uri)
if robots in ROBOTS_TXT:
if isinstance(ROBOTS_TXT[robots], bool):
return ROBOTS_TXT[robots]
else:
robotsTxt = ROBOTS_TXT[robots]
else:
try:
# We are getting the robots.txt manually from here, because otherwise
# We are getting the robots.txt manually from here, because otherwise we can't change the User Agent
robotsTxt = get(robots, ignore_robots_txt=True)
if robotsTxt.status_code in (401, 403):
ROBOTS_TXT[robots] = False
return False
elif robotsTxt.status_code != 200:
ROBOTS_TXT[robots] = True
return True
robotsTxt = robotsTxt.text
ROBOTS_TXT[robots] = robotsTxt
except Exception as ex:
return True
@ -1394,6 +1403,7 @@ if __name__ == "__main__":
SEEN_HOSTS_FILE = os.path.join(arguments.state_dir, "seen_hosts")
RECENTLY_CHECKED_CONTEXTS_FILE = os.path.join(arguments.state_dir, 'recent_context')
ROBOTS_TXT = {}
seen_urls = OrderedSet([])
if os.path.exists(SEEN_URLS_FILE):