itsulu/FediFetcher
2
0
Fork 0
mirror of https://github.com/nicholasr-itsulu/FediFetcher.git synced 2026-05-30 23:41:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix

Browse source
This commit is contained in:
nanos 2024-06-26 16:45:30 +01:00
parent 7e8ca17640
commit 90988872b7
1 changed files with 2 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
find_posts.py
View file

@ -1018,7 +1018,7 @@ def can_fetch(user_agent, url):
robotsCachePath = os.path.join(arguments.state_dir, f'robots-{parsed_uri.netloc}')
if os.path.exists(robotsCachePath):
with open(robotsCachePath, "r", encoding="utf-8") as f:
logger.debug(f"Getting robots.text file from cache {file_name}")
logger.debug(f"Getting robots.txt file from cache {parsed_uri.netloc}")
robotsTxt = f.read()
ROBOTS_TXT[robots] = robotsTxt
@ -1484,7 +1484,7 @@ if __name__ == "__main__":
file_path = os.path.join(arguments.state_dir,file_name)
if file_name.startswith('robots-') and os.path.isfile(file_path):
if os.path.getmtime(file_path) < time.time() - 60 * 60 * 24:
logger.debug(f"Removing cached robots.text file {file_name}")
logger.debug(f"Removing cached robots.txt file {file_name}")
os.remove(file_path)