fix: strip whitespace if it's included in the list

This commit is contained in:
B Tasker 2024-08-18 17:03:20 +01:00
parent b7bd6a92f3
commit c9f6521a61
No known key found for this signature in database
GPG key ID: 8DC652174C1EBA9B

View file

@ -1578,7 +1578,7 @@ if __name__ == "__main__":
SEEN_HOSTS_FILE = os.path.join(arguments.state_dir, "seen_hosts") SEEN_HOSTS_FILE = os.path.join(arguments.state_dir, "seen_hosts")
RECENTLY_CHECKED_CONTEXTS_FILE = os.path.join(arguments.state_dir, 'recent_context') RECENTLY_CHECKED_CONTEXTS_FILE = os.path.join(arguments.state_dir, 'recent_context')
INSTANCE_BLOCKLIST = arguments.instance_blocklist.split(",") INSTANCE_BLOCKLIST = [x.strip() for x in arguments.instance_blocklist.split(",")]
ROBOTS_TXT = {} ROBOTS_TXT = {}
seen_urls = OrderedSet([]) seen_urls = OrderedSet([])