fix: strip whitespace if it's included in the list
This commit is contained in:
parent
b7bd6a92f3
commit
c9f6521a61
1 changed files with 1 additions and 1 deletions
|
|
@ -1578,7 +1578,7 @@ if __name__ == "__main__":
|
|||
SEEN_HOSTS_FILE = os.path.join(arguments.state_dir, "seen_hosts")
|
||||
RECENTLY_CHECKED_CONTEXTS_FILE = os.path.join(arguments.state_dir, 'recent_context')
|
||||
|
||||
INSTANCE_BLOCKLIST = arguments.instance_blocklist.split(",")
|
||||
INSTANCE_BLOCKLIST = [x.strip() for x in arguments.instance_blocklist.split(",")]
|
||||
ROBOTS_TXT = {}
|
||||
|
||||
seen_urls = OrderedSet([])
|
||||
|
|
|
|||
Loading…
Reference in a new issue