Browse Source

Merge branch 'master' of git://natpen.net/gus

master
René Wagner 7 months ago
parent
commit
6e54e52014
  1. 2
      gus/crawl.py

2
gus/crawl.py

@ -495,6 +495,8 @@ def crawl_page(
# RobotFileParser will return the higher level value (*) if
# no indexer section is found
can_fetch = robots_file.can_fetch("indexer", gr.normalized_url)
# same approach as above - last value wins
crawl_delay = robots_file.crawl_delay("indexer")
if not can_fetch:

Loading…
Cancel
Save