Browse Source

more exception handling on link update

master
René Wagner 2 weeks ago
parent
commit
d3b1dd8e77
  1. 6
      gus/crawl.py

6
gus/crawl.py

@ -268,8 +268,10 @@ def index_links(from_resource, contained_resources):
"is_cross_host_like": Link.get_is_cross_host_like(from_resource, cr),
}
)
Link.insert_many(data).execute()
try:
Link.insert_many(data).execute()
except Exception as e:
logging.error("Error insert links: %s",e)
def fetch_robots_file(robot_host):
robot_url = urljoin("gemini://{}".format(robot_host), "/robots.txt")

Loading…
Cancel
Save