diff --git a/tools/search/crawler/lib/genericsite.py b/tools/search/crawler/lib/genericsite.py index 7e46ae5b..b74bb469 100644 --- a/tools/search/crawler/lib/genericsite.py +++ b/tools/search/crawler/lib/genericsite.py @@ -31,6 +31,8 @@ class GenericSiteCrawler(BaseSiteCrawler): self.queue.put((x, 0.5, False)) def exclude_url(self, url): + if ".." in url: + return True if self.robots and self.robots.block_url(url): return True for r in self.extra_excludes: