VladPolskiy/postgres-web
1
0
Fork 0
mirror of https://github.com/postgres/pgweb.git synced 2025-07-29 11:59:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Don't generate http requests with two Host: headers

Browse Source 
This would cause http 400 requests at the server, unsurprisingly.
This commit is contained in:
Magnus Hagander
2012-02-07 13:05:41 +01:00
parent 03794e3157
commit f9486b54f1
1 changed files with 6 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
tools/search/crawler/lib/basecrawler.py
View File

@ -164,14 +164,14 @@ class BaseSiteCrawler(object):
try:
# Unfortunatley, persistent connections seem quite unreliable,
# so create a new one for each page.
h = httplib.HTTPConnection(host=self.serverip and self.serverip or self.hostname,
port=80,
strict=True,
timeout=10)
h.putrequest("GET", url)
h.putheader("User-agent","pgsearch/0.2")
if self.serverip:
h = httplib.HTTPConnection(host=self.serverip, port=80, strict=True, timeout=10)
h.putrequest("GET", url, skip_host=1)
h.putheader("Host", self.hostname)
else:
h = httplib.HTTPConnection(host=self.hostname, port=80, strict=True, timeout=10)
h.putrequest("GET", url)
h.putheader("User-agent","pgsearch/0.2")
h.putheader("Connection","close")
if self.scantimes.has_key(url):
h.putheader("If-Modified-Since", formatdate(time.mktime(self.scantimes[url].timetuple())))