mirror of
https://github.com/postgres/pgweb.git
synced 2025-08-03 15:38:59 +00:00
6e89d2fb66
It seems the newer tidy in Debian Stretch breaks with the output from the old docs toolchain, causing indention to happen inside <pre> blocks which clearly breaks rendering. Turn it off for thos, but keep it enabled for version 11 and up (at this point that's just dev), because the output becomes a lot easier to read when trying to debug things.
143 lines
3.9 KiB
Python
Executable File
143 lines
3.9 KiB
Python
Executable File
#!/usr/bin/env python
|
|
|
|
# Script to load documentation from tarballs
|
|
|
|
import sys
|
|
import os
|
|
import tarfile
|
|
import re
|
|
import tidy
|
|
from optparse import OptionParser
|
|
from ConfigParser import ConfigParser
|
|
|
|
import psycopg2
|
|
|
|
pagecount = 0
|
|
quiet = False
|
|
|
|
re_titlematch = re.compile('<title\s*>([^<]+)</title\s*>', re.IGNORECASE)
|
|
|
|
## Load a single page
|
|
def load_doc_file(filename, f):
|
|
tidyopts = dict(drop_proprietary_attributes=1,
|
|
alt_text='',
|
|
hide_comments=1,
|
|
output_xhtml=1,
|
|
show_body_only=1,
|
|
clean=1,
|
|
char_encoding='utf8',
|
|
indent='auto',
|
|
)
|
|
|
|
# Postgres 10 started using xml toolchain and now produces docmentation in utf8. So we need
|
|
# to figure out which version it is.
|
|
rawcontents = f.read()
|
|
if rawcontents.startswith('<?xml version="1.0" encoding="UTF-8"'):
|
|
# Version 10, use utf8
|
|
encoding = 'utf-8'
|
|
# XML builds also don't need clean=1, and that one adds some interesting CSS properties
|
|
del tidyopts['clean']
|
|
else:
|
|
encoding = 'latin1'
|
|
|
|
# PostgreSQL prior to 11 used an older toolchain to build the docs, which does not support
|
|
# indented HTML. So turn it off on those, but keep it on the newer versions where it works,
|
|
# because it makes things a lot easier to debug.
|
|
if float(ver) < 11 and float(ver) > 0:
|
|
tidyopts['indent'] = 'no'
|
|
|
|
contents = unicode(rawcontents, encoding)
|
|
|
|
tm = re_titlematch.search(contents)
|
|
if tm:
|
|
title = tm.group(1)
|
|
else:
|
|
title = ""
|
|
if not quiet: print "--- file: %s (%s) ---" % (filename, title)
|
|
|
|
s = tidy.parseString(contents.encode('utf-8'), **tidyopts)
|
|
curs.execute("INSERT INTO docs (file, version, title, content) VALUES (%(f)s, %(v)s, %(t)s, %(c)s)",{
|
|
'f': filename,
|
|
'v': ver,
|
|
't': title,
|
|
'c': str(s),
|
|
})
|
|
global pagecount
|
|
pagecount += 1
|
|
|
|
## Main execution
|
|
|
|
parser = OptionParser(usage="usage: %prog [options] <version> <tarfile>")
|
|
parser.add_option("-q", "--quiet", action="store_true", dest="quiet",
|
|
help="Run quietly")
|
|
(options, args) = parser.parse_args()
|
|
|
|
if len(args) != 2:
|
|
parser.print_usage()
|
|
sys.exit(1)
|
|
|
|
quiet = options.quiet
|
|
ver = sys.argv[1]
|
|
tarfilename = sys.argv[2]
|
|
|
|
config = ConfigParser()
|
|
config.read(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'docload.ini'))
|
|
|
|
if not os.path.isfile(tarfilename):
|
|
print "File %s not found" % tarfilename
|
|
sys.exit(1)
|
|
|
|
tf = tarfile.open(tarfilename)
|
|
|
|
connection = psycopg2.connect(config.get('db', 'dsn'))
|
|
|
|
curs = connection.cursor()
|
|
# Verify that the version exists, and what we're loading
|
|
curs.execute("SELECT current FROM core_version WHERE tree=%(v)s", {'v': ver})
|
|
r = curs.fetchall()
|
|
if len(r) != 1:
|
|
print "Version %s not found in the system, cannot load!" % ver
|
|
sys.exit(1)
|
|
|
|
iscurrent = r[0][0]
|
|
|
|
# Remove any old docs for this version (still protected by a transaction while
|
|
# we perform the load)
|
|
curs.execute("DELETE FROM docs WHERE version=%(v)s", {'v': ver})
|
|
|
|
|
|
re_htmlfile = re.compile('[^/]*/doc/src/sgml/html/.*')
|
|
re_tarfile = re.compile('[^/]*/doc/postgres.tar.gz$')
|
|
for member in tf:
|
|
if re_htmlfile.match(member.name):
|
|
load_doc_file(os.path.basename(member.name), tf.extractfile(member))
|
|
if re_tarfile.match(member.name):
|
|
f = tf.extractfile(member)
|
|
inner_tar = tarfile.open(fileobj=f)
|
|
for inner_member in inner_tar:
|
|
# Some old versions have index.html as a symlink - so let's
|
|
# just ignore all symlinks to be on the safe side.
|
|
if inner_member.issym(): continue
|
|
|
|
if inner_member.name.endswith('.html') or inner_member.name.endswith('.htm'):
|
|
load_doc_file(inner_member.name, inner_tar.extractfile(inner_member))
|
|
tf.close()
|
|
|
|
# Update the docs loaded timestamp
|
|
curs.execute("UPDATE core_version SET docsloaded=CURRENT_TIMESTAMP WHERE tree=%(v)s", {'v': ver})
|
|
|
|
# Issue varnish purge for all docs of this version
|
|
if ver == "0":
|
|
# Special handling of developer docs...
|
|
ver = "devel"
|
|
|
|
curs.execute("SELECT varnish_purge('^/docs/' || %(v)s || '/')", {'v': ver})
|
|
if iscurrent:
|
|
curs.execute("SELECT varnish_purge('^/docs/current/')")
|
|
|
|
connection.commit()
|
|
connection.close()
|
|
|
|
if not quiet: print "Done (%i pages)." % pagecount
|
|
|