Implement basic varnish purging

This allows all models inherited from PgModel to specify which
URLs to purge by either setting a field or defining a function
called purge_urls, at which point they will be purged whenever
the save signal is fired.

Also implements a form under /admin/purge/ that allows for manual
purging. This should probably be extended in the future to show
the status of the pgq slaves, but that will come later.

Includes a SQL function that posts the expires to a pgq queue. For
a local deployment, this can be replaced with a simple void function
to turn off varnish purging.

pgweb/contributors/models.py

@@ -1,19 +1,22 @@
 from django.db import models
 
+from pgweb.util.bases import PgModel
 
-class ContributorType(models.Model):
+class ContributorType(PgModel, models.Model):
 	typename = models.CharField(max_length=32, null=False, blank=False)
 	sortorder = models.IntegerField(null=False, default=100)
 	extrainfo = models.TextField(null=True, blank=True)
 	detailed = models.BooleanField(null=False, default=True)
-	
+
+	purge_urls = ('community/contributors/', )
+
 	def __unicode__(self):
 		return self.typename
 	
 	class Meta:
 		ordering = ('sortorder',)
 
-class Contributor(models.Model):
+class Contributor(PgModel, models.Model):
 	ctype = models.ForeignKey(ContributorType)
 	lastname = models.CharField(max_length=100, null=False, blank=False)
 	firstname = models.CharField(max_length=100, null=False, blank=False)
@@ -23,6 +26,8 @@ class Contributor(models.Model):
 	location = models.CharField(max_length=100, null=True, blank=True)
 	contribution = models.TextField(null=True, blank=True)
 
+	purge_urls = ('community/contributors/', )
+
 	def __unicode__(self):
 		return "%s %s" % (self.firstname, self.lastname)
 	

pgweb/core/models.py

@@ -36,6 +36,11 @@ class Version(models.Model):
 	class Meta:
 		ordering = ('-tree', )
 
+	def purge_urls(self):
+		yield '/$'
+		yield 'versions.rss'
+		# FIXME: probably a lot more?
+
 
 class Country(models.Model):
 	name = models.CharField(max_length=100, null=False, blank=False)

pgweb/core/views.py

@@ -1,9 +1,9 @@
 from django.shortcuts import render_to_response, get_object_or_404
-from django.http import HttpResponse, Http404
+from django.http import HttpResponse, Http404, HttpResponseRedirect
 from django.template import TemplateDoesNotExist, loader, Context
 from django.contrib.auth.decorators import login_required
 from django.db.models import Count
-from django.db import connection
+from django.db import connection, transaction
 
 from datetime import date, datetime
 from os import uname
@@ -12,7 +12,7 @@ from pgweb.util.decorators import ssl_required, cache
 from pgweb.util.contexts import NavContext
 from pgweb.util.helpers import simple_form, PgXmlHelper
 from pgweb.util.moderation import get_all_pending_moderations
-from pgweb.util.misc import get_client_ip, is_behind_cache
+from pgweb.util.misc import get_client_ip, is_behind_cache, varnish_purge
 from pgweb.util.sitestruct import get_all_pages_struct
 
 # models needed for the pieces on the frontpage
@@ -147,3 +147,18 @@ def admin_pending(request):
 	return render_to_response('core/admin_pending.html', {
 			'app_list': get_all_pending_moderations(),
 			})
+
+# Purge objects from varnish, for the admin pages
+@login_required
+def admin_purge(request):
+	if request.method == 'POST':
+		url = request.POST['url']
+		if url == '':
+			return HttpResponseRedirect('.')
+		varnish_purge(url)
+		transaction.commit_unless_managed()
+		return render_to_response('core/admin_purge.html', {
+				'purge_completed': '^%s' % url,
+				})
+	else:
+		return render_to_response('core/admin_purge.html')

pgweb/docs/models.py

@@ -1,6 +1,7 @@
 from django.db import models
 from django.contrib.auth.models import User
 from pgweb.util.bases import PgModel
+from pgweb.core.models import Version
 
 from datetime import datetime
 
@@ -24,6 +25,14 @@ class DocComment(PgModel, models.Model):
 
 	send_notification = True
 
+	def purge_urls(self):
+		yield '/docs/%s/interactive/%s' % (self.version, self.file)
+		try:
+			if Version.objects.get(tree=self.version).current:
+				yield '/docs/current/interactive/%s' % self.file
+		except Version.DoesNotExist:
+			pass
+
 	class Meta:
 		ordering = ('-posted_at',)
 

pgweb/events/models.py

@@ -24,6 +24,13 @@ class Event(models.Model, PgModel):
 	send_notification = True
 	markdown_fields = ('details', )
 	
+	def purge_urls(self):
+		yield '/about/event/%s/' % self.pk
+		yield '/about/eventarchive/'
+		yield 'events.rss'
+		# FIXME: when to expire the front page?
+		yield '/$'
+
 	def __unicode__(self):
 		return "%s: %s" % (self.startdate, self.title)
 

pgweb/featurematrix/models.py

@@ -1,5 +1,7 @@
 from django.db import models
 
+from pgweb.util.bases import PgModel
+
 choices_map = {
  0: {'str': 'No',       'class': 'no', 'bgcolor': '#ffdddd'},
  1: {'str': 'Yes',      'class': 'yes', 'bgcolor': '#ddffdd'},
@@ -8,10 +10,12 @@ choices_map = {
 }
 choices = [(k, v['str']) for k,v in choices_map.items()]
 
-class FeatureGroup(models.Model):
+class FeatureGroup(PgModel, models.Model):
 	groupname = models.CharField(max_length=100, null=False, blank=False)
 	groupsort = models.IntegerField(null=False, blank=False)
 
+	purge_urls = ('about/featurematrix/', )
+
 	def __unicode__(self):
 		return self.groupname
 
@@ -20,7 +24,7 @@ class FeatureGroup(models.Model):
 		# Return a list of all the columns for the matrix
 		return [b for a,b in versions]
 
-class Feature(models.Model):
+class Feature(PgModel, models.Model):
 	group = models.ForeignKey(FeatureGroup, null=False, blank=False)
 	featurename = models.CharField(max_length=100, null=False, blank=False)
 	featuredescription = models.TextField(null=False, blank=True)
@@ -33,6 +37,8 @@ class Feature(models.Model):
 	v84 = models.IntegerField(null=False, blank=False, default=0, verbose_name="8.4", choices=choices)
 	v85 = models.IntegerField(null=False, blank=False, default=0, verbose_name="8.5a3", choices=choices)
 
+	purge_urls = ('about/featurematrix/.*', )
+
 	def __unicode__(self):
 		# To make it look good in the admin interface, just don't render it
 		return ''

pgweb/lists/models.py

@@ -1,16 +1,20 @@
 from django.db import models
 
-class MailingListGroup(models.Model):
+from pgweb.util.bases import PgModel
+
+class MailingListGroup(PgModel, models.Model):
 	groupname = models.CharField(max_length=64, null=False, blank=False)
 	sortkey = models.IntegerField(null=False, default=10)
 
+	purge_urls = ('community/lists/', )
+
 	def __unicode__(self):
 		return self.groupname
 	
 	class Meta:
 		ordering = ('sortkey', )
 
-class MailingList(models.Model):
+class MailingList(PgModel, models.Model):
 	group = models.ForeignKey(MailingListGroup, null=False)
 	listname = models.CharField(max_length=64, null=False, blank=False)
 	active = models.BooleanField(null=False, default=False)
@@ -18,6 +22,8 @@ class MailingList(models.Model):
 	description = models.TextField(null=False, blank=True)
 	shortdesc = models.TextField(null=False, blank=True)
 
+	purge_urls = ('community/lists/', )
+
 	@property
 	def maybe_shortdesc(self):
 		if self.shortdesc:

pgweb/news/models.py

@@ -12,6 +12,13 @@ class NewsArticle(PgModel, models.Model):
 
 	send_notification = True
 	markdown_fields = ('content',)
+
+	def purge_urls(self):
+		yield '/about/news/%s/' % self.pk
+		yield '/about/newsarchive/'
+		yield 'news.rss'
+		# FIXME: when to expire the front page?
+		yield '/$'
 	
 	def __unicode__(self):
 		return "%s: %s" % (self.date, self.title)

pgweb/profserv/models.py

@@ -4,7 +4,7 @@ from django.contrib.auth.models import User
 from pgweb.core.models import Organisation
 from pgweb.util.bases import PgModel
 
-class ProfessionalService(models.Model):
+class ProfessionalService(PgModel, models.Model):
 	submitter = models.ForeignKey(User, null=False, blank=False)
 	approved = models.BooleanField(null=False, blank=False, default=False)
 
@@ -28,6 +28,7 @@ class ProfessionalService(models.Model):
 	provides_hosting = models.BooleanField(null=False, default=False)
 	interfaces = models.CharField(max_length=512, null=True, blank=True)
 	
+	purge_urls = ('support/professional_', )
 	
 	send_notification = True
 	

pgweb/pwn/models.py

@@ -1,14 +1,21 @@
 from django.db import models
 
+from pgweb.util.bases import PgModel
+
 from datetime import date
 
-class PwnPost(models.Model):
+class PwnPost(PgModel, models.Model):
 	date = models.DateField(null=False, blank=False, default=date.today, unique=True)
 	intro  = models.TextField(null=False, blank=False)
 	content = models.TextField(null=False, blank=False)
 
 	markdown_fields = ('intro', 'content',)
 
+	def purge_urls(self):
+		yield 'community/weeklynews/$'
+		yield 'community/weeklynews/pwn%s/' % self.linkdate()
+		yield 'weeklynews.rss'
+
 	def __unicode__(self):
 		return "PostgreSQL Weekly News %s" % self.date
 

pgweb/quotes/models.py

@@ -9,7 +9,9 @@ class Quote(models.Model, PgModel):
 	link = models.URLField(null=False, blank=False)
 	
 	send_notification = True
-	
+
+	purge_urls = ('about/quotesarchive/', '/$', )
+
 	def __unicode__(self):
 		if len(self.quote) > 75:
 			return "%s..." % self.quote[:75]

pgweb/sponsors/models.py

@@ -2,31 +2,37 @@ from django.db import models
 
 from core.models import Country
 
-class SponsorType(models.Model):
+from pgweb.util.bases import PgModel
+
+class SponsorType(PgModel, models.Model):
 	typename = models.CharField(max_length=32, null=False, blank=False)
 	description = models.TextField(null=False, blank=False)
 	sortkey = models.IntegerField(null=False, default=10)
-	
+
+	purge_urls = ('about/servers/', 'about/sponsors/', )
+
 	def __unicode__(self):
 		return self.typename
 	
 	class Meta:
 		ordering = ('sortkey', )
 		
-class Sponsor(models.Model):
+class Sponsor(PgModel, models.Model):
 	sponsortype = models.ForeignKey(SponsorType, null=False)
 	name = models.CharField(max_length=128, null=False, blank=False)
 	url = models.URLField(null=False, blank=False)
 	logoname = models.CharField(max_length=64, null=False, blank=False)
 	country = models.ForeignKey(Country, null=False)
-	
+
+	purge_urls = ('about/sponsors/', )
+
 	def __unicode__(self):
 		return self.name
 	
 	class Meta:
 		ordering = ('name', )
 
-class Server(models.Model):
+class Server(PgModel, models.Model):
 	name = models.CharField(max_length=32, null=False, blank=False)
 	sponsors = models.ManyToManyField(Sponsor)
 	dedicated = models.BooleanField(null=False, default=True)
@@ -35,6 +41,8 @@ class Server(models.Model):
 	location = models.CharField(max_length=128, null=False, blank=False)
 	usage = models.TextField(null=False, blank=False)
 	
+	purge_urls = ('about/servers/', )
+
 	def __unicode__(self):
 		return self.name
 	

pgweb/survey/models.py

@@ -1,6 +1,8 @@
 from django.db import models
 from django.contrib.auth.models import User
 
+from pgweb.util.bases import PgModel
+
 from datetime import datetime
 
 # internal text/value object
@@ -14,7 +16,7 @@ class SurveyAnswerValues(object):
 		self.votes = votes
 		self.votespercent = votespercent
 
-class Survey(models.Model):
+class Survey(PgModel, models.Model):
 	question = models.CharField(max_length=100, null=False, blank=False)
 	opt1 = models.CharField(max_length=100, null=False, blank=False)
 	opt2 = models.CharField(max_length=100, null=False, blank=False)
@@ -27,6 +29,8 @@ class Survey(models.Model):
 	posted = models.DateTimeField(null=False, default=datetime.now)
 	current = models.BooleanField(null=False, default=False)
 
+	purge_urls = ('community/survey', )
+
 	def __unicode__(self):
 		return self.question
 
@@ -78,7 +82,7 @@ class Survey(models.Model):
 		# free to save this one.
 		super(Survey, self).save()
 
-class SurveyAnswer(models.Model):
+class SurveyAnswer(PgModel, models.Model):
 	survey = models.ForeignKey(Survey, null=False, blank=False, primary_key=True)
 	tot1 = models.IntegerField(null=False, default=0)
 	tot2 = models.IntegerField(null=False, default=0)
@@ -89,6 +93,8 @@ class SurveyAnswer(models.Model):
 	tot7 = models.IntegerField(null=False, default=0)
 	tot8 = models.IntegerField(null=False, default=0)
 
+	purge_urls = ('community/survey', )
+
 class SurveyLock(models.Model):
 	ipaddr = models.IPAddressField(null=False, blank=False)
 	time = models.DateTimeField(null=False, default=datetime.now)

pgweb/urls.py

@@ -108,6 +108,7 @@ urlpatterns = patterns('',
 
 	# Override some URLs in admin, to provide our own pages
 	(r'^admin/pending/$', 'pgweb.core.views.admin_pending'),
+	(r'^admin/purge/$', 'pgweb.core.views.admin_purge'),
     # Uncomment the next line to enable the admin:
     (r'^admin/(.*)', admin.site.root),
 

pgweb/util/bases.py

@@ -1,17 +1,31 @@
 from email.mime.text import MIMEText
 
-from django.db.models.signals import pre_save
+from django.db.models.signals import pre_save, post_save
 from django.db import models
 from django.conf import settings
 
 from util.middleware import get_current_user
-from util.misc import sendmail
+from util.misc import sendmail, varnish_purge
 
 class PgModel(object):
 	send_notification = False
+	purge_urls = ()
 	notify_fields = None
 	modifying_user = None
 
+	def PostSaveHandler(self):
+		"""
+		If a set of URLs are available as purge_urls, then send commands
+		to the cache to purge those urls.
+		"""
+		if callable(self.purge_urls):
+			purgelist = self.purge_urls()
+		else:
+			if not self.purge_urls: return
+			purgelist = self.purge_urls
+		map(varnish_purge, purgelist)
+
+
 	def PreSaveHandler(self):
 		"""If send_notification is set to True, send a default formatted notification mail"""
 		
@@ -131,6 +145,11 @@ def my_pre_save_handler(sender, **kwargs):
 	if isinstance(instance, PgModel):
 		instance.PreSaveHandler()
 
+def my_post_save_handler(sender, **kwargs):
+	instance = kwargs['instance']
+	if isinstance(instance, PgModel):
+		instance.PostSaveHandler()
+
 def register_basic_signal_handlers():
 	pre_save.connect(my_pre_save_handler)
-
+	post_save.connect(my_post_save_handler)

pgweb/util/misc.py

@@ -1,5 +1,6 @@
 from subprocess import Popen, PIPE
 from email.mime.text import MIMEText
+from django.db import connection
 from django.conf import settings
 
 from pgweb.util.helpers import template_to_string
@@ -58,3 +59,14 @@ def get_client_ip(request):
 			return request.META['REMOTE_ADDR']
 	else:
 		return request.META['REMOTE_ADDR']
+
+
+
+
+def varnish_purge(url):
+	"""
+	Purge the specified URL from Varnish. Will add initial anchor to the URL,
+	but no trailing one, so by default a wildcard match is done.
+	"""
+	url = '^%s' % url
+	connection.cursor().execute("SELECT varnish_purge(%s)", (url, ))

sql/varnish.sql

@@ -0,0 +1,16 @@
+BEGIN;
+
+--
+-- Create a function to purge from varnish cache
+-- By defalut this adds the object to a pgq queue,
+-- but this function can be replaced with a void one
+-- when running a development version.
+--
+
+CREATE OR REPLACE FUNCTION varnish_purge(url text)
+RETURNS bigint
+AS $$
+   SELECT pgq.insert_event('varnish', 'P', $1);
+$$ LANGUAGE 'sql';
+
+COMMIT;

templates/admin/index.html

@@ -11,7 +11,8 @@
 
 {% block content %}
 <p>
-View <a href="/admin/pending/">pending</a> moderation requests.
+View <a href="/admin/pending/">pending</a> moderation requests.<br/>
+Purge contents from <a href="/admin/purge/">varnish</a>.
 </p>
 <div id="content-main">
 

templates/core/admin_purge.html

@@ -0,0 +1,24 @@
+{%extends "admin/base_site.html"%}
+
+{%block breadcrumbs%}
+<div class="breadcrumbs"><a href="/admin/">Home</a> &rsaquo; Pending</div>
+{%endblock%}
+
+{% block bodyclass %}change-list{% endblock %}
+{% block coltype %}flex{% endblock %}
+
+{%block content%}
+<h1>Purge URL from Varnish</h1>
+
+<div id="content-main">
+{%if purge_completed %}
+  <div class="module">
+    Purge completed: {{purge_completed}}
+  </div>
+{%endif%}
+ <form method="POST" action=".">
+URL (regex): <input type="text" name="url">
+  <input type="submit" value="Purge" />
+ </form>
+</div>
+{%endblock%}