Switch email sending go through a queue table in the database

Import the code from the PostgreSQL Europe website to handle this, since it's
well proven by now.

Any points that send email now just write them to the database using the
functions in queuedmail.util. This means we can now submit notification
emails and such things within transactions and have them properly roll bcak
if something goes wrong (so no more incorrect notifications when there is
a database error).

These emails are picked up by a cronjob that runs frequently (typically
once per minute or once every 2 minutes) that submits them to the local
mailserver. By doing it out of line, this gives us a much better way of
dealing with cases where mail delivery is really slow.

The submission from the cronjob is now done with smtp to localhost instead
of opening a pipe to the sendmail command - though this should have no
major effects on anything.

This also removes the setting SUPPRESS_NOTIFICATIONS, as no notifications
are actually ever sent unless the cronjob is run. On development systems
they will just go into the queuedmail table, and can be deleted from there.

docs/dev_install.rst

@@ -25,7 +25,6 @@ with. Here's a quick step-by-step on how to do that:
 
 	DEBUG=True
 	TEMPLATE_DEBUG=DEBUG
-	SUPPRESS_NOTIFICATIONS=True
 	SITE_ROOT="http://localhost:8000"
 	NO_HTTPS_REDIRECT=True
 	SESSION_COOKIE_SECURE=False

pgweb/lists/views.py

@@ -3,11 +3,10 @@ from django.http import HttpResponse
 from django.views.decorators.csrf import csrf_exempt
 from django.conf import settings
 
-from email.mime.text import MIMEText
 import simplejson as json
 
 from pgweb.util.contexts import NavContext
-from pgweb.util.misc import sendmail
+from pgweb.mailqueue.util import send_simple_mail
 
 from models import MailingList, MailingListGroup
 from forms import SubscribeForm
@@ -26,11 +25,12 @@ def subscribe(request):
 					mailtxt += "set digest\n"
 			else:
 				mailtxt += "unsubscribe %s\n" % form.cleaned_data['lists']
-			msg = MIMEText(mailtxt, _charset='utf-8')
+
-			msg['Subject'] = ''
+			send_simple_mail(form.cleaned_data['email'],
-			msg['To'] = settings.LISTSERVER_EMAIL
+							 settings.LISTSERVER_EMAIL,
-			msg['From'] = form.cleaned_data['email']
+							 '',
-			sendmail(msg)
+							 mailtxt)
+
 			return render_to_response('lists/subscribed.html', {
 			}, NavContext(request, "community"))
 	else:

pgweb/mailqueue/admin.py

@@ -0,0 +1,5 @@
+from django.contrib import admin
+
+from models import QueuedMail
+
+admin.site.register(QueuedMail)

pgweb/mailqueue/models.py

@@ -0,0 +1,11 @@
+from django.db import models
+
+class QueuedMail(models.Model):
+	sender = models.EmailField(max_length=100, null=False, blank=False)
+	receiver = models.EmailField(max_length=100, null=False, blank=False)
+	# We store the raw MIME message, so if there are any attachments or
+	# anything, we just push them right in there!
+	fullmsg = models.TextField(null=False, blank=False)
+
+	def __unicode__(self):
+		return "%s: %s -> %s" % (self.pk, self.sender, self.receiver)

pgweb/mailqueue/util.py

@@ -0,0 +1,37 @@
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from email.mime.nonmultipart import MIMENonMultipart
+from email.Utils import formatdate
+from email import encoders
+
+from models import QueuedMail
+
+def send_simple_mail(sender, receiver, subject, msgtxt, attachments=None):
+	# attachment format, each is a tuple of (name, mimetype,contents)
+	# content should be *binary* and not base64 encoded, since we need to
+	# use the base64 routines from the email library to get a properly
+	# formatted output message
+	msg = MIMEMultipart()
+	msg['Subject'] = subject
+	msg['To'] = receiver
+	msg['From'] = sender
+	msg['Date'] = formatdate(localtime=True)
+
+	msg.attach(MIMEText(msgtxt, _charset='utf-8'))
+
+	if attachments:
+		for filename, contenttype, content in attachments:
+			main,sub = contenttype.split('/')
+			part = MIMENonMultipart(main,sub)
+			part.set_payload(content)
+			part.add_header('Content-Disposition', 'attachment; filename="%s"' % filename)
+			encoders.encode_base64(part)
+			msg.attach(part)
+
+
+	# Just write it to the queue, so it will be transactionally rolled back
+	QueuedMail(sender=sender, receiver=receiver, fullmsg=msg.as_string()).save()
+
+def send_mail(sender, receiver, fullmsg):
+	# Send an email, prepared as the full MIME encoded mail already
+	QueuedMail(sender=sender, receiver=receiver, fullmsg=fullmsg).save()

pgweb/settings.py

@@ -99,6 +99,7 @@ INSTALLED_APPS = [
     'django.contrib.markup',
 	'django.contrib.staticfiles',
     'pgweb.core',
+    'pgweb.mailqueue',
     'pgweb.account',
     'pgweb.news',
     'pgweb.events',
@@ -148,7 +149,6 @@ NOTIFICATION_EMAIL="someone@example.com"               # Address to send notific
 NOTIFICATION_FROM="someone@example.com"                # Address to send notifications *from*
 LISTSERVER_EMAIL="someone@example.com"                 # Address to majordomo
 BUGREPORT_EMAIL="someone@example.com"                  # Address to pgsql-bugs list
-SUPPRESS_NOTIFICATIONS=False                           # Set to true to disable all notification mails
 NO_HTTPS_REDIRECT=False                                # Set to true to disable redirects to https when
                                                        # developing locally
 FRONTEND_SERVERS=()                                    # A tuple containing the *IP addresses* of all the

pgweb/util/admin.py

@@ -1,10 +1,8 @@
 from django.contrib import admin
 from django.conf import settings
 
-from email.mime.text import MIMEText
-
 from pgweb.core.models import ModerationNotification
-from util.misc import sendmail
+from mailqueue.util import send_simple_mail
 
 
 class PgwebAdmin(admin.ModelAdmin):
@@ -78,29 +76,20 @@ class PgwebAdmin(admin.ModelAdmin):
 												obj,
 												request.POST['new_notification'])
 
-				msg = MIMEText(msgstr, _charset='utf-8')
+				send_simple_mail(settings.NOTIFICATION_FROM,
-				msg['Subject'] = "postgresql.org moderation notification"
+								 obj.org.email,
-				msg['To'] = obj.org.email
+								 "postgresql.org moderation notification",
-				msg['From'] = settings.NOTIFICATION_FROM
+								 msgstr)
-				if hasattr(settings,'SUPPRESS_NOTIFICATIONS') and settings.SUPPRESS_NOTIFICATIONS:
-					print msg.as_string()
-				else:
-					sendmail(msg)
 
 				# Also generate a mail to the moderators
-				msg = MIMEText(_get_moderator_notification_text(request.POST.has_key('remove_after_notify'),
+				send_simple_mail(settings.NOTIFICATION_FROM,
-																obj,
+								 settings.NOTIFICATION_EMAIL,
-																request.POST['new_notification'],
+								 "Moderation comment on %s %s" % (obj.__class__._meta.verbose_name, obj.id),
-																request.user.username
+								 _get_moderator_notification_text(request.POST.has_key('remove_after_notify'),
-																),
+																  obj,
-							   _charset='utf-8')
+																  request.POST['new_notification'],
-				msg['Subject'] = "Moderation comment on %s %s" % (obj.__class__._meta.verbose_name, obj.id)
+																  request.user.username
-				msg['To'] = settings.NOTIFICATION_EMAIL
+															  ))
-				msg['From'] = settings.NOTIFICATION_FROM
-				if hasattr(settings,'SUPPRESS_NOTIFICATIONS') and settings.SUPPRESS_NOTIFICATIONS:
-					print msg.as_string()
-				else:
-					sendmail(msg)
 
 				if request.POST.has_key('remove_after_notify'):
 					# Object should not be saved, it should be deleted

pgweb/util/bases.py

@@ -1,11 +1,10 @@
-from email.mime.text import MIMEText
-
 from django.db.models.signals import pre_save, post_save
 from django.db import models
 from django.conf import settings
 
 from util.middleware import get_current_user
-from util.misc import sendmail, varnish_purge
+from util.misc import varnish_purge
+from mailqueue.util import send_simple_mail
 
 class PgModel(object):
 	send_notification = False
@@ -43,15 +42,10 @@ class PgModel(object):
 
 
 		# Build the mail text
-		msg = MIMEText(cont, _charset='utf-8')
+		send_simple_mail(settings.NOTIFICATION_FROM,
-		msg['Subject'] = "%s by %s" % (subj, get_current_user())
+						 settings.NOTIFICATION_EMAIL,
-		msg['To'] = settings.NOTIFICATION_EMAIL
+						 "%s by %s" % (subj, get_current_user()),
-		msg['From'] = settings.NOTIFICATION_FROM
+						 cont)
-
-		if hasattr(settings,'SUPPRESS_NOTIFICATIONS') and settings.SUPPRESS_NOTIFICATIONS:
-			print msg.as_string()
-		else:
-			sendmail(msg)
 
 	def delete(self):
 		# We can't compare the object, but we should be able to construct something anyway
@@ -60,14 +54,11 @@ class PgModel(object):
 				self._meta.verbose_name,
 				self.id,
 				get_current_user())
-			msg = MIMEText(self.full_text_representation(), _charset='utf-8')
+
-			msg['Subject'] = subject
+			send_simple_mail(settings.NOTIFICATION_FROM,
-			msg['To'] = settings.NOTIFICATION_EMAIL
+							 settings.NOTIFICATION_EMAIL,
-			msg['From'] = settings.NOTIFICATION_FROM
+							 subject,
-			if hasattr(settings,'SUPPRESS_NOTIFICATIONS') and settings.SUPPRESS_NOTIFICATIONS:
+							 self.full_text_representation())
-				print msg.as_string()
-			else:
-				sendmail(msg)
 
 		# Now call our super to actually delete the object
 		super(PgModel, self).delete()

pgweb/util/misc.py

@@ -1,24 +1,12 @@
-from subprocess import Popen, PIPE
-from email.mime.text import MIMEText
 from django.db import connection
 from django.conf import settings
 
+from pgweb.mailqueue.util import send_simple_mail
 from pgweb.util.helpers import template_to_string
 
-def sendmail(msg):
-	pipe = Popen("/usr/sbin/sendmail -t", shell=True, stdin=PIPE).stdin
-	pipe.write(msg.as_string())
-	pipe.close()
-
 def send_template_mail(sender, receiver, subject, templatename, templateattr={}):
-	msg = MIMEText(
+	send_simple_mail(sender, receiver, subject,
-		template_to_string(templatename, templateattr),
+					 template_to_string(templatename, templateattr))
-		_charset='utf-8')
-	msg['Subject'] = subject
-	msg['To'] = receiver
-	msg['From'] = sender
-	sendmail(msg)
-
 
 def is_behind_cache(request):
 	"""

tools/mailqueue/send_queued_mail.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+#
+# Script to send off all queued email.
+#
+# This script is intended to be run frequently from cron. We queue things
+# up in the db so that they get automatically rolled back as necessary,
+# but once we reach this point we're just going to send all of them one
+# by one.
+#
+
+import sys
+import os
+import smtplib
+
+# Set up to run in django environment
+from django.core.management import setup_environ
+sys.path.append(os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])), '../../pgweb'))
+import settings
+setup_environ(settings)
+
+from django.db import connection, transaction
+
+from pgweb.mailqueue.models import QueuedMail
+
+if __name__ == "__main__":
+	# Grab advisory lock, if available. Lock id is just a random number
+	# since we only need to interlock against ourselves. The lock is
+	# automatically released when we're done.
+	curs = connection.cursor()
+	curs.execute("SELECT pg_try_advisory_lock(2896780)")
+	if not curs.fetchall()[0][0]:
+		print "Failed to get advisory lock, existing send_queued_mail process stuck?"
+		sys.exit(1)
+
+	for m in QueuedMail.objects.all():
+		# Yes, we do a new connection for each run. Just because we can.
+		# If it fails we'll throw an exception and just come back on the
+		# next cron job. And local delivery should never fail...
+		smtp = smtplib.SMTP("localhost")
+		smtp.sendmail(m.sender, m.receiver, m.fullmsg.encode('utf-8'))
+		smtp.close()
+		m.delete()
+		transaction.commit()
+	connection.close()