Add support for sending out news as HTML email

When a news article is approved, it gets delivered as an email to the
pgsql-announce mailinglist. It will render the markdown of the news
article into a HTML part of the email, and include the markdown raw as
the text part (for those unable or unwilling to read html mail).

For each organisation, a mail template can be specified. Initially only
two templates are supported, one "default" and one "pgproject" which is
for official project news. The intention is *not* to provide generic
templates, but we may want to extend this to certain related projects in
the future *maybe* (such as regional NPOs).

These templates are stored in templates/news/mail/*.html, and for each
template *all* images found in templates/news/mail/img.<template>/ will
be attached to the email. "Conditional image inclusion" currently not
supported.

To do CSS inlining on top of the markdown output, module pynliner is now
required (available in the python3-pynliner package on Debian).

A testing script is added as news_send_email.py in order to easier test
out templates. This is *not* intended for production sending, so it will
for example send unmoderated news. By sending, it adds it to the
outgoing mailqueue in the system, so unless the cronjob is set up to
send, nothing will happen until that is run manually.

Support is included for tagged delivery using pglister, by directly
mapping NewsTags to pglister tags.

pgweb/core/migrations/0003_mailtemplate.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.27 on 2020-07-07 15:18
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0002_block_oauth'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='organisation',
+            name='mailtemplate',
+            field=models.CharField(choices=[('default', 'Default template'), ('pgproject', 'PostgreSQL project news')], default='default', max_length=50),
+        ),
+    ]

pgweb/core/models.py

@@ -123,6 +123,12 @@ class OrganisationType(models.Model):
         return self.typename
 
 
+_mail_template_choices = (
+    ('default', 'Default template'),
+    ('pgproject', 'PostgreSQL project news'),
+)
+
+
 class Organisation(TwostateModerateModel):
     name = models.CharField(max_length=100, null=False, blank=False, unique=True)
     address = models.TextField(null=False, blank=True)
@@ -131,6 +137,7 @@ class Organisation(TwostateModerateModel):
     phone = models.CharField(max_length=100, null=False, blank=True)
     orgtype = models.ForeignKey(OrganisationType, null=False, blank=False, verbose_name="Organisation type", on_delete=models.CASCADE)
     managers = models.ManyToManyField(User, blank=False)
+    mailtemplate = models.CharField(max_length=50, null=False, blank=False, default='default', choices=_mail_template_choices)
     lastconfirmed = models.DateTimeField(null=False, blank=False, auto_now_add=True)
 
     account_edit_suburl = 'organisations'

pgweb/core/templatetags/pgfilters.py

@@ -1,7 +1,9 @@
 from django.template.defaultfilters import stringfilter
 from django import template
-import json
+from django.template.loader import get_template
 
+import json
+import pynliner
 
 register = template.Library()
 
@@ -93,3 +95,38 @@ def joinandor(value, andor):
         value = list(value)
 
     return ", ".join([str(x) for x in value[:-1]]) + ' ' + andor + ' ' + str(value[-1])
+
+
+# CSS inlining (used for HTML email)
+@register.tag
+class InlineCss(template.Node):
+    def __init__(self, nodes, arg):
+        self.nodes = nodes
+        self.arg = arg
+
+    def render(self, context):
+        contents = self.nodes.render(context)
+        path = self.arg.resolve(context, True)
+        if path is not None:
+            css = get_template(path).render()
+        else:
+            css = ''
+
+        p = pynliner.Pynliner().from_string(contents)
+        p.with_cssString(css)
+        return p.run()
+
+
+@register.tag
+def inlinecss(parser, token):
+    nodes = parser.parse(('endinlinecss',))
+
+    parser.delete_first_token()
+
+    # First part of token is the tagname itself
+    css = token.split_contents()[1]
+
+    return InlineCss(
+        nodes,
+        parser.compile_filter(css),
+    )

pgweb/core/views.py

@@ -392,6 +392,8 @@ def admin_moderate(request, objtype, objid):
                                          "The {} with title {}\nhas been approved and is now published.".format(obj._meta.verbose_name, obj.title),
                                          modnote,
                                          "approved")
+                if hasattr(obj, 'on_approval'):
+                    obj.on_approval(request)
             elif modstate == ModerationState.REJECTED:
                 _send_moderation_message(request,
                                          obj,

pgweb/news/management/commands/news_send_email.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+#
+# Script to send out a news email
+# THIS IS FOR TESTING ONLY
+# Normal emails are triggered automatically on moderation!
+# Note that emails are queued up in the MailQueue model, to be sent asynchronously
+# by the sender (or viewed locally).
+#
+#
+
+from django.core.management.base import BaseCommand, CommandError
+
+from pgweb.news.models import NewsArticle
+from pgweb.news.util import send_news_email
+
+
+def yesno(prompt):
+    while True:
+        r = input(prompt)
+        if r.lower().startswith('y'):
+            return True
+        elif r.lower().startswith('n'):
+            return False
+
+
+class Command(BaseCommand):
+    help = 'Test news email'
+
+    def add_arguments(self, parser):
+        parser.add_argument('id', type=int, help='id of news article to post')
+
+    def handle(self, *args, **options):
+        try:
+            news = NewsArticle.objects.get(pk=options['id'])
+        except NewsArticle.DoesNotExist:
+            raise CommandError("News article not found.")
+
+        print("Title: {}".format(news.title))
+        print("Moderation state: {}".format(news.modstate_string))
+        if not yesno('Proceed to send mail for this article?'):
+            raise CommandError("OK, aborting")
+
+        send_news_email(news)
+        print("Sent.")

pgweb/news/models.py

@@ -3,6 +3,8 @@ from datetime import date
 from pgweb.core.models import Organisation
 from pgweb.util.moderation import TristateModerateModel, ModerationState
 
+from .util import send_news_email
+
 
 class NewsTag(models.Model):
     urlname = models.CharField(max_length=20, null=False, blank=False, unique=True)
@@ -72,3 +74,6 @@ class NewsArticle(TristateModerateModel):
     def block_edit(self):
         # Don't allow editing of news articles that have been published
         return self.modstate in (ModerationState.PENDING, ModerationState.APPROVED)
+
+    def on_approval(self, request):
+        send_news_email(self)

pgweb/news/util.py

@@ -0,0 +1,67 @@
+from django.template.loader import get_template
+from django.conf import settings
+
+import os
+import hmac
+import hashlib
+
+from pgweb.mailqueue.util import send_simple_mail
+
+
+def _get_contenttype_from_extension(f):
+    _map = {
+        'png': 'image/png',
+        'jpg': 'image/jpeg',
+    }
+    e = os.path.splitext(f)[1][1:]
+    if e not in _map:
+        raise Exception("Unknown extension {}".format(e))
+    return _map[e]
+
+
+def send_news_email(news):
+    # To generate HTML email, pick a template based on the organisation and render it.
+
+    html = get_template('news/mail/{}.html'.format(news.org.mailtemplate)).render({
+        'news': news,
+    })
+
+    # Enumerate all files for this template, if any
+    attachments = []
+    basedir = os.path.abspath(os.path.join(settings.PROJECT_ROOT, '../templates/news/mail/img.{}'.format(news.org.mailtemplate)))
+    if os.path.isdir(basedir):
+        for f in os.listdir(basedir):
+            a = {
+                'contenttype': '{}; name={}'.format(_get_contenttype_from_extension(f), f),
+                'filename': f,
+                'disposition': 'inline; filename="{}"'.format(f),
+                'id': '<{}>'.format(f),
+            }
+            with open(os.path.join(basedir, f), "rb") as f:
+                a['content'] = f.read()
+            attachments.append(a)
+
+    # If configured to, add the tags and sign them so that a pglister delivery system can filter
+    # recipients based on it.
+    if settings.NEWS_MAIL_TAGKEY:
+        tagstr = ",".join([t.urlname for t in news.tags.all()])
+        h = hmac.new(tagstr.encode('ascii'), settings.NEWS_MAIL_TAGKEY.encode('ascii'), hashlib.sha256)
+        headers = {
+            'X-pglister-tags': tagstr,
+            'X-pglister-tagsig': h.hexdigest(),
+        }
+    else:
+        headers = {}
+
+    send_simple_mail(
+        settings.NEWS_MAIL_SENDER,
+        settings.NEWS_MAIL_RECEIVER,
+        news.title,
+        news.content,
+        replyto=news.org.email,
+        sendername="PostgreSQL news",  # XXX: Somehow special case based on organisation here as well?
+        receivername=settings.NEWS_MAIL_RECEIVER_NAME,
+        htmlbody=html,
+        attachments=attachments,
+        headers=headers,
+    )

pgweb/settings.py

@@ -153,6 +153,10 @@ BUGREPORT_EMAIL = "someone@example.com"                     # Address to pgsql-b
 BUGREPORT_NOREPLY_EMAIL = "someone-noreply@example.com"     # Address to no-reply pgsql-bugs address
 DOCSREPORT_EMAIL = "someone@example.com"                    # Address to pgsql-docs list
 DOCSREPORT_NOREPLY_EMAIL = "someone-noreply@example.com"    # Address to no-reply pgsql-docs address
+NEWS_MAIL_SENDER = "someone-noreply@example.com"            # Address news is sent from
+NEWS_MAIL_RECEIVER = "some-announce@example.com"            # Address news is sent to
+NEWS_MAIL_RECEIVER_NAME = "Some Announcement List"          # Name field for sending news
+NEWS_MAIL_TAGKEY = ""                                       # Key used to sign tags for pglister delivery
 FRONTEND_SERVERS = ()                                       # A tuple containing the *IP addresses* of all the
                                                             # varnish frontend servers in use.
 FTP_MASTERS = ()                                            # A tuple containing the *IP addresses* of all machines

requirements.txt

@@ -7,3 +7,4 @@ requests-oauthlib==0.4.0
 cvss==1.9
 pytidylib==0.3.2
 pycodestyle==2.4.0
+pynliner==0.8.0

templates/news/mail/base.html

@@ -0,0 +1,161 @@
+{%load pgfilters%}
+{%comment%}
+Base email template, not to be used individually.
+Original imported from https://github.com/leemunroe/responsive-html-email-template.
+MIT licensed. Local modifications are done, so care needs to be taken on an update.
+{%endcomment%}
+<!doctype html>
+<html>
+  <head>
+    <meta name="viewport" content="width=device-width">
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+    <title>{%block title%}{%endblock%}</title>
+    <style>
+{%comment%}
+    /* -------------------------------------
+        INLINED WITH htmlemail.io/inline
+    ------------------------------------- */
+    /* -------------------------------------
+        RESPONSIVE AND MOBILE FRIENDLY STYLES
+    ------------------------------------- */
+{%endcomment%}
+    @media only screen and (max-width: 620px) {
+      table[class=body] h1 {
+        font-size: 28px !important;
+        margin-bottom: 10px !important;
+      }
+      table[class=body] p,
+            table[class=body] ul,
+            table[class=body] ol,
+            table[class=body] td,
+            table[class=body] span,
+            table[class=body] a {
+        font-size: 16px !important;
+      }
+      table[class=body] .wrapper,
+            table[class=body] .article {
+        padding: 10px !important;
+      }
+      table[class=body] .content {
+        padding: 0 !important;
+      }
+      table[class=body] .container {
+        padding: 0 !important;
+        width: 100% !important;
+      }
+      table[class=body] .main {
+        border-left-width: 0 !important;
+        border-radius: 0 !important;
+        border-right-width: 0 !important;
+      }
+      table[class=body] .btn table {
+        width: 100% !important;
+      }
+      table[class=body] .btn a {
+        width: 100% !important;
+      }
+      table[class=body] .img-responsive {
+        height: auto !important;
+        max-width: 100% !important;
+        width: auto !important;
+      }
+    }
+{%comment%}
+    /* -------------------------------------
+        PRESERVE THESE STYLES IN THE HEAD
+    ------------------------------------- */
+{%endcomment%}
+    @media all {
+      .ExternalClass {
+        width: 100%;
+      }
+      .ExternalClass,
+            .ExternalClass p,
+            .ExternalClass span,
+            .ExternalClass font,
+            .ExternalClass td,
+            .ExternalClass div {
+        line-height: 100%;
+      }
+      .apple-link a {
+        color: inherit !important;
+        font-family: inherit !important;
+        font-size: inherit !important;
+        font-weight: inherit !important;
+        line-height: inherit !important;
+        text-decoration: none !important;
+      }
+      #MessageViewBody a {
+        color: inherit;
+        text-decoration: none;
+        font-size: inherit;
+        font-family: inherit;
+        font-weight: inherit;
+        line-height: inherit;
+      }
+      .btn-primary table td:hover {
+        background-color: #34495e !important;
+      }
+      .btn-primary a:hover {
+        background-color: #34495e !important;
+        border-color: #34495e !important;
+      }
+    }
+    </style>
+  </head>
+  <body class="" style="background-color: #f6f6f6; font-family: sans-serif; -webkit-font-smoothing: antialiased; font-size: 14px; line-height: 1.4; margin: 0; padding: 0; -ms-text-size-adjust: 100%; -webkit-text-size-adjust: 100%;">
+    <table border="0" cellpadding="0" cellspacing="0" class="body" style="border-collapse: separate; mso-table-lspace: 0pt; mso-table-rspace: 0pt; width: 100%; background-color: #f6f6f6;">
+      <tr>
+        <td style="font-family: sans-serif; font-size: 14px; vertical-align: top;">&nbsp;</td>
+        <td class="container" style="font-family: sans-serif; font-size: 14px; vertical-align: top; display: block; Margin: 0 auto; max-width: 580px; padding: 10px; width: 580px;">
+          <div class="content" style="box-sizing: border-box; display: block; Margin: 0 auto; max-width: 580px; padding: 10px;">
+
+{%comment%}            <!-- START CENTERED WHITE CONTAINER -->{%endcomment%}
+            <span class="preheader" style="color: transparent; display: none; height: 0; max-height: 0; max-width: 0; opacity: 0; overflow: hidden; mso-hide: all; visibility: hidden; width: 0;">{%block preheader%}{%endblock%}</span>
+            <table class="main" style="border-collapse: separate; mso-table-lspace: 0pt; mso-table-rspace: 0pt; width: 100%; background: #ffffff; border-radius: 3px;">
+
+{%comment%}              <!-- START MAIN CONTENT AREA -->{%endcomment%}
+              <tr>
+                <td class="wrapper" style="font-family: sans-serif; font-size: 14px; vertical-align: top; box-sizing: border-box; padding: 20px;">
+                  <table border="0" cellpadding="0" cellspacing="0" style="border-collapse: separate; mso-table-lspace: 0pt; mso-table-rspace: 0pt; width: 100%;">
+                    <tr>
+                      <td style="font-family: sans-serif; font-size: 14px; vertical-align: top;">
+{%inlinecss "news/mail/inline.css"%}
+{%block content%}{%endblock%}
+{%endinlinecss%}
+                      </td>
+                    </tr>
+                  </table>
+                </td>
+              </tr>
+{%comment%}
+            <!-- END MAIN CONTENT AREA -->{%endcomment%}
+            </table>
+{%comment%}
+            <!-- START FOOTER -->{%endcomment%}
+            <div class="footer" style="clear: both; Margin-top: 10px; text-align: center; width: 100%;">
+              <table border="0" cellpadding="0" cellspacing="0" style="border-collapse: separate; mso-table-lspace: 0pt; mso-table-rspace: 0pt; width: 100%;">
+                <tr>
+                  <td class="content-block" style="font-family: sans-serif; vertical-align: top; padding-bottom: 10px; padding-top: 10px; font-size: 12px; color: #999999; text-align: center;">
+                    <span class="apple-link" style="color: #999999; font-size: 12px; text-align: center;">{%block footer%}{%endblock%}</span>
+		    <br><br>{%block unsubscribe%}
+You were sent this email as a subscriber of the <em>pgsql-announce</em> mailinglist, for one
+of the content tag{{news.tags.all|pluralize}} {{news.tags.all|joinandor:"or"}}. To unsubscribe from
+further emails, please visit https://lists.postgresql.org/unsubscribe/.
+{%endblock%}
+                  </td>
+                </tr>
+              </table>
+            </div>
+{%comment%}
+            <!-- END FOOTER -->
+
+          <!-- END CENTERED WHITE CONTAINER -->
+{%endcomment%}
+          </div>
+        </td>
+        <td style="font-family: sans-serif; font-size: 14px; vertical-align: top;">&nbsp;</td>
+      </tr>
+    </table>
+  </body>
+</html>

templates/news/mail/default.html

@@ -0,0 +1,16 @@
+{%extends "news/mail/base.html"%}
+{%load markup%}
+{%block title%}{{news.title}}{%endblock%}
+
+{%block content%}
+<div>
+  <h1>{{news.title}}</h1>
+</div>
+{{news.content|markdown}}
+{%endblock%}
+
+{%block footer%}
+This email was sent to you from {{news.org}}. It was delivered on their behalf by
+the PostgreSQL project. Any questions about the content of the message should be
+sent to {{news.org}}.
+{%endblock%}

templates/news/mail/inline.css

@@ -0,0 +1,38 @@
+h1,
+h2,
+h3,
+h4 {
+    color: #000000;
+    font-family: sans-serif;
+    font-weight: 400;
+    line-height: 1.4;
+    margin: 0;
+    margin-bottom: 30px;
+}
+
+h1 {
+    font-size: 25px;
+    font-weight: 300;
+    text-align: center;
+}
+
+p,
+ul,
+ol {
+    font-family: sans-serif;
+    font-size: 14px;
+    font-weight: normal;
+    margin: 0;
+    margin-bottom: 15px;
+}
+p li,
+ul li,
+ol li {
+    list-style-position: inside;
+    margin-left: 5px;
+}
+
+a {
+    color: #3498db;
+    text-decoration: underline;
+}

templates/news/mail/pgproject.html

@@ -0,0 +1,16 @@
+{%extends "news/mail/base.html"%}
+{%load markup%}
+{%block title%}{{news.title}}{%endblock%}
+
+{%block content%}
+<div>
+  <img style="float: left" height="50" width="50" src="cid:slonik.png" alt="PostgreSQL logo">
+  <img style="float: right" height="50" width="50" src="cid:slonik.png" alt="PostgreSQL logo">
+  <h1>{{news.title}}</h1>
+</div>
+{{news.content|markdown}}
+{%endblock%}
+
+{%block footer%}
+This email was sent to you from the PostgreSQL project.
+{%endblock%}