diff --git a/js/models/messagecollection.js b/js/models/messagecollection.js
index b7c950842..1316e6b31 100644
--- a/js/models/messagecollection.js
+++ b/js/models/messagecollection.js
@@ -21,21 +21,6 @@ define(function(require) {
 		model: Message,
 		comparator: function(message) {
 			return message.get('dateInt') * -1;
-		},
-		initialize: function() {
-			this.on('add', this._fetchAvatar);
-		},
-
-		_fetchAvatar: function(message) {
-			var url = OC.generateUrl('/apps/mail/api/avatars/url/{email}', {
-				email: message.get('fromEmail')
-			});
-
-			Promise.resolve($.ajax(url)).then(function() {
-				message.set('senderImage', OC.generateUrl('/apps/mail/api/avatars/image/{email}', {
-					email: message.get('fromEmail')
-				}));
-			}).catch(console.error.bind(this));
 		}
 	});
 
diff --git a/js/views/messagesitem.js b/js/views/messagesitem.js
index 3688d1435..b87720745 100644
--- a/js/views/messagesitem.js
+++ b/js/views/messagesitem.js
@@ -13,6 +13,7 @@ define(function(require) {
 
 	var $ = require('jquery');
 	var _ = require('underscore');
+	var OC = require('OC');
 	var Marionette = require('backbone.marionette');
 	var Radio = require('radio');
 	var MessageTemplate = require('templates/message-list-item.html');
@@ -74,7 +75,30 @@ define(function(require) {
 			});
 
 			$('.action.delete').tooltip({placement: 'left'});
+
+			this._fetchAvatar();
 		},
+
+		/**
+		 * @private
+		 * @returns {undefined}
+		 */
+		_fetchAvatar: function() {
+			var url = OC.generateUrl('/apps/mail/api/avatars/url/{email}', {
+				email: this.model.get('fromEmail')
+			});
+
+			Promise.resolve($.ajax(url)).then(function(avatar) {
+				if (avatar.isExternal) {
+					this.model.set('senderImage', OC.generateUrl('/apps/mail/api/avatars/image/{email}', {
+						email: this.model.get('fromEmail')
+					}));
+				} else {
+					this.model.set('senderImage', avatar.url);
+				}
+			}.bind(this));
+		},
+
 		toggleMessageStar: function(event) {
 			event.stopPropagation();
 
@@ -83,12 +107,12 @@ define(function(require) {
 			// directly change star state in the interface for quick feedback
 			if (starred) {
 				this.getUI('star')
-					.removeClass('icon-starred')
-					.addClass('icon-star');
+						.removeClass('icon-starred')
+						.addClass('icon-star');
 			} else {
 				this.getUI('star')
-					.removeClass('icon-star')
-					.addClass('icon-starred');
+						.removeClass('icon-star')
+						.addClass('icon-starred');
 			}
 
 			Radio.message.trigger('flag', this.model, 'flagged', !starred);
diff --git a/lib/Contracts/IAvatarService.php b/lib/Contracts/IAvatarService.php
index 0af36d309..fc685d6ce 100644
--- a/lib/Contracts/IAvatarService.php
+++ b/lib/Contracts/IAvatarService.php
@@ -33,12 +33,12 @@ interface IAvatarService {
 	 * @param string $uid
 	 * @return string|null URL, if one can be found
 	 */
-	public function getAvatarUrl($email, $uid);
+	public function getAvatar($email, $uid);
 
 	/**
 	 * @param string $email
 	 * @param string $uid
-	 * @return mixed|null image data
+	 * @return array|null image data
 	 */
 	public function getAvatarImage($email, $uid);
 }
diff --git a/lib/Controller/AvatarsController.php b/lib/Controller/AvatarsController.php
index 776023517..36c303fda 100644
--- a/lib/Controller/AvatarsController.php
+++ b/lib/Controller/AvatarsController.php
@@ -22,14 +22,13 @@
 
 namespace OCA\Mail\Controller;
 
-use OC;
 use OCA\Mail\Contracts\IAvatarService;
 use OCA\Mail\Http\AvatarDownloadResponse;
+use OCA\Mail\Http\JSONResponse;
 use OCP\AppFramework\Controller;
 use OCP\AppFramework\Http;
-use OCP\AppFramework\Http\JSONResponse;
 use OCP\AppFramework\Http\Response;
-use OCP\Files\IMimeTypeDetector;
+use OCP\AppFramework\Utility\ITimeFactory;
 use OCP\IRequest;
 
 class AvatarsController extends Controller {
@@ -40,25 +39,26 @@ class AvatarsController extends Controller {
 	/** @var string */
 	private $uid;
 
-	/** @var IMimeTypeDetector */
-	private $mimeDetector;
+	/** @var ITimeFactory */
+	private $timeFactory;
 
 	/**
 	 * @param string $appName
 	 * @param IRequest $request
 	 * @param IAvatarService $avatarService
-	 * @param IMimeTypeDetector $mimeDetector
 	 * @param string $UserId
+	 * @param ITimeFactory $timeFactory
 	 */
-	public function __construct($appName, IRequest $request, IAvatarService $avatarService, IMimeTypeDetector $mimeDetector, $UserId) {
+	public function __construct($appName, IRequest $request, IAvatarService $avatarService, $UserId, ITimeFactory $timeFactory) {
 		parent::__construct($appName, $request);
 		$this->avatarService = $avatarService;
-		$this->mimeDetector = $mimeDetector;
 		$this->uid = $UserId;
+		$this->timeFactory = $timeFactory;
 	}
 
 	/**
 	 * @NoAdminRequired
+	 * @NoCSRFRequired
 	 *
 	 * @param string $email
 	 * @return Response
@@ -68,24 +68,22 @@ class AvatarsController extends Controller {
 			return new JSONResponse([], Http::STATUS_BAD_REQUEST);
 		}
 
-		$avatarUrl = $this->avatarService->getAvatarUrl($email, $this->uid);
-		if (is_null($avatarUrl)) {
+		$avatar = $this->avatarService->getAvatar($email, $this->uid);
+		if (is_null($avatar)) {
 			// No avatar found
 			$response = new JSONResponse([], Http::STATUS_NOT_FOUND);
 
 			// Debounce this a bit
 			// (cache for one day)
-			$response->cacheFor(24 * 60 * 60);
+			$response->setCacheHeaders(24 * 60 * 60, $this->timeFactory);
 
 			return $response;
 		}
 
-		$response = new JSONResponse([
-			'url' => $avatarUrl,
-		]);
+		$response = new JSONResponse($avatar);
 
 		// Let the browser cache this for a week
-		$response->cacheFor(7 * 24 * 60 * 60);
+		$response->setCacheHeaders(7 * 24 * 60 * 60, $this->timeFactory);
 
 		return $response;
 	}
@@ -103,21 +101,28 @@ class AvatarsController extends Controller {
 		}
 
 		$imageData = $this->avatarService->getAvatarImage($email, $this->uid);
+		list($avatar, $image) = $imageData;
 
-		if (is_null($imageData)) {
+		if (is_null($imageData) || !$avatar->isExternal()) {
 			// This could happen if the cache invalidated meanwhile
-			$response = new Response();
-			$response->setStatus(Http::STATUS_NOT_FOUND);
-			// Clear cache
-			$response->cacheFor(0);
-			return $response;
+			return $this->noAvatarFoundResponse();
 		}
 
-		// TODO: limit to known MIME types
-		$mime = $this->mimeDetector->detectString($imageData);
-		$resp = new AvatarDownloadResponse($imageData);
-		$resp->addHeader('Content-Type', $mime);
+		$resp = new AvatarDownloadResponse($image);
+		$resp->addHeader('Content-Type', $avatar->getMime());
+
+		// Let the browser cache this for a week
+		$resp->setCacheHeaders(7 * 24 * 60 * 60, $this->timeFactory);
+
 		return $resp;
 	}
 
+	private function noAvatarFoundResponse() {
+		$response = new Response();
+		$response->setStatus(Http::STATUS_NOT_FOUND);
+		// Clear cache
+		$response->cacheFor(0);
+		return $response;
+	}
+
 }
diff --git a/lib/Http/AvatarDownloadResponse.php b/lib/Http/AvatarDownloadResponse.php
index a95e19359..6cc494834 100644
--- a/lib/Http/AvatarDownloadResponse.php
+++ b/lib/Http/AvatarDownloadResponse.php
@@ -25,6 +25,9 @@ use OCP\AppFramework\Http\DownloadResponse;
 
 class AvatarDownloadResponse extends DownloadResponse {
 
+	use CacheHeaders;
+
+	/** @var string */
 	private $content;
 
 	/**
@@ -33,8 +36,6 @@ class AvatarDownloadResponse extends DownloadResponse {
 	public function __construct($content) {
 		parent::__construct('avatar', 'application/octet-stream');
 		$this->content = $content;
-
-		$this->cacheFor(2 * 60 * 60);
 	}
 
 	/**
diff --git a/lib/Http/CacheHeaders.php b/lib/Http/CacheHeaders.php
new file mode 100644
index 000000000..b3dece80d
--- /dev/null
+++ b/lib/Http/CacheHeaders.php
@@ -0,0 +1,44 @@
+<?php
+
+/**
+ * @copyright 2017 Christoph Wurst <christoph@winzerhof-wurst.at>
+ *
+ * @author 2017 Christoph Wurst <christoph@winzerhof-wurst.at>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OCA\Mail\Http;
+
+use DateInterval;
+use DateTime;
+use OCP\AppFramework\Utility\ITimeFactory;
+
+trait CacheHeaders {
+
+	public function setCacheHeaders($cacheFor, ITimeFactory $timeFactory) {
+		$this->cacheFor(7 * 24 * 60 * 60);
+
+		$expires = new DateTime();
+		$expires->setTimestamp($timeFactory->getTime());
+		$expires->add(new DateInterval('PT' . $cacheFor . 'S'));
+		$this->addHeader('Expires', $expires->format(DateTime::RFC1123));
+
+		$this->addHeader('Pragma', 'cache');
+	}
+
+}
diff --git a/lib/Http/JSONResponse.php b/lib/Http/JSONResponse.php
new file mode 100644
index 000000000..1e6976d65
--- /dev/null
+++ b/lib/Http/JSONResponse.php
@@ -0,0 +1,32 @@
+<?php
+
+/**
+ * @copyright 2017 Christoph Wurst <christoph@winzerhof-wurst.at>
+ *
+ * @author 2017 Christoph Wurst <christoph@winzerhof-wurst.at>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OCA\Mail\Http;
+
+use OCP\AppFramework\Http\JSONResponse as Base;
+
+class JSONResponse extends Base {
+
+	use CacheHeaders;
+}
diff --git a/lib/Service/Avatar/AddressbookSource.php b/lib/Service/Avatar/AddressbookSource.php
index 4a751ab1f..44b920f92 100644
--- a/lib/Service/Avatar/AddressbookSource.php
+++ b/lib/Service/Avatar/AddressbookSource.php
@@ -41,11 +41,18 @@ class AddressbookSource implements IAvatarSource {
 	}
 
 	/**
-	 * @param string $email
-	 * @return string|null
+	 * @param string $email sender email address
+	 * @param AvatarFactory $factory
+	 * @return Avatar|null avatar URL if one can be found
 	 */
-	public function fetch($email) {
-		return $this->contactsIntegration->getPhoto($email);
+	public function fetch($email, AvatarFactory $factory) {
+		$url = $this->contactsIntegration->getPhoto($email);
+
+		if (is_null($url)) {
+			return null;
+		}
+
+		return $factory->createInternal($url, null);
 	}
 
 }
diff --git a/lib/Service/Avatar/Avatar.php b/lib/Service/Avatar/Avatar.php
new file mode 100644
index 000000000..25296b962
--- /dev/null
+++ b/lib/Service/Avatar/Avatar.php
@@ -0,0 +1,85 @@
+<?php
+
+/**
+ * @copyright 2017 Christoph Wurst <christoph@winzerhof-wurst.at>
+ *
+ * @author 2017 Christoph Wurst <christoph@winzerhof-wurst.at>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OCA\Mail\Service\Avatar;
+
+use JsonSerializable;
+
+class Avatar implements JsonSerializable {
+
+	/** @var string */
+	private $url;
+
+	/** @var string|null */
+	private $mime;
+
+	/** @var bool */
+	private $isExternal;
+
+	/**
+	 * @param string $url
+	 * @param string|null $mime
+	 * @param bool $isExternal
+	 */
+	public function __construct($url, $mime = null, $isExternal = true) {
+		$this->url = $url;
+		$this->mime = $mime;
+		$this->isExternal = $isExternal;
+	}
+
+	/**
+	 * @return string
+	 */
+	public function getUrl() {
+		return $this->url;
+	}
+
+	/**
+	 * Get the MIME type of this avatar
+	 *
+	 * @return string|null
+	 */
+	public function getMime() {
+		return $this->mime;
+	}
+
+	/**
+	 * @return boolean
+	 */
+	public function isExternal() {
+		return $this->isExternal;
+	}
+
+	/**
+	 * @return array
+	 */
+	public function jsonSerialize() {
+		return [
+			'isExternal' => $this->isExternal,
+			'mime' => $this->mime,
+			'url' => $this->url,
+		];
+	}
+
+}
diff --git a/lib/Service/Avatar/AvatarFactory.php b/lib/Service/Avatar/AvatarFactory.php
new file mode 100644
index 000000000..eeb8f5d81
--- /dev/null
+++ b/lib/Service/Avatar/AvatarFactory.php
@@ -0,0 +1,50 @@
+<?php
+
+/**
+ * @copyright 2017 Christoph Wurst <christoph@winzerhof-wurst.at>
+ *
+ * @author 2017 Christoph Wurst <christoph@winzerhof-wurst.at>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OCA\Mail\Service\Avatar;
+
+class AvatarFactory {
+
+	/**
+	 * Create a new avatar whose URL points to an internal endpoint
+	 *
+	 * @param string $url
+	 * @return Avatar
+	 */
+	public function createInternal($url) {
+		return new Avatar($url, null, false);
+	}
+
+	/**
+	 * Create a new avatar whose URL points to an external endpoint
+	 *
+	 * @param string $url
+	 * @param string $mime
+	 * @return Avatar
+	 */
+	public function createExternal($url, $mime) {
+		return new Avatar($url, $mime);
+	}
+
+}
diff --git a/lib/Service/Avatar/Cache.php b/lib/Service/Avatar/Cache.php
index 2017b2e96..8455671d3 100644
--- a/lib/Service/Avatar/Cache.php
+++ b/lib/Service/Avatar/Cache.php
@@ -35,11 +35,15 @@ class Cache {
 	/** @var ICache */
 	private $cache;
 
+	/** @var AvatarFactory */
+	private $avatarFactory;
+
 	/**
 	 * @param ICacheFactory $cacheFactory
 	 */
-	public function __construct(ICacheFactory $cacheFactory) {
+	public function __construct(ICacheFactory $cacheFactory, AvatarFactory $avatarFactory) {
 		$this->cache = $cacheFactory->create('mail.avatars');
+		$this->avatarFactory = $avatarFactory;
 	}
 
 	/**
@@ -62,18 +66,29 @@ class Cache {
 
 	/**
 	 * @param string $email
-	 * @return string|null avatar URL
+	 * @return Avatar|null avatar URL
 	 */
-	public function getUrl($email, $uid) {
-		return $this->cache->get($this->buildUrlKey($email, $uid));
+	public function get($email, $uid) {
+		$cached = $this->cache->get($this->buildUrlKey($email, $uid));
+
+		if (is_null($cached)) {
+			return null;
+		}
+
+		if ($cached['isExternal']) {
+			return $this->avatarFactory->createExternal($cached['url'], $cached['mime']);
+		} else {
+			return $this->avatarFactory->createInternal($cached['url'], $cached['mime']);
+		}
 	}
 
 	/**
 	 * @param string $email
-	 * @param string $url
+	 * @param string $uid
+	 * @param Avatar $avatar
 	 */
-	public function addUrl($email, $uid, $url) {
-		$this->cache->set($this->buildUrlKey($email, $uid), $url, self::CACHE_TTL);
+	public function add($email, $uid, Avatar $avatar) {
+		$this->cache->set($this->buildUrlKey($email, $uid), $avatar->jsonSerialize(), self::CACHE_TTL);
 	}
 
 	/**
diff --git a/lib/Service/Avatar/CompositeAvatarSource.php b/lib/Service/Avatar/CompositeAvatarSource.php
index e569d1315..344e63c77 100644
--- a/lib/Service/Avatar/CompositeAvatarSource.php
+++ b/lib/Service/Avatar/CompositeAvatarSource.php
@@ -47,15 +47,13 @@ class CompositeAvatarSource implements IAvatarSource {
 	}
 
 	/**
-	 * Find avatar URL with the help of avatar sources and return the first
-	 * valid result.
-	 *
-	 * @param string $email
-	 * @return string|null
+	 * @param string $email sender email address
+	 * @param AvatarFactory $factory
+	 * @return Avatar|null avatar URL if one can be found
 	 */
-	public function fetch($email) {
+	public function fetch($email, AvatarFactory $factory) {
 		foreach ($this->sources as $source) {
-			$avatar = $source->fetch($email);
+			$avatar = $source->fetch($email, $factory);
 
 			if (is_null($avatar)) {
 				continue;
diff --git a/lib/Service/Avatar/FaviconSource.php b/lib/Service/Avatar/FaviconSource.php
index 4adfd0cb5..9861592dd 100644
--- a/lib/Service/Avatar/FaviconSource.php
+++ b/lib/Service/Avatar/FaviconSource.php
@@ -25,6 +25,7 @@ namespace OCA\Mail\Service\Avatar;
 use Exception;
 use Horde_Mail_Rfc822_Address;
 use Mpclarkson\IconScraper\Scraper;
+use OCP\Files\IMimeTypeDetector;
 use OCP\Http\Client\IClientService;
 
 class FaviconSource implements IAvatarSource {
@@ -35,20 +36,25 @@ class FaviconSource implements IAvatarSource {
 	/** @var Scraper */
 	private $scraper;
 
+	/** @var IMimeTypeDetector */
+	private $mimeDetector;
+
 	/**
 	 * @param IClientService $clientService
 	 * @param Scraper $scraper
 	 */
-	public function __construct(IClientService $clientService, Scraper $scraper) {
+	public function __construct(IClientService $clientService, Scraper $scraper, IMimeTypeDetector $mimeDetector) {
 		$this->clientService = $clientService;
 		$this->scraper = $scraper;
+		$this->mimeDetector = $mimeDetector;
 	}
 
 	/**
-	 * @param string $email
-	 * @return string|null
+	 * @param string $email sender email address
+	 * @param AvatarFactory $factory
+	 * @return Avatar|null avatar URL if one can be found
 	 */
-	public function fetch($email) {
+	public function fetch($email, AvatarFactory $factory) {
 		$horde = new Horde_Mail_Rfc822_Address($email);
 		// TODO: fall back to insecure HTTP?
 		$domain = 'https://' . $horde->host;
@@ -77,8 +83,9 @@ class FaviconSource implements IAvatarSource {
 			if (strlen($body) === 0) {
 				continue;
 			}
+			$mime = $this->mimeDetector->detectString($body);
 
-			return $url;
+			return $factory->createExternal($url, $mime);
 		}
 
 		return null;
diff --git a/lib/Service/Avatar/GravatarSource.php b/lib/Service/Avatar/GravatarSource.php
index 99e5dbbcd..1c4132597 100644
--- a/lib/Service/Avatar/GravatarSource.php
+++ b/lib/Service/Avatar/GravatarSource.php
@@ -39,17 +39,18 @@ class GravatarSource implements IAvatarSource {
 	}
 
 	/**
-	 * @param string $email
-	 * @return string|null
+	 * @param string $email sender email address
+	 * @param AvatarFactory $factory
+	 * @return Avatar|null avatar URL if one can be found
 	 */
-	public function fetch($email) {
+	public function fetch($email, AvatarFactory $factory) {
 		$gravatar = new Gravatar(['size' => 128], true);
-		$avatar = $gravatar->avatar($email, ['d' => 404], true);
+		$avatarUrl = $gravatar->avatar($email, ['d' => 404], true);
 
 		$client = $this->clientService->newClient();
 
 		try {
-			$response = $client->get($avatar);
+			$response = $client->get($avatarUrl);
 		} catch (Exception $exception) {
 			return null;
 		}
@@ -60,7 +61,8 @@ class GravatarSource implements IAvatarSource {
 			return null;
 		}
 
-		return $avatar;
+		// TODO: check whether it's really always a jpeg
+		return $factory->createExternal($avatarUrl, 'image/jpeg');
 	}
 
 }
diff --git a/lib/Service/Avatar/IAvatarSource.php b/lib/Service/Avatar/IAvatarSource.php
index e3dae61a8..4be65d597 100644
--- a/lib/Service/Avatar/IAvatarSource.php
+++ b/lib/Service/Avatar/IAvatarSource.php
@@ -24,11 +24,14 @@
 
 namespace OCA\Mail\Service\Avatar;
 
+use OCP\Files\IMimeTypeDetector;
+
 interface IAvatarSource {
 
 	/**
 	 * @param string $email sender email address
-	 * @return string|null avatar URL if one can be found
+	 * @param AvatarFactory $factory
+	 * @return Avatar|null avatar URL if one can be found
 	 */
-	public function fetch($email);
+	public function fetch($email, AvatarFactory $factory);
 }
diff --git a/lib/Service/AvatarService.php b/lib/Service/AvatarService.php
index d70aad19b..f794bc556 100644
--- a/lib/Service/AvatarService.php
+++ b/lib/Service/AvatarService.php
@@ -22,6 +22,8 @@
 namespace OCA\Mail\Service;
 
 use OCA\Mail\Contracts\IAvatarService;
+use OCA\Mail\Service\Avatar\Avatar;
+use OCA\Mail\Service\Avatar\AvatarFactory;
 use OCA\Mail\Service\Avatar\Cache as AvatarCache;
 use OCA\Mail\Service\Avatar\CompositeAvatarSource;
 use OCA\Mail\Service\Avatar\Downloader;
@@ -42,67 +44,89 @@ class AvatarService implements IAvatarService {
 	/** @var IURLGenerator */
 	private $urlGenerator;
 
+	/** @var AvatarFactory */
+	private $avatarFactory;
+
 	/**
 	 * @param CompositeAvatarSource $source
 	 * @param Downloader $downloader
 	 * @param AvatarCache $cache
 	 * @param IURLGenerator $urlGenerator
+	 * @param AvatarFactory $avatarFactory
 	 */
-	public function __construct(CompositeAvatarSource $source, Downloader $downloader, AvatarCache $cache, IURLGenerator $urlGenerator) {
+	public function __construct(CompositeAvatarSource $source, Downloader $downloader, AvatarCache $cache, IURLGenerator $urlGenerator, AvatarFactory $avatarFactory) {
 		$this->source = $source;
 		$this->cache = $cache;
 		$this->urlGenerator = $urlGenerator;
 		$this->downloader = $downloader;
+		$this->avatarFactory = $avatarFactory;
+	}
+
+	/**
+	 * @param Avatar $avatar
+	 */
+	private function hasAllowedMime(Avatar $avatar) {
+		if ($avatar->isExternal()) {
+			$mime = $avatar->getMime();
+
+			return in_array($mime, [
+				'image/jpeg',
+				'image/png',
+			]);
+		} else {
+			// We trust internal URLs by default
+			return true;
+		}
 	}
 
 	/**
 	 * @param string $email
 	 * @param string $uid
-	 * @return string|null
+	 * @return Avatar|null
 	 */
-	public function getAvatarUrl($email, $uid) {
-		$cachedUrl = $this->cache->getUrl($email, $uid);
-		if (!is_null($cachedUrl)) {
-			return $cachedUrl;
+	public function getAvatar($email, $uid) {
+		$cachedAvatar = $this->cache->get($email, $uid);
+		if (!is_null($cachedAvatar)) {
+			return $cachedAvatar;
 		}
 
-		$url = $this->source->fetch($email);
-		if (is_null($url)) {
+		$avatar = $this->source->fetch($email, $this->avatarFactory);
+		if (is_null($avatar) || !$this->hasAllowedMime($avatar)) {
 			// Cannot locate any avatar -> nothing to do here
 			return null;
 		}
 
 		// Cache for the next call
-		$this->cache->addUrl($email, $uid, $url);
+		$this->cache->add($email, $uid, $avatar);
 
-		return $url;
+		return $avatar;
 	}
 
 	/**
 	 * @param string $email
 	 * @param string $uid
-	 * @return string|null image data
+	 * @return array|null image data
 	 */
 	public function getAvatarImage($email, $uid) {
-		$url = $this->getAvatarUrl($email, $uid);
-		if (is_null($url)) {
+		$avatar = $this->getAvatar($email, $uid);
+		if (is_null($avatar)) {
 			return null;
 		}
 
-		$cachedImage = $this->cache->getImage($url, $uid);
+		$cachedImage = $this->cache->getImage($avatar->getUrl(), $uid);
 		if (!is_null($cachedImage)) {
 			return base64_decode($cachedImage);
 		}
 
-		$image = $this->downloader->download($url);
+		$image = $this->downloader->download($avatar->getUrl());
 		if (is_null($image)) {
 			return null;
 		}
 
 		// Cache for the next call
-		$this->cache->addImage($url, $uid, base64_encode($image));
+		$this->cache->addImage($avatar->getUrl(), $uid, base64_encode($image));
 
-		return $image;
+		return [$avatar, $image];
 	}
 
 }
diff --git a/tests/Controller/AvatarControllerTest.php b/tests/Controller/AvatarControllerTest.php
index 6c4e32ffa..0026e2c56 100644
--- a/tests/Controller/AvatarControllerTest.php
+++ b/tests/Controller/AvatarControllerTest.php
@@ -27,10 +27,12 @@ namespace OCA\Mail\Tests\Controller;
 use OCA\Mail\Contracts\IAvatarService;
 use OCA\Mail\Controller\AvatarsController;
 use OCA\Mail\Http\AvatarDownloadResponse;
+use OCA\Mail\Http\JSONResponse;
+use OCA\Mail\Service\Avatar\Avatar;
 use OCA\Mail\Tests\TestCase;
 use OCP\AppFramework\Http;
-use OCP\AppFramework\Http\JSONResponse;
 use OCP\AppFramework\Http\Response;
+use OCP\AppFramework\Utility\ITimeFactory;
 use OCP\IRequest;
 use PHPUnit_Framework_MockObject_MockObject;
 
@@ -39,6 +41,9 @@ class AvatarControllerTest extends TestCase {
 	/** @var IAvatarService|PHPUnit_Framework_MockObject_MockObject */
 	private $avatarService;
 
+	/** @var ITimeFactory|PHPUnit_Framework_MockObject_MockObject */
+	private $timeFactory;
+
 	/** @var AvatarsController */
 	private $controller;
 
@@ -47,35 +52,40 @@ class AvatarControllerTest extends TestCase {
 
 		$request = $this->createMock(IRequest::class);
 		$this->avatarService = $this->createMock(IAvatarService::class);
+		$this->timeFactory = $this->createMocK(ITimeFactory::class);
+		$this->timeFactory->expects($this->any())
+			->method('getTime')
+			->willReturn(10000);
 
-		$this->controller = new AvatarsController('mail', $request, $this->avatarService, 'jane');
+		$this->controller = new AvatarsController('mail', $request, $this->avatarService, 'jane', $this->timeFactory);
 	}
 
 	public function testGetUrl() {
 		$email = 'john@doe.com';
+		$avatar = new Avatar('https://doe.com/favicon.ico');
 		$this->avatarService->expects($this->once())
-			->method('getAvatarUrl')
+			->method('getAvatar')
 			->with($email, 'jane')
-			->willReturn('https://doe.com/favicon.ico');
+			->willReturn($avatar);
 
 		$resp = $this->controller->url($email);
 
-		$expected = new JSONResponse(['url' => 'https://doe.com/favicon.ico']);
-		$expected->cacheFor(7 * 24 * 60 * 60);
+		$expected = new JSONResponse($avatar);
+		$expected->setCacheHeaders(7 * 24 * 60 * 60, $this->timeFactory);
 		$this->assertEquals($expected, $resp);
 	}
 
 	public function testGetUrlNoAvatarFound() {
 		$email = 'john@doe.com';
 		$this->avatarService->expects($this->once())
-			->method('getAvatarUrl')
+			->method('getAvatar')
 			->with($email, 'jane')
 			->willReturn(null);
 
 		$resp = $this->controller->url($email);
 
 		$expected = new JSONResponse([], Http::STATUS_NOT_FOUND);
-		$expected->cacheFor(24 * 60 * 60);
+		$expected->setCacheHeaders(24 * 60 * 60, $this->timeFactory);
 		$this->assertEquals($expected, $resp);
 	}
 
@@ -84,11 +94,13 @@ class AvatarControllerTest extends TestCase {
 		$this->avatarService->expects($this->once())
 			->method('getAvatarImage')
 			->with($email, 'jane')
-			->willReturn('data');
+			->willReturn([new Avatar('johne@doe.com', 'image/jpeg'), 'data']);
 
 		$resp = $this->controller->image($email);
 
 		$expected = new AvatarDownloadResponse('data');
+		$expected->addHeader('Content-Type', 'image/jpeg');
+		$expected->setCacheHeaders(7 * 24 * 60 * 60, $this->timeFactory);
 		$this->assertEquals($expected, $resp);
 	}
 
diff --git a/tests/Http/AvatarDownloadResponseTest.php b/tests/Http/AvatarDownloadResponseTest.php
new file mode 100644
index 000000000..07b7a3b20
--- /dev/null
+++ b/tests/Http/AvatarDownloadResponseTest.php
@@ -0,0 +1,39 @@
+<?php
+
+/**
+ * @copyright 2017 Christoph Wurst <christoph@winzerhof-wurst.at>
+ *
+ * @author 2017 Christoph Wurst <christoph@winzerhof-wurst.at>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OCA\Mail\Tests\Http;
+
+use OCA\Mail\Http\AvatarDownloadResponse;
+use OCA\Mail\Tests\TestCase;
+
+class AvatarDownloadResponseTest extends TestCase {
+
+	public function testResponse() {
+		$response = new AvatarDownloadResponse('data');
+
+		$this->assertEquals('application/octet-stream', $response->getHeaders()['Content-Type']);
+		$this->assertEquals('data', $response->render());
+	}
+
+}
diff --git a/tests/Integration/Service/AvatarServiceIntegrationTest.php b/tests/Integration/Service/AvatarServiceIntegrationTest.php
index 246f5ab88..970148ce6 100644
--- a/tests/Integration/Service/AvatarServiceIntegrationTest.php
+++ b/tests/Integration/Service/AvatarServiceIntegrationTest.php
@@ -51,17 +51,24 @@ class AvatarServiceIntegrationTest extends TestCase {
 	}
 
 	public function testJansGravatar() {
-		$avatar = $this->service->getAvatarUrl('hey@jancborchardt.net', 'john');
-		$this->assertEquals('https://secure.gravatar.com/avatar/2fd3f4d5d762955e5b603794a888fa97?size=128&d=404', $avatar);
+		$avatar = $this->service->getAvatar('hey@jancborchardt.net', 'john');
+		$this->assertNotNull($avatar);
+		$this->assertEquals('https://secure.gravatar.com/avatar/2fd3f4d5d762955e5b603794a888fa97?size=128&d=404', $avatar->getUrl());
 		$image = $this->service->getAvatarImage('hey@jancborchardt.net', 'john');
 		$this->assertNotNull($image);
 	}
 
 	public function testJansNicknameFavicon() {
-		$avatar = $this->service->getAvatarUrl('janli@nextcloud.com', 'jane');
-		$this->assertEquals('https://nextcloud.com/media/screenshot-150x150.png?x16328', $avatar);
+		$avatar = $this->service->getAvatar('janli@nextcloud.com', 'jane');
+		$this->assertNotNull($avatar);
+		$this->assertEquals('https://nextcloud.com/media/screenshot-150x150.png?x16328', $avatar->getUrl());
 		$image = $this->service->getAvatarImage('janli@nextcloud.com', 'jane');
 		$this->assertNotNull($image);
 	}
 
+	public function testChristophsFavicon() {
+		$avatar = $this->service->getAvatar('christoph@winzerhof-wurst.at', 'jan');
+		$this->assertNull($avatar); // There is none
+	}
+
 }
diff --git a/tests/Service/Avatar/AddressbookSourceTest.php b/tests/Service/Avatar/AddressbookSourceTest.php
index f85c483d2..8ca6a1ef8 100644
--- a/tests/Service/Avatar/AddressbookSourceTest.php
+++ b/tests/Service/Avatar/AddressbookSourceTest.php
@@ -25,6 +25,8 @@
 namespace OCA\Mail\Tests\Service\Avatar;
 
 use OCA\Mail\Service\Avatar\AddressbookSource;
+use OCA\Mail\Service\Avatar\Avatar;
+use OCA\Mail\Service\Avatar\AvatarFactory;
 use OCA\Mail\Service\ContactsIntegration;
 use OCA\Mail\Tests\TestCase;
 use PHPUnit_Framework_MockObject_MockObject;
@@ -47,22 +49,29 @@ class AddressbookSourceTest extends TestCase {
 
 	public function testFetch() {
 		$email = 'john@doe.com';
+		$avatarFactory = $this->createMock(AvatarFactory::class);
 		$this->ci->expects($this->once())
 			->method('getPhoto')
 			->willReturn('https://next.cloud/photo');
+		$avatar = new Avatar('https://next.cloud/photo');
+		$avatarFactory->expects($this->once())
+			->method('createInternal')
+			->with('https://next.cloud/photo')
+			->willReturn($avatar);
 
-		$avatar = $this->source->fetch($email);
+		$actualAvatar = $this->source->fetch($email, $avatarFactory);
 
-		$this->assertSame('https://next.cloud/photo', $avatar);
+		$this->assertSame($avatar, $actualAvatar);
 	}
 
 	public function testFetchNoneFound() {
 		$email = 'john@doe.com';
+		$avatarFactory = $this->createMock(AvatarFactory::class);
 		$this->ci->expects($this->once())
 			->method('getPhoto')
 			->willReturn(null);
 
-		$avatar = $this->source->fetch($email);
+		$avatar = $this->source->fetch($email, $avatarFactory);
 
 		$this->assertNull($avatar);
 	}
diff --git a/tests/Service/Avatar/CacheTest.php b/tests/Service/Avatar/CacheTest.php
index 8fa490870..06235916d 100644
--- a/tests/Service/Avatar/CacheTest.php
+++ b/tests/Service/Avatar/CacheTest.php
@@ -24,6 +24,8 @@
 
 namespace OCA\Mail\Tests\Service\Avatar;
 
+use OCA\Mail\Service\Avatar\Avatar;
+use OCA\Mail\Service\Avatar\AvatarFactory;
 use OCA\Mail\Service\Avatar\Cache;
 use OCA\Mail\Tests\TestCase;
 use OCP\ICache;
@@ -40,6 +42,9 @@ class CacheTest extends TestCase {
 	/** @var ICache|PHPUnit_Framework_MockObject_MockObject */
 	private $cacheImpl;
 
+	/** @var AvatarFactory|PHPUnit_Framework_MockObject_MockObject */
+	private $avatarFactory;
+
 	/** @var Cache */
 	private $cache;
 
@@ -52,10 +57,11 @@ class CacheTest extends TestCase {
 			->method('create')
 			->with('mail.avatars')
 			->willReturn($this->cacheImpl);
-		$this->cache = new Cache($this->cacheFactory);
+		$this->avatarFactory = $this->createMock(AvatarFactory::class);
+		$this->cache = new Cache($this->cacheFactory, $this->avatarFactory);
 	}
 
-	public function testGetNonCachedUrl() {
+	public function testGetNonCachedAvatar() {
 		$email = 'john@doe.com';
 		$uid = 'jane';
 		$this->cacheImpl->expects($this->once())
@@ -63,32 +69,38 @@ class CacheTest extends TestCase {
 			->with(base64_encode(json_encode([$email, $uid])))
 			->willReturn(null);
 
-		$url = $this->cache->getUrl($email, $uid);
+		$cachedAvatar = $this->cache->get($email, $uid);
 
-		$this->assertNull($url);
+		$this->assertNull($cachedAvatar);
 	}
 
-	public function testGetCachedUrl() {
+	public function testGetCachedAvatar() {
 		$email = 'john@doe.com';
 		$uid = 'jane';
 		$this->cacheImpl->expects($this->once())
 			->method('get')
 			->with(base64_encode(json_encode([$email, $uid])))
-			->willReturn('https://doe.com/favicon.ico');
+			->willReturn(['isExternal' => true, 'mime' => 'image/jpeg', 'url' => 'https://…']);
+		$expected = new Avatar('https://…', 'image/jpeg');
+		$this->avatarFactory->expects($this->once())
+			->method('createExternal')
+			->with('https://…', 'image/jpeg')
+			->willReturn($expected);
 
-		$url = $this->cache->getUrl($email, $uid);
+		$cachedAvatar = $this->cache->get($email, $uid);
 
-		$this->assertEquals('https://doe.com/favicon.ico', $url);
+		$this->assertEquals($expected, $cachedAvatar);
 	}
 
-	public function testSetUrl() {
+	public function testSetAvatar() {
 		$email = 'john@doe.com';
 		$uid = 'jane';
+		$avatar = new Avatar('https://…', 'image/jpeg');
 		$this->cacheImpl->expects($this->once())
 			->method('set')
-			->with(base64_encode(json_encode([$email, $uid])), 'https://doe.com/favicon.ico', 7 * 24 * 60 * 60);
+			->with(base64_encode(json_encode([$email, $uid])), ['isExternal' => true, 'mime' => 'image/jpeg', 'url' => 'https://…'], 7 * 24 * 60 * 60);
 
-		$this->cache->addUrl($email, $uid, 'https://doe.com/favicon.ico');
+		$this->cache->add($email, $uid, $avatar);
 	}
 
 	public function testGetImage() {
diff --git a/tests/Service/Avatar/CompositeAvatarSourceTest.php b/tests/Service/Avatar/CompositeAvatarSourceTest.php
index 41c0a6c7d..e3511c5ee 100644
--- a/tests/Service/Avatar/CompositeAvatarSourceTest.php
+++ b/tests/Service/Avatar/CompositeAvatarSourceTest.php
@@ -25,6 +25,8 @@
 namespace OCA\Mail\Tests\Service\Avatar;
 
 use OCA\Mail\Service\Avatar\AddressbookSource;
+use OCA\Mail\Service\Avatar\Avatar;
+use OCA\Mail\Service\Avatar\AvatarFactory;
 use OCA\Mail\Service\Avatar\CompositeAvatarSource;
 use OCA\Mail\Service\Avatar\FaviconSource;
 use OCA\Mail\Service\Avatar\GravatarSource;
@@ -57,40 +59,41 @@ class CompositeAvatarSourceTest extends TestCase {
 
 	public function testFetchNoneFound() {
 		$email = 'jane@doe.com';
-		$uid = 'john';
+		$avatarFactory = $this->createMock(AvatarFactory::class);
 		$this->addressbookSource->expects($this->once())
 			->method('fetch')
-			->with($email)
+			->with($email, $avatarFactory)
 			->willReturn(null);
 		$this->gravatarSource->expects($this->once())
 			->method('fetch')
-			->with($email)
+			->with($email, $avatarFactory)
 			->willReturn(null);
 		$this->faviconSource->expects($this->once())
 			->method('fetch')
-			->with($email)
+			->with($email, $avatarFactory)
 			->willReturn(null);
 
-		$avatar = $this->source->fetch($email, $uid);
+		$actualAvatar = $this->source->fetch($email, $avatarFactory);
 
-		$this->assertNull($avatar);
+		$this->assertNull($actualAvatar);
 	}
 
 	public function testFetchFromGravatar() {
 		$email = 'jane@doe.com';
-		$uid = 'john';
+		$avatar = new Avatar('https://gravatar.com', 'image/jpeg');
+		$avatarFactory = $this->createMock(AvatarFactory::class);
 		$this->addressbookSource->expects($this->once())
 			->method('fetch')
-			->with($email)
+			->with($email, $avatarFactory)
 			->willReturn(null);
 		$this->gravatarSource->expects($this->once())
 			->method('fetch')
-			->with($email)
-			->willReturn('https://gravatar.com');
+			->with($email, $avatarFactory)
+			->willReturn($avatar);
 
-		$avatar = $this->source->fetch($email, $uid);
+		$actualAvatar = $this->source->fetch($email, $avatarFactory);
 
-		$this->assertEquals('https://gravatar.com', $avatar);
+		$this->assertEquals($avatar, $actualAvatar);
 	}
 
 }
diff --git a/tests/Service/Avatar/FaviconSourceTest.php b/tests/Service/Avatar/FaviconSourceTest.php
index 0668e966f..3fe86370d 100644
--- a/tests/Service/Avatar/FaviconSourceTest.php
+++ b/tests/Service/Avatar/FaviconSourceTest.php
@@ -26,8 +26,11 @@ namespace OCA\Mail\Tests\Service\Avatar;
 
 use Mpclarkson\IconScraper\Icon;
 use Mpclarkson\IconScraper\Scraper;
+use OCA\Mail\Service\Avatar\Avatar;
+use OCA\Mail\Service\Avatar\AvatarFactory;
 use OCA\Mail\Service\Avatar\FaviconSource;
 use OCA\Mail\Tests\TestCase;
+use OCP\Files\IMimeTypeDetector;
 use OCP\Http\Client\IClient;
 use OCP\Http\Client\IClientService;
 use OCP\Http\Client\IResponse;
@@ -41,6 +44,9 @@ class FaviconSourceTest extends TestCase {
 	/** @var Scraper|PHPUnit_Framework_MockObject_MockObject */
 	private $scraper;
 
+	/** @var IMimeTypeDetector|PHPUnit_Framework_MockObject_MockObject */
+	private $mimeDetector;
+
 	/** @var FaviconSource */
 	private $source;
 
@@ -49,18 +55,20 @@ class FaviconSourceTest extends TestCase {
 
 		$this->clientService = $this->createMock(IClientService::class);
 		$this->scraper = $this->createMock(Scraper::class);
+		$this->mimeDetector = $this->createMock(IMimeTypeDetector::class);
 
-		$this->source = new FaviconSource($this->clientService, $this->scraper);
+		$this->source = new FaviconSource($this->clientService, $this->scraper, $this->mimeDetector);
 	}
 
 	public function testFetchNoIconsFound() {
 		$email = 'hey@jancborchardt.net';
+		$avatarFactory = $this->createMock(AvatarFactory::class);
 		$this->scraper->expects($this->once())
 			->method('get')
 			->with('https://jancborchardt.net')
 			->willReturn([]);
 
-		$avatar = $this->source->fetch($email);
+		$avatar = $this->source->fetch($email, $avatarFactory);
 
 		$this->assertNull($avatar);
 	}
@@ -68,6 +76,8 @@ class FaviconSourceTest extends TestCase {
 	public function testFetchSingleIcon() {
 		$email = 'hey@jancborchardt.net';
 		$icon = $this->createMock(Icon::class);
+		$avatarFactory = $this->createMock(AvatarFactory::class);
+		$avatar = new Avatar('https://domain.tld/favicon.ico');
 		$this->scraper->expects($this->once())
 			->method('get')
 			->with('https://jancborchardt.net')
@@ -87,15 +97,24 @@ class FaviconSourceTest extends TestCase {
 		$response->expects($this->once())
 			->method('getBody')
 			->willReturn('data');
+		$this->mimeDetector->expects($this->once())
+			->method('detectString')
+			->with('data')
+			->willReturn('image/png');
+		$avatarFactory->expects($this->once())
+			->method('createExternal')
+			->with('https://domain.tld/favicon.ico', 'image/png')
+			->willReturn($avatar);
 
-		$avatar = $this->source->fetch($email);
+		$actualAvatar = $this->source->fetch($email, $avatarFactory);
 
-		$this->assertSame('https://domain.tld/favicon.ico', $avatar);
+		$this->assertSame($avatar, $actualAvatar);
 	}
 
 	public function testFetchEmptyIcon() {
 		$email = 'hey@jancborchardt.net';
 		$icon = $this->createMock(Icon::class);
+		$avatarFactory = $this->createMock(AvatarFactory::class);
 		$this->scraper->expects($this->once())
 			->method('get')
 			->with('https://jancborchardt.net')
@@ -116,7 +135,7 @@ class FaviconSourceTest extends TestCase {
 			->method('getBody')
 			->willReturn('');
 
-		$avatar = $this->source->fetch($email);
+		$avatar = $this->source->fetch($email, $avatarFactory);
 
 		$this->assertNull($avatar);
 	}
diff --git a/tests/Service/Avatar/GravatarSouceTest.php b/tests/Service/Avatar/GravatarSouceTest.php
index ea5a1d7f1..db4274e5e 100644
--- a/tests/Service/Avatar/GravatarSouceTest.php
+++ b/tests/Service/Avatar/GravatarSouceTest.php
@@ -25,6 +25,8 @@
 namespace OCA\Mail\Tests\Service\Avatar;
 
 use Exception;
+use OCA\Mail\Service\Avatar\Avatar;
+use OCA\Mail\Service\Avatar\AvatarFactory;
 use OCA\Mail\Service\Avatar\GravatarSource;
 use OCA\Mail\Tests\TestCase;
 use OCP\Http\Client\IClient;
@@ -51,6 +53,7 @@ class GravatarSourceTest extends TestCase {
 	public function testFetchExisting() {
 		$email = 'hey@jancborchardt.net';
 		$client = $this->createMock(IClient::class);
+		$avatarFactory = $this->createMock(AvatarFactory::class);
 		$this->clientService->expects($this->once())
 			->method('newClient')
 			->willReturn($client);
@@ -62,15 +65,21 @@ class GravatarSourceTest extends TestCase {
 		$response->expects($this->once())
 			->method('getBody')
 			->willReturn('data');
+		$avatar = new Avatar('https://next.cloud/photo');
+		$avatarFactory->expects($this->once())
+			->method('createExternal')
+			->with('https://secure.gravatar.com/avatar/2fd3f4d5d762955e5b603794a888fa97?size=128&d=404')
+			->willReturn($avatar);
 
-		$avatar = $this->source->fetch($email);
+		$actualAvatar = $this->source->fetch($email, $avatarFactory);
 
-		$this->assertEquals('https://secure.gravatar.com/avatar/2fd3f4d5d762955e5b603794a888fa97?size=128&d=404', $avatar);
+		$this->assertEquals($avatar, $actualAvatar);
 	}
 
 	public function testFetchHttpError() {
 		$email = 'hey@jancborchardt.net';
 		$client = $this->createMock(IClient::class);
+		$avatarFactory = $this->createMock(AvatarFactory::class);
 		$this->clientService->expects($this->once())
 			->method('newClient')
 			->willReturn($client);
@@ -79,9 +88,9 @@ class GravatarSourceTest extends TestCase {
 			->with('https://secure.gravatar.com/avatar/2fd3f4d5d762955e5b603794a888fa97?size=128&d=404')
 			->willThrowException(new Exception());
 
-		$avatar = $this->source->fetch($email);
+		$actualAvatar = $this->source->fetch($email, $avatarFactory);
 
-		$this->assertNull($avatar);
+		$this->assertNull($actualAvatar);
 	}
 
 }
diff --git a/tests/Service/AvatarServiceTest.php b/tests/Service/AvatarServiceTest.php
index a6081498e..bf4bc60ce 100644
--- a/tests/Service/AvatarServiceTest.php
+++ b/tests/Service/AvatarServiceTest.php
@@ -24,6 +24,8 @@
 
 namespace OCA\Mail\Tests\Service;
 
+use OCA\Mail\Service\Avatar\Avatar;
+use OCA\Mail\Service\Avatar\AvatarFactory;
 use OCA\Mail\Service\Avatar\Cache;
 use OCA\Mail\Service\Avatar\CompositeAvatarSource;
 use OCA\Mail\Service\Avatar\Downloader;
@@ -49,6 +51,9 @@ class AvatarServiceTest extends TestCase {
 	/** @var IURLGenerator|PHPUnit_Framework_MockObject_MockObject */
 	private $urlGenerator;
 
+	/** @var AvatarFactory|PHPUnit_Framework_MockObject_MockObject */
+	private $avatarFactory;
+
 	/** @var AvatarService */
 	private $avatarService;
 
@@ -59,69 +64,92 @@ class AvatarServiceTest extends TestCase {
 		$this->downloader = $this->createMock(Downloader::class);
 		$this->cache = $this->createMock(Cache::class);
 		$this->urlGenerator = $this->createMock(IURLGenerator::class);
+		$this->avatarFactory = $this->createMock(AvatarFactory::class);
 
-		$this->avatarService = new AvatarService($this->source, $this->downloader, $this->cache, $this->urlGenerator);
+		$this->avatarService = new AvatarService($this->source, $this->downloader, $this->cache, $this->urlGenerator, $this->avatarFactory);
 	}
 
 	public function testGetCachedAvatarUrl() {
 		$email = 'jane@doe.com';
 		$uid = 'john';
 		$this->cache->expects($this->once())
-			->method('getUrl')
+			->method('get')
 			->with($email, $uid)
 			->willReturn('https://doe.com/favicon.ico');
 
-		$url = $this->avatarService->getAvatarUrl($email, $uid);
+		$url = $this->avatarService->getAvatar($email, $uid);
 
 		$this->assertEquals('https://doe.com/favicon.ico', $url);
 	}
 
-	public function testGetAvatarUrlNoAvatarFound() {
+	public function testGetAvatarNoAvatarFound() {
 		$email = 'jane@doe.com';
 		$uid = 'john';
 		$this->cache->expects($this->once())
-			->method('getUrl')
+			->method('get')
 			->with($email)
 			->willReturn(null);
 		$this->source->expects($this->once())
 			->method('fetch')
-			->with($email)
+			->with($email, $this->avatarFactory)
 			->willReturn(null);
 		$this->cache->expects($this->never())
-			->method('addUrl');
+			->method('add');
 
-		$url = $this->avatarService->getAvatarUrl($email, $uid);
+		$url = $this->avatarService->getAvatar($email, $uid);
 
 		$this->assertNull($url);
 	}
 
-	public function testGetAvatarUrl() {
+	public function testGetAvatarMimeNotAllowed() {
 		$email = 'jane@doe.com';
 		$uid = 'john';
 		$this->cache->expects($this->once())
-			->method('getUrl')
+			->method('get')
+			->with($email)
+			->willReturn(null);
+		$avatar = new Avatar('http://…', 'application/xml');
+		$this->source->expects($this->once())
+			->method('fetch')
+			->with($email, $this->avatarFactory)
+			->willReturn($avatar);
+		$this->cache->expects($this->never())
+			->method('add');
+
+		$url = $this->avatarService->getAvatar($email, $uid);
+
+		$this->assertNull($url);
+	}
+
+	public function testGetAvatar() {
+		$email = 'jane@doe.com';
+		$uid = 'john';
+		$avatar = new Avatar('https://doe.com/favicon.ico', 'image/png');
+		$this->cache->expects($this->once())
+			->method('get')
 			->with($email)
 			->willReturn(null);
 		$this->source->expects($this->once())
 			->method('fetch')
-			->with($email)
-			->willReturn('https://doe.com/favicon.ico');
+			->with($email, $this->avatarFactory)
+			->willReturn($avatar);
 		$this->cache->expects($this->once())
-			->method('addUrl')
-			->with($email, $uid, 'https://doe.com/favicon.ico');
+			->method('add')
+			->with($email, $uid, $avatar);
 
-		$url = $this->avatarService->getAvatarUrl($email, $uid);
+		$actualAvatar = $this->avatarService->getAvatar($email, $uid);
 
-		$this->assertEquals('https://doe.com/favicon.ico', $url);
+		$this->assertEquals($avatar, $actualAvatar);
 	}
 
 	public function testGetCachedAvatarImage() {
 		$email = 'jane@doe.com';
 		$uid = 'john';
+		$avatar = new Avatar('https://doe.com/favicon.ico', 'image/png');
 		$this->cache->expects($this->once())
-			->method('getUrl')
+			->method('get')
 			->with($email, $uid)
-			->willReturn('https://doe.com/favicon.ico');
+			->willReturn($avatar);
 		$this->cache->expects($this->once())
 			->method('getImage')
 			->with('https://doe.com/favicon.ico', $uid)
@@ -136,7 +164,7 @@ class AvatarServiceTest extends TestCase {
 		$email = 'jane@doe.com';
 		$uid = 'john';
 		$this->cache->expects($this->once())
-			->method('getUrl')
+			->method('get')
 			->with($email, $uid)
 			->willReturn(null);
 
@@ -148,10 +176,11 @@ class AvatarServiceTest extends TestCase {
 	public function testGetAvatarImageDownloadImage() {
 		$email = 'jane@doe.com';
 		$uid = 'john';
+		$avatar = new Avatar('https://doe.com/favicon.ico', 'image/jpg');
 		$this->cache->expects($this->once())
-			->method('getUrl')
+			->method('get')
 			->with($email, $uid)
-			->willReturn('https://doe.com/favicon.ico');
+			->willReturn($avatar);
 		$this->cache->expects($this->once())
 			->method('getImage')
 			->with('https://doe.com/favicon.ico', $uid)
@@ -164,18 +193,19 @@ class AvatarServiceTest extends TestCase {
 			->method('addImage')
 			->with('https://doe.com/favicon.ico', $uid, self::BLACK_DOT_BASE64);
 
-		$image = $this->avatarService->getAvatarImage($email, $uid);
+		$data = $this->avatarService->getAvatarImage($email, $uid);
 
-		$this->assertEquals(base64_decode(self::BLACK_DOT_BASE64), $image);
+		$this->assertEquals([$avatar, base64_decode(self::BLACK_DOT_BASE64)], $data);
 	}
 
 	public function testGetAvatarImageDownloadImageFails() {
 		$email = 'jane@doe.com';
 		$uid = 'john';
+		$avatar = new Avatar('https://doe.com/favicon.ico', 'image/jpg');
 		$this->cache->expects($this->once())
-			->method('getUrl')
+			->method('get')
 			->with($email, $uid)
-			->willReturn('https://doe.com/favicon.ico');
+			->willReturn($avatar);
 		$this->cache->expects($this->once())
 			->method('getImage')
 			->with('https://doe.com/favicon.ico', $uid)