First pass at implementing face clustering

Signed-off-by: Marcel Klehr <mklehr@gmx.net>

.eslintrc.js

@@ -4,5 +4,8 @@ module.exports = {
 	],
 	parserOptions: {
 		requireConfigFile: false,
+	},
+	rules: {
+		"node/no-unpublished-import": "off"
 	}
 }

appinfo/info.xml

@@ -54,6 +54,8 @@ The app does not send any sensitive data to cloud providers or similar services.
 	<settings>
 		<admin>OCA\Recognize\Settings\AdminSettings</admin>
 		<admin-section>OCA\Recognize\Settings\AdminSection</admin-section>
+		<personal>OCA\Recognize\Settings\UserSettings</personal>
+		<personal-section>OCA\Recognize\Settings\UserSection</personal-section>
 	</settings>
 
 	<repair-steps>

appinfo/routes.php

@@ -28,5 +28,6 @@ return [
 		['name' => 'admin#avx', 'url' => '/admin/avx', 'verb' => 'GET'],
 		['name' => 'admin#platform', 'url' => '/admin/platform', 'verb' => 'GET'],
 		['name' => 'admin#musl', 'url' => '/admin/musl', 'verb' => 'GET'],
+		['name' => 'user#update_cluster', 'url' => '/user/cluster/{id}', 'verb' => 'POST'],
 	],
 ];

composer.json

@@ -2,7 +2,8 @@
     "require": {
         "symfony/process": "^5.2",
         "ext-json": "*",
-        "php": ">=7.4"
+        "php": ">=7.4",
+        "rubix/ml": "^2.0"
     },
     "autoload": {
         "psr-4": {

lib/AppInfo/Application.php

@@ -21,7 +21,7 @@ class Application extends App implements IBootstrap {
 	}
 
 	public function register(IRegistrationContext $context): void {
-		// noop
+        @include_once __DIR__ . '/../../vendor/autoload.php';
 	}
 
 	public function boot(IBootContext $context): void {

lib/Classifiers/Images/FaceVectorsClassifier.php

@@ -0,0 +1,154 @@
+<?php
+/*
+ * Copyright (c) 2021. The Nextcloud Recognize contributors.
+ *
+ * This file is licensed under the Affero General Public License version 3 or later. See the COPYING file.
+ */
+
+namespace OCA\Recognize\Classifiers\Images;
+
+use OCA\Recognize\Db\FaceDetection;
+use OCA\Recognize\Db\FaceDetectionMapper;
+use OCA\Recognize\Service\Logger;
+use OCP\DB\Exception;
+use OCP\Files\File;
+use OCP\IConfig;
+use OCP\Files\InvalidPathException;
+use OCP\Files\NotFoundException;
+use Psr\Log\LoggerInterface;
+use Symfony\Component\Process\Exception\ProcessTimedOutException;
+use Symfony\Component\Process\Exception\RuntimeException;
+use Symfony\Component\Process\Process;
+
+class FaceVectorsClassifier {
+	public const IMAGE_TIMEOUT = 120; // seconds
+	public const IMAGE_PUREJS_TIMEOUT = 360; // seconds
+
+	private LoggerInterface $logger;
+
+	private IConfig $config;
+
+    private FaceDetectionMapper $faceDetections;
+
+    public function __construct(Logger $logger, IConfig $config, FaceDetectionMapper $faceDetections) {
+		$this->logger = $logger;
+		$this->config = $config;
+        $this->faceDetections = $faceDetections;
+    }
+
+	/**
+	 * @param File[] $files
+	 * @return void
+	 * @throws \OCP\Files\NotFoundException
+	 */
+	public function classify(array $files): void {
+		$paths = array_map(static function ($file) {
+			return $file->getStorage()->getLocalFile($file->getInternalPath());
+		}, $files);
+
+		$this->logger->debug('Classifying '.var_export($paths, true));
+
+		$command = [
+			$this->config->getAppValue('recognize', 'node_binary'),
+			dirname(__DIR__, 3) . '/src/classifier_facevectors.js',
+			'-'
+		];
+
+		$this->logger->debug('Running '.var_export($command, true));
+		$proc = new Process($command, __DIR__);
+		if ($this->config->getAppValue('recognize', 'tensorflow.gpu', 'false') === 'true') {
+			$proc->setEnv(['RECOGNIZE_GPU' => 'true']);
+		}
+		if ($this->config->getAppValue('recognize', 'tensorflow.purejs', 'false') === 'true') {
+			$proc->setEnv(['RECOGNIZE_PUREJS' => 'true']);
+			$proc->setTimeout(count($paths) * self::IMAGE_PUREJS_TIMEOUT);
+		} else {
+			$proc->setTimeout(count($paths) * self::IMAGE_TIMEOUT);
+		}
+		$proc->setInput(implode("\n", $paths));
+		try {
+			$proc->start();
+
+			// Set cores
+			$cores = $this->config->getAppValue('recognize', 'tensorflow.cores', '0');
+			if ($cores !== '0') {
+				@exec('taskset -cp '.implode(',', range(0, (int)$cores, 1)).' ' . $proc->getPid());
+			}
+
+			$i = 0;
+			$errOut = '';
+			foreach ($proc as $type => $data) {
+				if ($type !== $proc::OUT) {
+					$errOut .= $data;
+					$this->logger->debug('Classifier process output: '.$data);
+					continue;
+				}
+				$lines = explode("\n", $data);
+                $buffer = '';
+				foreach ($lines as $result) {
+					if (trim($result) === '') {
+						continue;
+					}
+                    $buffer .= $result;
+                    try {
+                        json_decode($buffer, true, 512, JSON_OBJECT_AS_ARRAY | JSON_THROW_ON_ERROR | JSON_INVALID_UTF8_IGNORE);
+                        $invalid = false;
+                    }catch(\JsonException $e) {
+                        $invalid = true;
+                    }
+                    if ($invalid) {
+                        continue;
+                    }
+					$this->logger->debug('Result for ' . $files[$i]->getName() . ' = ' . $buffer);
+					try {
+						// decode json
+						$faces = json_decode($buffer, true, 512, JSON_OBJECT_AS_ARRAY | JSON_THROW_ON_ERROR | JSON_INVALID_UTF8_IGNORE);
+
+						// assign tags
+						foreach($this->faceDetections->findByFileId($files[$i]->getId()) as $existingFaceDetection) {
+                            try {
+                                $this->faceDetections->delete($existingFaceDetection);
+                            } catch (Exception $e) {
+                                $this->logger->debug('Could not delete existing face detection');
+                            }
+                        }
+
+                        foreach($faces as $face) {
+                            $faceDetection = new FaceDetection();
+                            $faceDetection->setX($face['x']);
+                            $faceDetection->setY($face['y']);
+                            $faceDetection->setWidth($face['width']);
+                            $faceDetection->setHeight($face['height']);
+                            $faceDetection->setVector($face['vector']);
+                            $faceDetection->setFileId($files[$i]->getId());
+                            $faceDetection->setUserId($files[$i]->getOwner()->getUID());
+                            $this->faceDetections->insert($faceDetection);
+                        }
+					} catch (InvalidPathException $e) {
+						$this->logger->warning('File with invalid path encountered');
+					} catch (NotFoundException $e) {
+						$this->logger->warning('File to tag was not found');
+					} catch (\JsonException $e) {
+						$this->logger->warning('JSON exception');
+						$this->logger->warning($e->getMessage());
+						$this->logger->warning($result);
+					} catch (Exception $e) {
+                        $this->logger->warning('Could not create DB entry for face detection');
+                        $this->logger->warning($e->getMessage());
+                    }
+                    $i++;
+				}
+			}
+			if ($i !== count($files)) {
+				$this->logger->warning('Classifier process output: '.$errOut);
+				throw new \RuntimeException('Classifier process error');
+			}
+		} catch (ProcessTimedOutException $e) {
+			$this->logger->warning($proc->getErrorOutput());
+			throw new \RuntimeException('Classifier process timeout');
+		} catch (RuntimeException $e) {
+			$this->logger->warning($proc->getErrorOutput());
+			throw new \RuntimeException('Classifier process could not be started');
+		}
+	}
+}

lib/Command/ClassifyImages.php

@@ -3,6 +3,7 @@
 namespace OCA\Recognize\Command;
 
 use OCA\Recognize\Service\ClassifyImagesService;
+use OCA\Recognize\Service\FaceClusterAnalyzer;
 use OCA\Recognize\Service\Logger;
 use OCP\IConfig;
 use OCP\IUser;
@@ -19,15 +20,20 @@ class ClassifyImages extends Command {
 	private Logger $logger;
 
 	private IConfig $config;
+    /**
+     * @var \OCA\Recognize\Service\FaceClusterAnalyzer
+     */
+    private FaceClusterAnalyzer $faceClusterAnalyzer;
 
 
-	public function __construct(IUserManager $userManager, ClassifyImagesService $imageClassifier, Logger $logger, IConfig $config) {
+    public function __construct(IUserManager $userManager, ClassifyImagesService $imageClassifier, Logger $logger, IConfig $config, FaceClusterAnalyzer $faceClusterAnalyzer) {
 		parent::__construct();
 		$this->userManager = $userManager;
 		$this->imageClassifier = $imageClassifier;
 		$this->logger = $logger;
 		$this->config = $config;
-	}
+        $this->faceClusterAnalyzer = $faceClusterAnalyzer;
+    }
 
 	/**
 	 * Configure the command
@@ -61,6 +67,7 @@ class ClassifyImages extends Command {
 						$this->config->setAppValue('recognize', 'images.status', 'true');
 					}
 				} while ($anythingClassified);
+                $this->faceClusterAnalyzer->findClusters($user);
 			}
 		} catch (\Exception $ex) {
 			$this->config->setAppValue('recognize', 'images.status', 'false');

lib/Controller/UserController.php

@@ -0,0 +1,73 @@
+<?php
+
+namespace OCA\Recognize\Controller;
+
+use OCA\Recognize\Db\FaceClusterMapper;
+use OCA\Recognize\Db\FaceDetectionMapper;
+use OCA\Recognize\Service\TagManager;
+use OCP\AppFramework\Controller;
+use OCP\AppFramework\Http;
+use OCP\AppFramework\Http\JSONResponse;
+use OCP\DB\Exception;
+use OCP\IRequest;
+use OCP\IUserSession;
+
+class UserController extends Controller {
+
+    /**
+     * @var \OCA\Recognize\Db\FaceClusterMapper
+     */
+    private FaceClusterMapper $clusters;
+    /**
+     * @var \OCP\IUserSession
+     */
+    private IUserSession $userSession;
+    /**
+     * @var \OCA\Recognize\Db\FaceDetectionMapper
+     */
+    private FaceDetectionMapper $faceDetections;
+    /**
+     * @var \OCA\Recognize\Service\TagManager
+     */
+    private TagManager $tagManager;
+
+    public function __construct($appName, IRequest $request, FaceClusterMapper $clusters, IUserSession $userSession, FaceDetectionMapper $faceDetections, TagManager $tagManager) {
+		parent::__construct($appName, $request);
+        $this->clusters = $clusters;
+        $this->userSession = $userSession;
+        $this->faceDetections = $faceDetections;
+        $this->tagManager = $tagManager;
+    }
+
+    public function updateCluster(int $id, string $title) {
+        if (!$this->userSession->isLoggedIn()) {
+            return new JSONResponse([], Http::STATUS_FORBIDDEN);
+        }
+        /**
+         * @var $cluster \OCA\Recognize\Db\FaceCluster
+         */
+        try {
+            $cluster = $this->clusters->find($id);
+        }catch(Exception $e) {
+            return new JSONResponse([], Http::STATUS_INTERNAL_SERVER_ERROR);
+        }
+        if ($cluster->getUserId() !== $this->userSession->getUser()->getUID()) {
+            return new JSONResponse([], Http::STATUS_FORBIDDEN);
+        }
+        $oldTitle = $cluster->getTitle();
+        $cluster->setTitle($title);
+        try {
+            $this->clusters->update($cluster);
+        } catch (Exception $e) {
+            return new JSONResponse([], Http::STATUS_INTERNAL_SERVER_ERROR);
+        }
+        /**
+         * @var $detections \OCA\Recognize\Db\FaceDetection[]
+         */
+        $detections = $this->faceDetections->findByClusterId($cluster->getId());
+        foreach ($detections as $detection) {
+            $this->tagManager->assignFace($detection->getFileId(), $cluster->getTitle(), $oldTitle);
+        }
+        return new JSONResponse($cluster->toArray(), Http::STATUS_OK);
+    }
+}

lib/Db/FaceCluster.php

@@ -0,0 +1,36 @@
+<?php
+namespace OCA\Recognize\Db;
+
+use OCP\AppFramework\Db\Entity;
+
+/**
+ * Class FaceCluster
+ *
+ * @package OCA\Recognize\Db
+ * @method string getTitle()
+ * @method setTitle(string $title)
+ * @method string getUserId()
+ * @method setUserId(string $userId)
+ */
+class FaceCluster extends Entity {
+	protected $title;
+	protected $userId;
+
+	public static $columns = ['id', 'title', 'user_id'];
+	public static $fields = ['id', 'title', 'userId'];
+
+	public function __construct() {
+		// add types in constructor
+		$this->addType('id', 'integer');
+		$this->addType('title', 'string');
+		$this->addType('userId', 'string');
+	}
+
+	public function toArray(): array {
+		$array = [];
+		foreach (self::$fields as $field) {
+			$array[$field] = $this->{$field};
+		}
+		return $array;
+	}
+}

lib/Db/FaceClusterMapper.php

@@ -0,0 +1,44 @@
+<?php
+
+namespace OCA\Recognize\Db;
+
+use OCP\AppFramework\Db\QBMapper;
+use OCP\DB\QueryBuilder\IQueryBuilder;
+use OCP\IDBConnection;
+
+class FaceClusterMapper extends QBMapper {
+    public function __construct(IDBConnection $db) {
+        parent::__construct($db, 'recognize_face_clusters', FaceCluster::class);
+        $this->db = $db;
+    }
+
+    /**
+     * @throws \OCP\DB\Exception
+     */
+    function find(int $id): FaceCluster {
+        $qb = $this->db->getQueryBuilder();
+        $qb->select(FaceCluster::$columns)
+            ->from('recognize_face_clusters')
+            ->where($qb->expr()->eq('id', $qb->createPositionalParameter($id)));
+        return $this->findEntity($qb);
+    }
+
+    /**
+     * @throws \OCP\DB\Exception
+     */
+    function findByUserId(string $userId): array {
+        $qb = $this->db->getQueryBuilder();
+        $qb->select(FaceCluster::$columns)
+            ->from('recognize_face_clusters')
+            ->where($qb->expr()->eq('user_id', $qb->createPositionalParameter($userId)));
+        return $this->findEntities($qb);
+    }
+
+    public function assocFaceWithCluster(FaceDetection $faceDetection, FaceCluster $faceCluster) {
+        $qb = $this->db->getQueryBuilder();
+        $qb->insert('recognize_faces2clusters')->values([
+            'cluster_id' => $qb->createPositionalParameter($faceCluster->getId(), IQueryBuilder::PARAM_INT),
+            'face_detection_id' => $qb->createPositionalParameter($faceDetection->getId(), IQueryBuilder::PARAM_INT),
+        ])->executeStatement();
+    }
+}

lib/Db/FaceDetection.php

@@ -0,0 +1,52 @@
+<?php
+namespace OCA\Recognize\Db;
+
+use OCP\AppFramework\Db\Entity;
+
+/**
+ * Class FaceDetection
+ *
+ * @package OCA\Recognize\Db
+ * @method int getFileId()
+ * @method setFileId(int $fileId)
+ * @method int getX()
+ * @method int getY()
+ * @method int getHeight()
+ * @method int getWidth()
+ * @method setX(int $x)
+ * @method setY(int $y)
+ * @method setHeight(int $height)
+ * @method setWidth(int $width)
+ */
+class FaceDetection extends Entity {
+	protected $fileId;
+    protected $userId;
+	protected $x;
+	protected $y;
+	protected $height;
+	protected $width;
+	protected $vector;
+
+	public static $columns = ['id', 'user_id', 'file_id', 'x', 'y', 'height', 'width', 'vector'];
+	public static $fields = ['id', 'userId', 'fileId', 'x', 'y', 'height', 'width', 'vector'];
+
+	public function __construct() {
+		// add types in constructor
+		$this->addType('id', 'integer');
+		$this->addType('fileId', 'integer');
+		$this->addType('userId', 'string');
+		$this->addType('x', 'float');
+		$this->addType('y', 'float');
+		$this->addType('height', 'float');
+		$this->addType('width', 'float');
+		$this->addType('vector', 'json');
+	}
+
+	public function toArray(): array {
+		$array = [];
+		foreach (self::$fields as $field) {
+			$array[$field] = $this->{$field};
+		}
+		return $array;
+	}
+}

lib/Db/FaceDetectionMapper.php

@@ -0,0 +1,60 @@
+<?php
+
+namespace OCA\Recognize\Db;
+
+use OCP\AppFramework\Db\QBMapper;
+use OCP\DB\QueryBuilder\IQueryBuilder;
+use OCP\IDBConnection;
+
+class FaceDetectionMapper extends QBMapper {
+    public function __construct(IDBConnection $db) {
+        parent::__construct($db, 'recognize_face_detections', FaceDetection::class);
+        $this->db = $db;
+    }
+
+    /**
+     * @throws \OCP\AppFramework\Db\DoesNotExistException
+     * @throws \OCP\AppFramework\Db\MultipleObjectsReturnedException
+     * @throws \OCP\DB\Exception
+     */
+    public function find(int $id): FaceDetection {
+        $qb = $this->db->getQueryBuilder();
+        $qb
+            ->select(FaceDetection::$columns)
+            ->from('recognize_face_detections')
+            ->where($qb->expr()->eq('id', $qb->createNamedParameter($id)));
+
+        return $this->findEntity($qb);
+    }
+
+    public function findByClusterId(int $clusterId) : array {
+        $qb = $this->db->getQueryBuilder();
+        $qb->select(FaceDetection::$columns)
+            ->from('recognize_face_detections', 'd')
+            ->join('d', 'recognize_faces2clusters', 'c', 'd.id = c.face_detection_id')
+            ->where($qb->expr()->eq('c.cluster_id', $qb->createPositionalParameter($clusterId)));
+        return $this->findEntities($qb);
+    }
+
+    /**
+     * @throws \OCP\DB\Exception
+     */
+    function findByUserId(string $userId): array {
+        $qb = $this->db->getQueryBuilder();
+        $qb->select(FaceDetection::$columns)
+            ->from('recognize_face_detections')
+            ->where($qb->expr()->eq('user_id', $qb->createPositionalParameter($userId)));
+        return $this->findEntities($qb);
+    }
+
+    /**
+     * @throws \OCP\DB\Exception
+     */
+    function findByFileId(int $fileId): array {
+        $qb = $this->db->getQueryBuilder();
+        $qb->select(FaceDetection::$columns)
+            ->from('recognize_face_detections')
+            ->where($qb->expr()->eq('file_id', $qb->createPositionalParameter($fileId, IQueryBuilder::PARAM_INT)));
+        return $this->findEntities($qb);
+    }
+}

lib/Migration/Version002002000Date20220614094721.php

@@ -0,0 +1,121 @@
+<?php
+/*
+ * Copyright (c) 2020. The Nextcloud Bookmarks contributors.
+ *
+ * This file is licensed under the Affero General Public License version 3 or later. See the COPYING file.
+ */
+
+namespace OCA\Recognize\Migration;
+
+use Closure;
+use OCP\DB\ISchemaWrapper;
+use OCP\DB\Types;
+use OCP\IDBConnection;
+use OCP\Migration\IOutput;
+use OCP\Migration\SimpleMigrationStep;
+
+/**
+ * Auto-generated migration step: Please modify to your needs!
+ */
+class Version002002000Date20220614094721 extends SimpleMigrationStep {
+
+	/**
+	 * @param IOutput $output
+	 * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
+	 * @param array $options
+	 *
+	 * @return void
+	 */
+	public function preSchemaChange(IOutput $output, Closure $schemaClosure, array $options) {
+	}
+
+	/**
+	 * @param IOutput $output
+	 * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
+	 * @param array $options
+	 *
+	 * @return ISchemaWrapper
+	 */
+	public function changeSchema(IOutput $output, Closure $schemaClosure, array $options) {
+		/** @var ISchemaWrapper $schema */
+		$schema = $schemaClosure();
+
+		if (!$schema->hasTable('recognize_face_detections')) {
+			$table = $schema->createTable('recognize_face_detections');
+            $table->addColumn('id', 'bigint', [
+                'autoincrement' => true,
+                'notnull' => true,
+                'length' => 64,
+            ]);
+            $table->addColumn('user_id', 'string', [
+                'notnull' => false,
+                'length' => 64,
+            ]);
+			$table->addColumn('file_id', 'bigint', [
+				'notnull' => false,
+				'length' => 64,
+			]);
+            $table->addColumn('x', Types::FLOAT, [
+                'notnull' => false,
+            ]);
+            $table->addColumn('y', Types::FLOAT, [
+                'notnull' => false,
+            ]);
+            $table->addColumn('height', Types::FLOAT, [
+                'notnull' => false,
+            ]);
+            $table->addColumn('width', Types::FLOAT, [
+                'notnull' => false,
+            ]);
+			$table->addColumn('vector', Types::TEXT, [
+				'notnull' => true,
+			]);
+            $table->setPrimaryKey(['id'], 'recognize_facedet_id');
+		}
+
+		if (!$schema->hasTable('recognize_face_clusters')) {
+			$table = $schema->createTable('recognize_face_clusters');
+			$table->addColumn('id', 'bigint', [
+				'autoincrement' => true,
+				'notnull' => true,
+				'length' => 64,
+			]);
+			$table->addColumn('title', 'string', [
+				'notnull' => true,
+				'length' => 4000,
+				'default' => '',
+			]);
+			$table->addColumn('user_id', 'string', [
+				'notnull' => true,
+				'length' => 64,
+				'default' => '',
+			]);
+			$table->setPrimaryKey(['id'], 'recognize_faceclust_id');
+		}
+
+		if (!$schema->hasTable('recognize_faces2clusters')) {
+			$table = $schema->createTable('recognize_faces2clusters');
+			$table->addColumn('cluster_id', 'bigint', [
+				'notnull' => false,
+				'length' => 64,
+			]);
+			$table->addColumn('face_detection_id', 'bigint', [
+				'notnull' => false,
+				'length' => 64,
+			]);
+            $table->setPrimaryKey(['cluster_id', 'face_detection_id'], 'recognize_faces2clust');
+		}
+		return $schema;
+	}
+
+	/**
+	 * @param IOutput $output
+	 * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
+	 * @param array $options
+	 *
+	 * @return void
+	 */
+	public function postSchemaChange(IOutput $output, Closure $schemaClosure, array $options) {
+
+	}
+}

lib/Service/ClassifyImagesService.php

@@ -3,7 +3,7 @@
 namespace OCA\Recognize\Service;
 
 use OC\User\NoUserException;
-use OCA\Recognize\Classifiers\Images\FacesClassifier;
+use OCA\Recognize\Classifiers\Images\FaceVectorsClassifier;
 use OCA\Recognize\Classifiers\Images\GeoClassifier;
 use OCA\Recognize\Classifiers\Images\ImagenetClassifier;
 use OCA\Recognize\Classifiers\Images\LandmarksClassifier;
@@ -14,13 +14,12 @@ use OCP\Files\NotPermittedException;
 class ClassifyImagesService {
 	private ImagenetClassifier $imagenet;
 
-	private FacesClassifier $facenet;
+	private FaceVectorsClassifier $facenet;
 
 	private ImagesFinderService $imagesFinder;
 
 	private IRootFolder $rootFolder;
 
-	private ReferenceFacesFinderService $referenceFacesFinder;
 	/**
 	 * @var \Psr\Log\LoggerInterface
 	 */
@@ -32,12 +31,11 @@ class ClassifyImagesService {
 
 	private GeoClassifier $geo;
 
-	public function __construct(FacesClassifier $facenet, ImagenetClassifier $imagenet, IRootFolder $rootFolder, ImagesFinderService $imagesFinder, ReferenceFacesFinderService $referenceFacesFinder, Logger $logger, IConfig $config, LandmarksClassifier $landmarks, GeoClassifier $geo) {
+	public function __construct(FaceVectorsClassifier $facenet, ImagenetClassifier $imagenet, IRootFolder $rootFolder, ImagesFinderService $imagesFinder, Logger $logger, IConfig $config, LandmarksClassifier $landmarks, GeoClassifier $geo) {
 		$this->facenet = $facenet;
 		$this->imagenet = $imagenet;
 		$this->rootFolder = $rootFolder;
 		$this->imagesFinder = $imagesFinder;
-		$this->referenceFacesFinder = $referenceFacesFinder;
 		$this->logger = $logger;
 		$this->config = $config;
 		$this->landmarks = $landmarks;
@@ -87,18 +85,18 @@ class ClassifyImagesService {
 		}
 
 		if ($this->config->getAppValue('recognize', 'faces.enabled', 'false') !== 'false') {
-			$this->logger->debug('Collecting contact photos of user '.$user);
-			$faces = $this->referenceFacesFinder->findReferenceFacesForUser($user);
-			if (count($faces) === 0) {
-				$this->logger->debug('No contact photos found of user '.$user);
+			//$this->logger->debug('Collecting contact photos of user '.$user);
+			//$faces = $this->referenceFacesFinder->findReferenceFacesForUser($user);
+			/*if (count($faces) === 0) {
+				//$this->logger->debug('No contact photos found of user '.$user);
 				if ($this->config->getAppValue('recognize', 'imagenet.enabled', 'false') !== 'true') {
 					return false;
 				} else {
 					return true;
 				}
-			}
+			}*/
 			$this->logger->debug('Classifying photos of user '.$user. ' using facenet');
-			$this->facenet->classify($faces, $images);
+			$this->facenet->classify($images);
 		}
 		return true;
 	}

lib/Service/FaceClusterAnalyzer.php

@@ -0,0 +1,57 @@
+<?php
+
+namespace OCA\Recognize\Service;
+
+use OCA\Recognize\Db\FaceCluster;
+use OCA\Recognize\Db\FaceClusterMapper;
+use OCA\Recognize\Db\FaceDetection;
+use OCA\Recognize\Db\FaceDetectionMapper;
+use Rubix\ML\Clusterers\DBSCAN;
+use Rubix\ML\Datasets\Unlabeled;
+use Rubix\ML\Graph\Trees\BallTree;
+use Rubix\ML\Kernels\Distance\Euclidean;
+
+class FaceClusterAnalyzer {
+
+    private FaceDetectionMapper $faceDetections;
+
+    private FaceClusterMapper $faceClusters;
+
+    private TagManager $tagManager;
+
+    public function __construct(FaceDetectionMapper $faceDetections, FaceClusterMapper $faceClusters, TagManager $tagManager) {
+        $this->faceDetections = $faceDetections;
+        $this->faceClusters = $faceClusters;
+        $this->tagManager = $tagManager;
+    }
+
+    /**
+     * @throws \OCP\DB\Exception
+     * @throws \JsonException
+     */
+    public function findClusters(string $userId) {
+        /**
+         * @var $detections FaceDetection[]
+         */
+        $detections = $this->faceDetections->findByUserId($userId);
+        $dataset = new Unlabeled(array_map(function (FaceDetection $detection) {
+            return $detection->getVector();
+        }, $detections));
+        $clusterer = new DBSCAN(0.4, 4, new BallTree(20, new Euclidean()));
+        $results = $clusterer->predict($dataset);
+        $numClusters = max($results);
+
+        for($i = 0; $i <= $numClusters; $i++) {
+            $keys = array_keys($results, $i);
+            $cluster = new FaceCluster();
+            $cluster->setTitle('');
+            $cluster->setUserId($userId);
+            $cluster = $this->faceClusters->insert($cluster);
+            foreach ($keys as $key) {
+                $detection = $detections[$key];
+                $this->faceClusters->assocFaceWithCluster($detection, $cluster);
+                $this->tagManager->assignTags($detection->getFileId(), ['face'.$i]);
+            }
+        }
+    }
+}

lib/Service/TagManager.php

@@ -20,6 +20,10 @@ class TagManager {
 		$this->objectMapper = $objectMapper;
 	}
 
+    /**
+     * @param $name
+     * @return \OCP\SystemTag\ISystemTag
+     */
 	public function getTag($name) : ISystemTag {
 		try {
 			$tag = $this->tagManager->getTag($name, true, true);
@@ -29,6 +33,9 @@ class TagManager {
 		return $tag;
 	}
 
+    /**
+     * @return \OCP\SystemTag\ISystemTag
+     */
 	public function getProcessedTag(): ISystemTag {
 		try {
 			$tag = $this->tagManager->getTag(self::RECOGNIZED_TAG, false, false);
@@ -38,6 +45,25 @@ class TagManager {
 		return $tag;
 	}
 
+    /**
+     * @param int $fileId
+     * @param string $name
+     * @param string $oldName
+     * @return void
+     */
+    public function assignFace(int $fileId, string $name, string $oldName = '') {
+        if ($oldName) {
+            $this->getTag($oldName);
+            $this->objectMapper->unassignTags($fileId, 'files', [$this->getTag($oldName)->getId()]);
+        }
+        $this->assignTags($fileId, [$name]);
+    }
+
+    /**
+     * @param int $fileId
+     * @param array $tags
+     * @return void
+     */
 	public function assignTags(int $fileId, array $tags): void {
 		$tags = array_map(function ($tag) {
 			return $this->getTag($tag)->getId();
@@ -47,16 +73,26 @@ class TagManager {
 		$this->objectMapper->assignTags($fileId, 'files', array_unique(array_merge($tags, $oldTags)));
 	}
 
+    /**
+     * @param array $fileIds
+     * @return array
+     */
 	public function getTagsForFiles(array $fileIds): array {
 		return array_map(function ($tags) {
 			return $this->tagManager->getTagsByIds($tags);
 		}, $this->objectMapper->getTagIdsForObjects($fileIds, 'files'));
 	}
 
+    /**
+     * @return array
+     */
 	public function findClassifiedFiles(): array {
 		return $this->objectMapper->getObjectIdsForTags($this->getProcessedTag()->getId(), 'files');
 	}
 
+    /**
+     * @return array
+     */
 	public function findMissedClassifications(): array {
 		$classified = $this->findClassifiedFiles();
 		$classifiedChunks = array_chunk($classified, 999, true);
@@ -71,6 +107,9 @@ class TagManager {
 		return $missed;
 	}
 
+    /**
+     * @return void
+     */
 	public function resetClassifications(): void {
 		$fileIds = $this->findClassifiedFiles();
 		foreach ($fileIds as $id) {
@@ -79,6 +118,9 @@ class TagManager {
 		}
 	}
 
+    /**
+     * @return void
+     */
 	public function removeEmptyTags(): void {
 		$tags = $this->tagManager->getAllTags();
 		foreach ($tags as $tag) {

lib/Settings/UserSection.php

@@ -0,0 +1,57 @@
+<?php
+/*
+ * Copyright (c) 2021. The Nextcloud Recognize contributors.
+ *
+ * This file is licensed under the Affero General Public License version 3 or later. See the COPYING file.
+ */
+
+namespace OCA\Recognize\Settings;
+
+use OCP\IL10N;
+use OCP\IURLGenerator;
+use OCP\Settings\IIconSection;
+
+class UserSection implements IIconSection {
+	private IL10N $l;
+
+	private IURLGenerator $urlgen;
+
+	public function __construct(IL10N $l, IURLGenerator $urlgen) {
+		$this->l = $l;
+		$this->urlgen = $urlgen;
+	}
+
+	/**
+	 * returns the ID of the section. It is supposed to be a lower case string
+	 *
+	 *
+	 * @return string
+	 */
+	public function getID(): string {
+		return 'recognize';
+	}
+
+	/**
+	 * returns the translated name as it should be displayed, e.g. 'LDAP / AD
+	 * integration'. Use the L10N service to translate it.
+	 *
+	 * @return string
+	 */
+	public function getName(): string {
+		return $this->l->t('Recognize');
+	}
+
+	/**
+	 * @return string
+	 */
+	public function getIcon(): string {
+		return $this->urlgen->imagePath('recognize', 'recognize-black.svg');
+	}
+
+	/**
+	 * @return int whether the form should be rather on the top or bottom of the settings navigation. The sections are arranged in ascending order of the priority values. It is required to return a value between 0 and 99.
+	 */
+	public function getPriority(): int {
+		return 80;
+	}
+}

lib/Settings/UserSettings.php

@@ -0,0 +1,75 @@
+<?php
+/*
+ * Copyright (c) 2021. The Nextcloud Recognize contributors.
+ *
+ * This file is licensed under the Affero General Public License version 3 or later. See the COPYING file.
+ */
+
+namespace OCA\Recognize\Settings;
+
+use OCA\Recognize\Db\FaceCluster;
+use OCA\Recognize\Db\FaceClusterMapper;
+use OCA\Recognize\Db\FaceDetection;
+use OCA\Recognize\Db\FaceDetectionMapper;
+use OCP\AppFramework\Http\TemplateResponse;
+use OCP\AppFramework\Services\IInitialState;
+use OCP\IConfig;
+use OCP\IUserSession;
+use OCP\Settings\ISettings;
+
+class UserSettings implements ISettings {
+
+    private IInitialState $initialState;
+
+    private IConfig $config;
+
+    private FaceDetectionMapper $faceDetections;
+
+    private FaceClusterMapper $faceClusters;
+    /**
+     * @var \OCP\IUserSession
+     */
+    private IUserSession $userSession;
+
+    public function __construct(IInitialState $initialState, IConfig $config, FaceDetectionMapper $faceDetections, FaceClusterMapper $faceClusters, IUserSession $userSession)
+    {
+        $this->initialState = $initialState;
+        $this->config = $config;
+        $this->faceDetections = $faceDetections;
+        $this->faceClusters = $faceClusters;
+        $this->userSession = $userSession;
+    }
+
+
+    /**
+     * @return TemplateResponse
+     */
+    public function getForm(): TemplateResponse {
+        $faceClusters = $this->faceClusters->findByUserId($this->userSession->getUser()->getUID());
+        $faceClusters = array_map(function(FaceCluster $cluster) {
+            return [
+                'title' => $cluster->getTitle(),
+                'id' => $cluster->getId(),
+                'detections' => array_map(function(FaceDetection $detection) {
+                    return $detection->toArray();
+                }, $this->faceDetections->findByClusterId($cluster->getId())),
+            ];
+        }, $faceClusters);
+        $this->initialState->provideInitialState('faceClusters', $faceClusters);
+        return new TemplateResponse('recognize', 'user');
+    }
+
+    /**
+     * @return string the section ID, e.g. 'sharing'
+     */
+    public function getSection(): string {
+        return 'recognize';
+    }
+
+    /**
+     * @return int whether the form should be rather on the top or bottom of the admin section. The forms are arranged in ascending order of the priority values. It is required to return a value between 0 and 100.
+     */
+    public function getPriority(): int {
+        return 50;
+    }
+}

src/admin.js

@@ -1,8 +1,3 @@
-/*
- * Copyright (c) 2020. The Nextcloud Bookmarks contributors.
- *
- * This file is licensed under the Affero General Public License version 3 or later. See the COPYING file.
- */
 import Vue from 'vue'
 import Tooltip from '@nextcloud/vue/dist/Directives/Tooltip'
 import App from './components/ViewAdmin'

src/classifier_facevectors.js

@@ -0,0 +1,107 @@
+const path = require('path')
+const _ = require('lodash')
+const fs = require('fs/promises')
+
+let tf, faceapi, Jimp
+let PUREJS = false
+if (process.env.RECOGNIZE_PUREJS === 'true') {
+	tf = require('@tensorflow/tfjs')
+	require('@tensorflow/tfjs-backend-wasm')
+	faceapi = require('@vladmandic/face-api/dist/face-api.node-wasm.js')
+	Jimp = require('jimp')
+	PUREJS = true
+} else {
+	try {
+		if (false && process.env.RECOGNIZE_GPU === 'true') {
+			tf = require('@tensorflow/tfjs-node-gpu')
+			faceapi = require('@vladmandic/face-api/dist/face-api.node-gpu.js')
+		} else {
+			tf = require('@tensorflow/tfjs-node')
+			faceapi = require('@vladmandic/face-api/dist/face-api.node.js')
+		}
+	} catch (e) {
+		console.error(e)
+		console.error('Trying js-only mode')
+		tf = require('@tensorflow/tfjs')
+		require('@tensorflow/tfjs-backend-wasm')
+		faceapi = require('@vladmandic/face-api/dist/face-api.node-wasm.js')
+		Jimp = require('jimp')
+		PUREJS = true
+	}
+}
+
+if (process.argv.length < 3) throw new Error('Incorrect arguments: node classifier_faces.js ...<IMAGE_FILES> | node classify.js -')
+
+/**
+ *
+ */
+async function main() {
+	const getStdin = (await import('get-stdin')).default
+	let paths, facesDefinitionJSON
+	if (process.argv[2] === '-') {
+		paths = (await getStdin()).split('\n')
+	} else {
+		paths = process.argv.slice(2)
+	}
+
+	await faceapi.nets.ssdMobilenetv1.loadFromDisk(path.resolve(__dirname, '..', 'node_modules/@vladmandic/face-api/model'))
+	await faceapi.nets.faceLandmark68Net.loadFromDisk(path.resolve(__dirname, '..', 'node_modules/@vladmandic/face-api/model'))
+	await faceapi.nets.faceRecognitionNet.loadFromDisk(path.resolve(__dirname, '..', 'node_modules/@vladmandic/face-api/model'))
+
+	for (const path of paths) {
+		try {
+			let tensor
+			if (PUREJS) {
+				tensor = await createTensor(await Jimp.read(path), 3)
+			} else {
+				tensor = await tf.node.decodeImage(await fs.readFile(path), 3)
+			}
+			const results = await faceapi.detectAllFaces(tensor).withFaceLandmarks().withFaceDescriptors()
+			tensor.dispose()
+
+			const vectors = results
+				.map(result => ({
+					vector: result.descriptor,
+					x: result.detection.relativeBox.x,
+					y: result.detection.relativeBox.y,
+					height: result.detection.relativeBox.height,
+					width: result.detection.relativeBox.width,
+					score: result.detection.score,
+				}))
+
+			console.log(JSON.stringify(vectors))
+		} catch (e) {
+			console.error(e)
+			console.log('[]')
+		}
+	}
+}
+
+tf.setBackend(process.env.RECOGNIZE_PUREJS === 'true' ? 'wasm' : 'tensorflow')
+	.then(() => main())
+	.catch(e => {
+		console.error(e)
+	})
+
+/**
+ * @param image
+ */
+async function createTensor(image) {
+	const NUM_OF_CHANNELS = 3
+	const values = new Float32Array(image.bitmap.width * image.bitmap.height * NUM_OF_CHANNELS)
+	let i = 0
+	image.scan(0, 0, image.bitmap.width, image.bitmap.height, (x, y) => {
+		const pixel = Jimp.intToRGBA(image.getPixelColor(x, y))
+		values[i * NUM_OF_CHANNELS + 0] = pixel.r
+		values[i * NUM_OF_CHANNELS + 1] = pixel.g
+		values[i * NUM_OF_CHANNELS + 2] = pixel.b
+		i++
+	})
+	const outShape = [
+		image.bitmap.height,
+		image.bitmap.width,
+		NUM_OF_CHANNELS,
+	]
+	const imageTensor = tf.tensor3d(values, outShape, 'float32')
+	return imageTensor
+}

src/components/FaceCluster.vue

@@ -0,0 +1,82 @@
+<template>
+	<div class="face-cluster">
+		<h3 style="display: flex; flex-direction: row">
+			<template v-if="!editing">
+				<span style="padding: 10px 0"><strong>Person</strong> {{ cluster.title || 'Untitled' }}</span><Actions>
+					<ActionButton icon="icon-rename" @click="editing = true">
+						Edit
+					</ActionButton>
+				</Actions>
+			</template>
+			<template v-else>
+				<span style="padding: 10px 0"><strong>Person</strong> <input v-model="cluster.title"
+					v-focus
+					type="text"
+					@blur="editingDone"
+					@keydown.enter.stop.prevent="editingDone"></span>
+			</template>
+		</h3>
+		<div style="display: flex; flex-direction: row; overflow-x: scroll; width: 80vw;">
+			<FaceDetection v-for="detection in cluster.detections"
+				:key="detection.id"
+				:file-id="detection.fileId"
+				:x="detection.x"
+				:y="detection.y"
+				:width="detection.width"
+				:height="detection.height" />
+		</div>
+	</div>
+</template>
+
+<script>
+import ActionButton from '@nextcloud/vue/dist/Components/ActionButton'
+import Actions from '@nextcloud/vue/dist/Components/Actions'
+import FaceDetection from './FaceDetection.vue'
+import axios from '@nextcloud/axios'
+import { generateUrl } from '@nextcloud/router'
+export default {
+	name: 'FaceCluster',
+	components: { FaceDetection, Actions, ActionButton },
+	directives: {
+		focus(el) {
+			setTimeout(() => {
+				el.focus()
+			}, 100)
+		},
+	},
+	props: {
+		cluster: {
+			type: Object,
+			required: true,
+		},
+	},
+	data() {
+		return {
+			editing: false,
+			error: null,
+		}
+	},
+	watch: {
+		error(error) {
+			if (!error) return
+			OC.Notification.showTemporary(error)
+		},
+	},
+	methods: {
+		async editingDone() {
+			this.editing = false
+			try {
+				await axios.post(generateUrl(`/apps/recognize/user/cluster/${this.cluster.id}`), { title: this.cluster.title })
+			} catch (e) {
+				this.error = 'Failed to save name'
+			}
+		},
+	},
+}
+</script>
+
+<style scoped>
+.face-cluster {
+	padding-bottom: 20px;
+}
+</style>

src/components/FaceDetection.vue

@@ -0,0 +1,60 @@
+<template>
+	<div class="face-detection">
+		<img ref="image" :src="`/index.php/core/preview?fileId=${fileId}&x=256&y=256&a=true`" @load="onLoaded">
+		<canvas ref="canvas" />
+	</div>
+</template>
+
+<script>
+export default {
+	name: 'FaceDetection',
+	props: {
+		fileId: {
+			type: Number,
+			required: true,
+		},
+		x: {
+			type: Number,
+			required: true,
+		},
+		y: {
+			type: Number,
+			required: true,
+		},
+		height: {
+			type: Number,
+			required: true,
+		},
+		width: {
+			type: Number,
+			required: true,
+		},
+	},
+	methods: {
+		onLoaded() {
+			const canvas = this.$refs.canvas
+			const ctx = canvas.getContext('2d')
+			ctx.strokeStyle = this.colorPrimaryElementLight
+			const width = this.$refs.image.naturalWidth
+			const height = this.$refs.image.naturalHeight
+			ctx.strokeRect(width * this.x, height * this.y, width * this.width, height * this.height)
+		},
+	},
+}
+</script>
+
+<style scoped>
+.face-detection {
+	height: 256px;
+	width: 256px;
+	position: relative;
+	margin-right: 8px;
+}
+canvas {
+	position: absolute;
+	top: 0;
+	left: 0;
+	right: 0;
+	bottom: 0;
+}
+</style>

src/components/ViewUser.vue

@@ -0,0 +1,32 @@
+<!--
+  - Copyright (c) 2021. The Recognize contributors.
+  -
+  - This file is licensed under the Affero General Public License version 3 or later. See the COPYING file.
+  -->
+
+<template>
+	<div id="recognize">
+		<SettingsSection :title="t('recognize', 'Face recognition')">
+			<p>These people have been spotted by Recognize in your photos. You can give each person a name.</p>
+			<FaceCluster v-for="cluster in faceClusters" :key="cluster.id" :cluster="cluster" />
+		</SettingsSection>
+	</div>
+</template>
+
+<script>
+import SettingsSection from '@nextcloud/vue/dist/Components/SettingsSection'
+import { loadState } from '@nextcloud/initial-state'
+import FaceCluster from './FaceCluster'
+
+export default {
+	name: 'ViewUser',
+	components: { FaceCluster, SettingsSection },
+	data() {
+		return {
+			faceClusters: loadState('recognize', 'faceClusters'),
+		}
+	},
+}
+</script>
+<style>
+</style>

src/mixins/AppGlobal.js

@@ -10,9 +10,18 @@ export default {
 		n,
 	},
 	computed: {
+		colorPrimary() {
+			return getComputedStyle(document.documentElement).getPropertyValue('--color-primary')
+		},
+		colorPrimaryLight() {
+			return getComputedStyle(document.documentElement).getPropertyValue('--color-primary-light')
+		},
 		colorPrimaryElement() {
 			return getComputedStyle(document.documentElement).getPropertyValue('--color-primary-element')
 		},
+		colorPrimaryElementLight() {
+			return getComputedStyle(document.documentElement).getPropertyValue('--color-primary-element-light')
+		},
 		colorPrimaryText() {
 			return getComputedStyle(document.documentElement).getPropertyValue('--color-primary-text')
 		},

src/user.js

@@ -0,0 +1,12 @@
+import Vue from 'vue'
+import Tooltip from '@nextcloud/vue/dist/Directives/Tooltip'
+import App from './components/ViewUser'
+import AppGlobal from './mixins/AppGlobal'
+
+Vue.mixin(AppGlobal)
+Vue.directive('tooltip', Tooltip)
+
+global.Recognize = new Vue({
+	el: '#recognize',
+	render: h => h(App),
+})

templates/user.php

@@ -0,0 +1,4 @@
+<?php
+script('recognize', 'recognize-user');
+?>
+<div id="recognize"></div>

webpack.js

@@ -3,3 +3,4 @@ const webpackConfig = require('@nextcloud/webpack-vue-config')
 
 module.exports = webpackConfig
 webpackConfig.entry.admin = path.join(__dirname, 'src', 'admin.js')
+webpackConfig.entry.user = path.join(__dirname, 'src', 'user.js')