Merge pull request #12298 from annando/api-suggestions

Contact suggestions are now cached

database.sql

@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 2022.12-dev (Giant Rhubarb)
--- DB_UPDATE_VERSION 1495
+-- DB_UPDATE_VERSION 1496
 -- ------------------------------------------
 
 
@@ -59,6 +59,7 @@ CREATE TABLE IF NOT EXISTS `user` (
 	`language` varchar(32) NOT NULL DEFAULT 'en' COMMENT 'default language',
 	`register_date` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'timestamp of registration',
 	`login_date` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'timestamp of last login',
+	`last-activity` date COMMENT 'Day of the last activity',
 	`default-location` varchar(255) NOT NULL DEFAULT '' COMMENT 'Default for item.location',
 	`allow_location` boolean NOT NULL DEFAULT '0' COMMENT '1 allows to display the location',
 	`theme` varchar(255) NOT NULL DEFAULT '' COMMENT 'user theme preference',
@@ -309,6 +310,20 @@ CREATE TABLE IF NOT EXISTS `2fa_trusted_browser` (
 	FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Two-factor authentication trusted browsers';
 
+--
+-- TABLE account-suggestion
+--
+CREATE TABLE IF NOT EXISTS `account-suggestion` (
+	`uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the account url',
+	`uid` mediumint unsigned NOT NULL COMMENT 'User ID',
+	`level` smallint unsigned COMMENT 'level of closeness',
+	`ignore` boolean NOT NULL DEFAULT '0' COMMENT 'If set, this account will not be suggested again',
+	 PRIMARY KEY(`uid`,`uri-id`),
+	 INDEX `uri-id_uid` (`uri-id`,`uid`),
+	FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
+	FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Account suggestion';
+
 --
 -- TABLE account-user
 --

doc/database.md

@@ -8,6 +8,7 @@ Database Tables
 | [2fa_app_specific_password](help/database/db_2fa_app_specific_password) | Two-factor app-specific _password |
 | [2fa_recovery_codes](help/database/db_2fa_recovery_codes) | Two-factor authentication recovery codes |
 | [2fa_trusted_browser](help/database/db_2fa_trusted_browser) | Two-factor authentication trusted browsers |
+| [account-suggestion](help/database/db_account-suggestion) | Account suggestion |
 | [account-user](help/database/db_account-user) | Remote and local accounts |
 | [addon](help/database/db_addon) | registered addons |
 | [apcontact](help/database/db_apcontact) | ActivityPub compatible contacts - used in the ActivityPub implementation |

doc/database/db_account-suggestion.md

@@ -0,0 +1,32 @@
+Table account-suggestion
+===========
+
+Account suggestion
+
+Fields
+------
+
+| Field  | Description                                                  | Type               | Null | Key | Default | Extra |
+| ------ | ------------------------------------------------------------ | ------------------ | ---- | --- | ------- | ----- |
+| uri-id | Id of the item-uri table entry that contains the account url | int unsigned       | NO   | PRI | NULL    |       |
+| uid    | User ID                                                      | mediumint unsigned | NO   | PRI | NULL    |       |
+| level  | level of closeness                                           | smallint unsigned  | YES  |     | NULL    |       |
+| ignore | If set, this account will not be suggested again             | boolean            | NO   |     | 0       |       |
+
+Indexes
+------------
+
+| Name       | Fields      |
+| ---------- | ----------- |
+| PRIMARY    | uid, uri-id |
+| uri-id_uid | uri-id, uid |
+
+Foreign Keys
+------------
+
+| Field | Target Table | Target Field |
+|-------|--------------|--------------|
+| uri-id | [item-uri](help/database/db_item-uri) | id |
+| uid | [user](help/database/db_user) | uid |
+
+Return to [database documentation](help/database)

doc/database/db_user.md

@@ -21,6 +21,7 @@ Fields
 | language                 | default language                                                                  | varchar(32)        | NO   |     | en                  |                |
 | register_date            | timestamp of registration                                                         | datetime           | NO   |     | 0001-01-01 00:00:00 |                |
 | login_date               | timestamp of last login                                                           | datetime           | NO   |     | 0001-01-01 00:00:00 |                |
+| last-activity            | Day of the last activity                                                          | date               | YES  |     | NULL                |                |
 | default-location         | Default for item.location                                                         | varchar(255)       | NO   |     |                     |                |
 | allow_location           | 1 allows to display the location                                                  | boolean            | NO   |     | 0                   |                |
 | theme                    | user theme preference                                                             | varchar(255)       | NO   |     |                     |                |

src/Factory/Api/Mastodon/Status.php

@@ -167,9 +167,23 @@ class Status extends BaseFactory
 		if (!empty($shared)) {
 			$shared_uri_id = $shared['post']['uri-id'];
 
-			$mentions    = array_merge($mentions, $this->mstdnMentionFactory->createFromUriId($shared_uri_id)->getArrayCopy());
+			foreach ($this->mstdnMentionFactory->createFromUriId($shared_uri_id)->getArrayCopy() as $mention) {
-			$tags        = array_merge($tags, $this->mstdnTagFactory->createFromUriId($shared_uri_id));
+				if (!in_array($mention, $mentions)) {
-			$attachments = array_merge($attachments, $this->mstdnAttachementFactory->createFromUriId($shared_uri_id));
+					$mentions[] = $mention;
+				}
+			}
+
+			foreach ($this->mstdnTagFactory->createFromUriId($shared_uri_id) as $tag) {
+				if (!in_array($tag, $tags)) {
+					$tags[] = $tag;
+				}
+			}
+
+			foreach ($this->mstdnAttachementFactory->createFromUriId($shared_uri_id) as $attachment) {
+				if (!in_array($attachment, $attachments)) {
+					$attachments[] = $attachment;
+				}
+			}
 
 			if (empty($card->toArray())) {
 				$card = $this->mstdnCardFactory->createFromUriId($shared_uri_id);

src/Model/Contact/Relation.php

@@ -260,6 +260,59 @@ class Relation
 		return true;
 	}
 
+	/**
+	 * Check if the cached suggestion is outdated
+	 *
+	 * @param integer $uid
+	 * @return boolean
+	 */
+	static public function areSuggestionsOutdated(int $uid): bool
+	{
+		return DI::pConfig()->get($uid, 'suggestion', 'last_update') + 3600 < time();
+	}
+
+	/**
+	 * Update contact suggestions for a given user
+	 *
+	 * @param integer $uid
+	 * @return void
+	 */
+	static public function updateCachedSuggestions(int $uid)
+	{
+		if (!self::areSuggestionsOutdated($uid)) {
+			return;
+		}
+
+		DBA::delete('account-suggestion', ['uid' => $uid, 'ignore' => false]);
+
+		foreach (self::getSuggestions($uid) as $contact) {
+			DBA::insert('account-suggestion', ['uri-id' => $contact['uri-id'], 'uid' => $uid, 'level' => 1], Database::INSERT_IGNORE);
+		}
+
+		DI::pConfig()->set($uid, 'suggestion', 'last_update', time());
+	}
+
+	/**
+	 * Returns a cached array of suggested contacts for given user id
+	 *
+	 * @param int $uid   User id
+	 * @param int $start optional, default 0
+	 * @param int $limit optional, default 80
+	 * @return array
+	 */
+	static public function getCachedSuggestions(int $uid, int $start = 0, int $limit = 80): array
+	{
+		$condition = ["`uid` = ? AND `uri-id` IN (SELECT `uri-id` FROM `account-suggestion` WHERE NOT `ignore` AND `uid` = ?)", 0, $uid];
+		$params = ['limit' => [$start, $limit]];
+		$cached = DBA::selectToArray('contact', [], $condition, $params);
+
+		if (!empty($cached)) {
+			return $cached;
+		} else {
+			return self::getSuggestions($uid, $start, $limit);
+		}
+	}
+
 	/**
 	 * Returns an array of suggested contacts for given user id
 	 *
@@ -270,6 +323,10 @@ class Relation
 	 */
 	static public function getSuggestions(int $uid, int $start = 0, int $limit = 80): array
 	{
+		if ($uid == 0) {
+			return [];
+		}
+
 		$cid = Contact::getPublicIdByUserId($uid);
 		$totallimit = $start + $limit;
 		$contacts = [];
@@ -285,11 +342,12 @@ class Relation
 				(SELECT `cid` FROM `contact-relation` WHERE `relation-cid` = ?)
 					AND NOT `cid` IN (SELECT `id` FROM `contact` WHERE `uid` = ? AND `nurl` IN
 						(SELECT `nurl` FROM `contact` WHERE `uid` = ? AND `rel` IN (?, ?))) AND `id` = `cid`)
-			AND NOT `hidden` AND `network` IN (?, ?, ?, ?)",
+			AND NOT `hidden` AND `network` IN (?, ?, ?, ?)
+			AND NOT `uri-id` IN (SELECT `uri-id` FROM `account-suggestion` WHERE `uri-id` = `contact`.`uri-id` AND `uid` = ?)",
 			$cid,
 			0,
 			$uid, Contact::FRIEND, Contact::SHARING,
-			Protocol::ACTIVITYPUB, Protocol::DFRN, $diaspora, $ostatus,
+			Protocol::ACTIVITYPUB, Protocol::DFRN, $diaspora, $ostatus, $uid
 			], [
 				'order' => ['last-item' => true],
 				'limit' => $totallimit,
@@ -315,9 +373,10 @@ class Relation
 				(SELECT `relation-cid` FROM `contact-relation` WHERE `cid` = ?)
 					AND NOT `cid` IN (SELECT `id` FROM `contact` WHERE `uid` = ? AND `nurl` IN
 						(SELECT `nurl` FROM `contact` WHERE `uid` = ? AND `rel` IN (?, ?))) AND `id` = `cid`)
-			AND NOT `hidden` AND `network` IN (?, ?, ?, ?)",
+			AND NOT `hidden` AND `network` IN (?, ?, ?, ?)
+			AND NOT `uri-id` IN (SELECT `uri-id` FROM `account-suggestion` WHERE `uri-id` = `contact`.`uri-id` AND `uid` = ?)",
 			$cid, 0, $uid, Contact::FRIEND, Contact::SHARING,
-			Protocol::ACTIVITYPUB, Protocol::DFRN, $diaspora, $ostatus],
+			Protocol::ACTIVITYPUB, Protocol::DFRN, $diaspora, $ostatus, $uid],
 			['order' => ['last-item' => true], 'limit' => $totallimit]
 		);
 
@@ -335,9 +394,10 @@ class Relation
 		// The query returns contacts that follow the given user but aren't followed by that user.
 		$results = DBA::select('contact', [],
 			["`nurl` IN (SELECT `nurl` FROM `contact` WHERE `uid` = ? AND `rel` = ?)
-			AND NOT `hidden` AND `uid` = ? AND `network` IN (?, ?, ?, ?)",
+			AND NOT `hidden` AND `uid` = ? AND `network` IN (?, ?, ?, ?)
+			AND NOT `uri-id` IN (SELECT `uri-id` FROM `account-suggestion` WHERE `uri-id` = `contact`.`uri-id` AND `uid` = ?)",
 			$uid, Contact::FOLLOWER, 0, 
-			Protocol::ACTIVITYPUB, Protocol::DFRN, $diaspora, $ostatus],
+			Protocol::ACTIVITYPUB, Protocol::DFRN, $diaspora, $ostatus, $uid],
 			['order' => ['last-item' => true], 'limit' => $totallimit]
 		);
 
@@ -355,9 +415,10 @@ class Relation
 		// The query returns any contact that isn't followed by that user.
 		$results = DBA::select('contact', [],
 			["NOT `nurl` IN (SELECT `nurl` FROM `contact` WHERE `uid` = ? AND `rel` IN (?, ?) AND `nurl` = `nurl`)
-			AND NOT `hidden` AND `uid` = ? AND `network` IN (?, ?, ?, ?)",
+			AND NOT `hidden` AND `uid` = ? AND `network` IN (?, ?, ?, ?)
+			AND NOT `uri-id` IN (SELECT `uri-id` FROM `account-suggestion` WHERE `uri-id` = `contact`.`uri-id` AND `uid` = ?)",
 			$uid, Contact::FRIEND, Contact::SHARING, 0, 
-			Protocol::ACTIVITYPUB, Protocol::DFRN, $diaspora, $ostatus],
+			Protocol::ACTIVITYPUB, Protocol::DFRN, $diaspora, $ostatus, $uid],
 			['order' => ['last-item' => true], 'limit' => $totallimit]
 		);
 

src/Model/User.php

@@ -665,6 +665,26 @@ class User
 		return $user;
 	}
 
+	/**
+	 * Update the day of the last activity of the given user
+	 *
+	 * @param integer $uid
+	 * @return void
+	 */
+	public static function updateLastActivity(int $uid)
+	{
+		$user = User::getById($uid, ['last-activity']);
+		if (empty($user)) {
+			return;
+		}
+
+		$current_day = DateTimeFormat::utcNow('Y-m-d');
+
+		if ($user['last-activity'] != $current_day) {
+			User::update(['last-activity' => $current_day], $uid);
+		}
+	}
+
 	/**
 	 * Generates a human-readable random password
 	 *

src/Module/Api/Mastodon/Accounts/Statuses.php

@@ -21,6 +21,7 @@
 
 namespace Friendica\Module\Api\Mastodon\Accounts;
 
+use Friendica\Core\Logger;
 use Friendica\Core\Protocol;
 use Friendica\Core\System;
 use Friendica\Database\DBA;
@@ -113,7 +114,11 @@ class Statuses extends BaseApi
 		$statuses = [];
 		while ($item = Post::fetch($items)) {
 			self::setBoundaries($item['uri-id']);
+			try {
 				$statuses[] = DI::mstdnStatus()->createFromUriId($item['uri-id'], $uid);
+			} catch (\Throwable $th) {
+				Logger::info('Post not fetchable', ['uri-id' => $item['uri-id'], 'uid' => $uid, 'error' => $th]);
+			}
 		}
 		DBA::close($items);
 

src/Module/Api/Mastodon/Suggestions.php

@@ -43,7 +43,7 @@ class Suggestions extends BaseApi
 			'limit' => 40, // Maximum number of results to return. Defaults to 40.
 		], $request);
 
-		$suggestions = Contact\Relation::getSuggestions($uid, 0, $request['limit']);
+		$suggestions = Contact\Relation::getCachedSuggestions($uid, 0, $request['limit']);
 
 		$accounts = [];
 

src/Module/Contact/Suggestions.php

@@ -57,7 +57,7 @@ class Suggestions extends \Friendica\BaseModule
 		$this->page['aside'] .= Widget::findPeople();
 		$this->page['aside'] .= Widget::follow();
 
-		$contacts = Contact\Relation::getSuggestions($this->session->getLocalUserId());
+		$contacts = Contact\Relation::getCachedSuggestions($this->session->getLocalUserId());
 		if (!$contacts) {
 			return $this->t('No suggestions available. If this is a new site, please try again in 24 hours.');
 		}

src/Security/Authentication.php

@@ -38,6 +38,8 @@ use Friendica\Util\DateTimeFormat;
 use Friendica\Util\Network;
 use LightOpenID;
 use Friendica\Core\L10n;
+use Friendica\Core\Worker;
+use Friendica\Model\Contact;
 use Psr\Log\LoggerInterface;
 
 /**
@@ -351,11 +353,16 @@ class Authentication
 		$this->setXAccMgmtStatusHeader($user_record);
 
 		if ($login_initial || $login_refresh) {
-			$this->dba->update('user', ['login_date' => DateTimeFormat::utcNow()], ['uid' => $user_record['uid']]);
+			$this->dba->update('user', ['last-activity' => DateTimeFormat::utcNow('Y-m-d'), 'login_date' => DateTimeFormat::utcNow()], ['uid' => $user_record['uid']]);
 
 			// Set the login date for all identities of the user
 			$this->dba->update('user', ['login_date' => DateTimeFormat::utcNow()],
 				['parent-uid' => $user_record['uid'], 'account_removed' => false]);
+
+			// Regularly update suggestions
+			if (Contact\Relation::areSuggestionsOutdated($user_record['uid'])) {
+				Worker::add(Worker::PRIORITY_MEDIUM, 'UpdateSuggestions', $user_record['uid']);
+			}
 		}
 
 		if ($login_initial) {

src/Security/OAuth.php

@@ -22,8 +22,11 @@
 namespace Friendica\Security;
 
 use Friendica\Core\Logger;
+use Friendica\Core\Worker;
 use Friendica\Database\Database;
 use Friendica\Database\DBA;
+use Friendica\Model\Contact;
+use Friendica\Model\User;
 use Friendica\Module\BaseApi;
 use Friendica\Util\DateTimeFormat;
 
@@ -100,6 +103,14 @@ class OAuth
 			return [];
 		}
 		Logger::debug('Token found', $token);
+
+		User::updateLastActivity($token['uid']);
+
+		// Regularly update suggestions
+		if (Contact\Relation::areSuggestionsOutdated($token['uid'])) {
+			Worker::add(Worker::PRIORITY_MEDIUM, 'UpdateSuggestions', $token['uid']);
+		}
+		
 		return $token;
 	}
 

src/Worker/Cron.php

@@ -130,6 +130,8 @@ class Cron
 
 			Worker::add(Worker::PRIORITY_LOW, 'CheckDeletedContacts');
 
+			Worker::add(Worker::PRIORITY_LOW, 'UpdateAllSuggestions');
+
 			if (DI::config()->get('system', 'optimize_tables')) {
 				Worker::add(Worker::PRIORITY_LOW, 'OptimizeTables');
 			}

src/Worker/UpdateAllSuggestions.php

@@ -0,0 +1,41 @@
+<?php
+/**
+ * @copyright Copyright (C) 2010-2022, the Friendica project
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace Friendica\Worker;
+
+use Friendica\Database\DBA;
+use Friendica\Model\Contact;
+use Friendica\Util\DateTimeFormat;
+
+/**
+ * Update contact suggestions for all aktive users
+ */
+class UpdateAllSuggestions
+{
+	public static function execute()
+	{
+		$users = DBA::select('user', ['uid'], ["`last-activity` > ?", DateTimeFormat::utc('now - 3 days', 'Y-m-d')]);
+		while ($user = DBA::fetch($users)) {
+			Contact\Relation::updateCachedSuggestions($user['uid']);
+		}
+		DBA::close($users);
+	}
+}

src/Worker/UpdateSuggestions.php

@@ -0,0 +1,35 @@
+<?php
+/**
+ * @copyright Copyright (C) 2010-2022, the Friendica project
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace Friendica\Worker;
+
+use Friendica\Model\Contact;
+
+/**
+ * Update contact suggestions
+ */
+class UpdateSuggestions
+{
+	public static function execute(int $uid)
+	{
+		Contact\Relation::updateCachedSuggestions($uid);
+	}
+}

static/dbstructure.config.php

@@ -55,7 +55,7 @@
 use Friendica\Database\DBA;
 
 if (!defined('DB_UPDATE_VERSION')) {
-	define('DB_UPDATE_VERSION', 1495);
+	define('DB_UPDATE_VERSION', 1496);
 }
 
 return [
@@ -115,6 +115,7 @@ return [
 			"language" => ["type" => "varchar(32)", "not null" => "1", "default" => "en", "comment" => "default language"],
 			"register_date" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "timestamp of registration"],
 			"login_date" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "timestamp of last login"],
+			"last-activity" => ["type" => "date", "comment" => "Day of the last activity"],
 			"default-location" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "Default for item.location"],
 			"allow_location" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "1 allows to display the location"],
 			"theme" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "user theme preference"],
@@ -371,6 +372,18 @@ return [
 			"uid" => ["uid"],
 		]
 	],
+	"account-suggestion" => [
+		"comment" => "Account suggestion",
+		"fields" => [
+			"uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the account url"],
+			"uid" => ["type" => "mediumint unsigned", "not null" => "1", "primary" => "1", "foreign" => ["user" => "uid"], "comment" => "User ID"],
+			"level" => ["type" => "smallint unsigned", "comment" => "level of closeness"],
+			"ignore" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "If set, this account will not be suggested again"],		],
+		"indexes" => [
+			"PRIMARY" => ["uid", "uri-id"],
+			"uri-id_uid" => ["uri-id", "uid"],
+		]
+	],
 	"account-user" => [
 		"comment" => "Remote and local accounts",
 		"fields" => [

view/theme/vier/theme.php

@@ -144,7 +144,7 @@ function vier_community_info()
 
 	// comunity_profiles
 	if ($show_profiles) {
-		$contacts = Contact\Relation::getSuggestions(DI::userSession()->getLocalUserId(), 0, 9);
+		$contacts = Contact\Relation::getCachedSuggestions(DI::userSession()->getLocalUserId(), 0, 9);
 
 		$tpl = Renderer::getMarkupTemplate('ch_directory_item.tpl');
 		if (DBA::isResult($contacts)) {