From e5c24f33f0c3c51a153a23612fed88be660d0de0 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 18 Sep 2022 15:40:44 +0200 Subject: [PATCH 1/2] The fcontact table is now updated in the background to improve performance --- database.sql | 3 ++- doc/database/db_fcontact.md | 1 + src/Model/Contact.php | 22 ++++++++++++---------- src/Model/FContact.php | 29 ++++++++++++++++------------- src/Module/NoScrape.php | 25 +++---------------------- src/Network/Probe.php | 12 ++++++++++++ static/dbstructure.config.php | 3 ++- 7 files changed, 48 insertions(+), 47 deletions(-) diff --git a/database.sql b/database.sql index b76a4e020..010b64542 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 2022.09-rc (Giant Rhubarb) --- DB_UPDATE_VERSION 1482 +-- DB_UPDATE_VERSION 1483 -- ------------------------------------------ @@ -625,6 +625,7 @@ CREATE TABLE IF NOT EXISTS `fcontact` ( `network` char(4) NOT NULL DEFAULT '' COMMENT '', `alias` varbinary(383) NOT NULL DEFAULT '' COMMENT '', `pubkey` text COMMENT '', + `created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '', `updated` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '', `interacting_count` int unsigned DEFAULT 0 COMMENT 'Number of contacts this contact interactes with', `interacted_count` int unsigned DEFAULT 0 COMMENT 'Number of contacts that interacted with this contact', diff --git a/doc/database/db_fcontact.md b/doc/database/db_fcontact.md index d082b1717..095c47aaf 100644 --- a/doc/database/db_fcontact.md +++ b/doc/database/db_fcontact.md @@ -25,6 +25,7 @@ Fields | network | | char(4) | NO | | | | | alias | | varbinary(383) | NO | | | | | pubkey | | text | YES | | NULL | | +| created | | datetime | NO | | 0001-01-01 00:00:00 | | | updated | | datetime | NO | | 0001-01-01 00:00:00 | | | interacting_count | Number of contacts this contact interactes with | int unsigned | YES | | 0 | | | interacted_count | Number of contacts that interacted with this contact | int unsigned | YES | | 0 | | diff --git a/src/Model/Contact.php b/src/Model/Contact.php index 6a5769399..1f5dbcdca 100644 --- a/src/Model/Contact.php +++ b/src/Model/Contact.php @@ -252,7 +252,7 @@ class Contact // Add internal fields $removal = []; if (!empty($fields)) { - foreach (['id', 'avatar', 'created', 'updated', 'last-update', 'success_update', 'failure_update', 'network'] as $internal) { + foreach (['id', 'next-update', 'network'] as $internal) { if (!in_array($internal, $fields)) { $fields[] = $internal; $removal[] = $internal; @@ -282,9 +282,8 @@ class Contact } // Update the contact in the background if needed - $updated = max($contact['success_update'], $contact['created'], $contact['updated'], $contact['last-update'], $contact['failure_update']); - if (($updated < DateTimeFormat::utc('now -7 days')) && in_array($contact['network'], Protocol::FEDERATED) && !self::isLocalById($contact['id'])) { - Worker::add(PRIORITY_LOW, "UpdateContact", $contact['id']); + if (Probe::isProbable($contact['network']) && ($contact['next-update'] < DateTimeFormat::utcNow())) { + Worker::add(['priority' => PRIORITY_LOW, 'dont_fork' => true], 'UpdateContact', $contact['id']); } // Remove the internal fields @@ -935,7 +934,6 @@ class Contact DI::cache()->delete(ActivityPub\Transmitter::CACHEKEY_CONTACTS . 'followers:' . $uid); DI::cache()->delete(ActivityPub\Transmitter::CACHEKEY_CONTACTS . 'following:' . $uid); - DI::cache()->delete(NoScrape::CACHEKEY . $uid); } /** @@ -1194,11 +1192,15 @@ class Contact return 0; } - $contact = self::getByURL($url, false, ['id', 'network', 'uri-id'], $uid); + $contact = self::getByURL($url, false, ['id', 'network', 'uri-id', 'next-update'], $uid); if (!empty($contact)) { $contact_id = $contact['id']; + if (Probe::isProbable($contact['network']) && ($contact['next-update'] < DateTimeFormat::utcNow())) { + Worker::add(['priority' => PRIORITY_LOW, 'dont_fork' => true], 'UpdateContact', $contact['id']); + } + if (empty($update) && (!empty($contact['uri-id']) || is_bool($update))) { Logger::debug('Contact found', ['url' => $url, 'uid' => $uid, 'update' => $update, 'cid' => $contact_id]); return $contact_id; @@ -2460,7 +2462,7 @@ class Contact $has_local_data = self::hasLocalData($id, $contact); - if (!in_array($ret['network'], array_merge(Protocol::FEDERATED, [Protocol::ZOT, Protocol::PHANTOM]))) { + if (!Probe::isProbable($ret['network'])) { // Periodical checks are only done on federated contacts $failed_next_update = null; $success_next_update = null; @@ -3320,12 +3322,12 @@ class Contact if (empty($url) || !is_string($url)) { continue; } - $contact = self::getByURL($url, false, ['id', 'updated']); + $contact = self::getByURL($url, false, ['id', 'network', 'next-update']); if (empty($contact['id']) && Network::isValidHttpUrl($url)) { Worker::add(PRIORITY_LOW, 'AddContact', 0, $url); ++$added; - } elseif ($contact['updated'] < DateTimeFormat::utc('now -7 days')) { - Worker::add(PRIORITY_LOW, 'UpdateContact', $contact['id']); + } elseif (Probe::isProbable($contact['network']) && ($contact['next-update'] < DateTimeFormat::utcNow())) { + Worker::add(['priority' => PRIORITY_LOW, 'dont_fork' => true], 'UpdateContact', $contact['id']); ++$updated; } else { ++$unchanged; diff --git a/src/Model/FContact.php b/src/Model/FContact.php index 62f2f9836..ae6c9cb31 100644 --- a/src/Model/FContact.php +++ b/src/Model/FContact.php @@ -23,6 +23,7 @@ namespace Friendica\Model; use Friendica\Core\Logger; use Friendica\Core\Protocol; +use Friendica\Core\Worker; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Network\Probe; @@ -43,6 +44,7 @@ class FContact */ public static function getByURL(string $handle, $update = null): array { + Logger::debug('Fetch fcontact', ['handle' => $handle, 'update' => $update]); $person = DBA::selectFirst('fcontact', [], ['network' => Protocol::DIASPORA, 'addr' => $handle]); if (!DBA::isResult($person)) { $urls = [$handle, str_replace('http://', 'https://', $handle), Strings::normaliseLink($handle)]; @@ -50,21 +52,17 @@ class FContact } if (DBA::isResult($person)) { - Logger::debug('In cache', ['person' => $person]); + Logger::debug('In cache', ['handle' => $handle]); if (is_null($update)) { - // update record occasionally so it doesn't get stale - $d = strtotime($person['updated'] . ' +00:00'); - if ($d < strtotime('now - 14 days')) { - $update = true; - } - - if (empty($person['guid']) || empty($person['uri-id'])) { - $update = true; + $update = empty($person['guid']) || empty($person['uri-id']) || ($person['created'] <= DBA::NULL_DATETIME); + if (GServer::getNextUpdateDate(true, $person['created'], $person['updated'], false) < DateTimeFormat::utcNow()) { + Logger::debug('Start background update', ['handle' => $handle]); + Worker::add(['priority' => PRIORITY_LOW, 'dont_fork' => true], 'UpdateFContact', $handle); } } } elseif (is_null($update)) { - $update = !DBA::isResult($person); + $update = true; } else { $person = []; } @@ -95,7 +93,8 @@ class FContact { $uriid = ItemURI::insert(['uri' => $arr['url'], 'guid' => $arr['guid']]); - $contact = Contact::getByUriId($uriid, ['id']); + $fcontact = DBA::selectFirst('fcontact', ['created'], ['url' => $arr['url'], 'network' => $arr['network']]); + $contact = Contact::getByUriId($uriid, ['id', 'created']); $apcontact = APContact::getByURL($arr['url'], false); if (!empty($apcontact)) { $interacted = $apcontact['following_count']; @@ -129,10 +128,14 @@ class FContact 'updated' => DateTimeFormat::utcNow(), ]; - $condition = ['url' => $arr['url'], 'network' => $arr['network']]; + if (empty($fcontact['created'])) { + $fields['created'] = $fields['updated']; + } elseif (!empty($contact['created']) && ($fcontact['created'] <= DBA::NULL_DATETIME)) { + $fields['created'] = $contact['created']; + } $fields = DI::dbaDefinition()->truncateFieldsForTable('fcontact', $fields); - DBA::update('fcontact', $fields, $condition, true); + DBA::update('fcontact', $fields, ['url' => $arr['url'], 'network' => $arr['network']], true); } /** diff --git a/src/Module/NoScrape.php b/src/Module/NoScrape.php index e3a01ceaa..363acd38c 100644 --- a/src/Module/NoScrape.php +++ b/src/Module/NoScrape.php @@ -22,11 +22,10 @@ namespace Friendica\Module; use Friendica\BaseModule; -use Friendica\Core\Cache\Enum\Duration; -use Friendica\Core\Protocol; use Friendica\Core\System; use Friendica\Database\DBA; use Friendica\DI; +use Friendica\Model\APContact; use Friendica\Model\User; /** @@ -36,8 +35,6 @@ use Friendica\Model\User; */ class NoScrape extends BaseModule { - const CACHEKEY = 'noscrape:'; - protected function rawContent(array $request = []) { $a = DI::app(); @@ -58,12 +55,6 @@ class NoScrape extends BaseModule System::jsonError(404, 'Profile not found'); } - $cachekey = self::CACHEKEY . $owner['uid']; - $result = DI::cache()->get($cachekey); - if (!is_null($result)) { - System::jsonExit($result); - } - $json_info = [ 'addr' => $owner['addr'], 'nick' => $which, @@ -98,16 +89,8 @@ class NoScrape extends BaseModule } if (!($owner['hide-friends'] ?? false)) { - $json_info['contacts'] = DBA::count('contact', - [ - 'uid' => $owner['uid'], - 'self' => 0, - 'blocked' => 0, - 'pending' => 0, - 'hidden' => 0, - 'archive' => 0, - 'network' => [Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS] - ]); + $apcontact = APContact::getByURL($owner['url']); + $json_info['contacts'] = max($apcontact['following_count'], $apcontact['followers_count']); } // We display the last activity (post or login), reduced to year and week number @@ -135,8 +118,6 @@ class NoScrape extends BaseModule } } - DI::cache()->set($cachekey, $json_info, Duration::DAY); - System::jsonExit($json_info); } } diff --git a/src/Network/Probe.php b/src/Network/Probe.php index d5d87ebf9..594ff506f 100644 --- a/src/Network/Probe.php +++ b/src/Network/Probe.php @@ -57,6 +57,18 @@ class Probe private static $baseurl; private static $istimeout; + /** + * Checks if the provided network can be probed + * + * @param string $network + * @return boolean + */ + public static function isProbable(string $network): bool + { + return (in_array($network, array_merge(Protocol::FEDERATED, [Protocol::ZOT, Protocol::PHANTOM]))); + + } + /** * Remove stuff from an URI that doesn't belong there * diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 9f1ab1585..4bba542c2 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -55,7 +55,7 @@ use Friendica\Database\DBA; if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1482); + define('DB_UPDATE_VERSION', 1483); } return [ @@ -684,6 +684,7 @@ return [ "network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => ""], "alias" => ["type" => "varbinary(383)", "not null" => "1", "default" => "", "comment" => ""], "pubkey" => ["type" => "text", "comment" => ""], + "created" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => ""], "updated" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => ""], "interacting_count" => ["type" => "int unsigned", "default" => 0, "comment" => "Number of contacts this contact interactes with"], "interacted_count" => ["type" => "int unsigned", "default" => 0, "comment" => "Number of contacts that interacted with this contact"], From 2525b3f2ae49fadaf5078ef7f8139232ba0d1b42 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Mon, 19 Sep 2022 07:52:40 +0200 Subject: [PATCH 2/2] Update src/Network/Probe.php Co-authored-by: Hypolite Petovan --- src/Network/Probe.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Network/Probe.php b/src/Network/Probe.php index 594ff506f..f075a514a 100644 --- a/src/Network/Probe.php +++ b/src/Network/Probe.php @@ -66,7 +66,6 @@ class Probe public static function isProbable(string $network): bool { return (in_array($network, array_merge(Protocol::FEDERATED, [Protocol::ZOT, Protocol::PHANTOM]))); - } /**