From e0e2e45b91f793c58b8fb06240cca0e6c7f85ea4 Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 28 Dec 2022 14:56:12 +0000 Subject: [PATCH] Reduce the amount of "UpdateContact" worker calls --- src/Model/Contact.php | 50 +++++++++++++++++++++++--- src/Protocol/ActivityPub/Delivery.php | 8 ++++- src/Protocol/ActivityPub/Processor.php | 6 ++++ src/Protocol/ActivityPub/Receiver.php | 23 +++++++----- src/Protocol/Diaspora.php | 3 +- src/Worker/UpdateContacts.php | 17 ++++++--- 6 files changed, 89 insertions(+), 18 deletions(-) diff --git a/src/Model/Contact.php b/src/Model/Contact.php index 2003b4608..e928df7f6 100644 --- a/src/Model/Contact.php +++ b/src/Model/Contact.php @@ -329,7 +329,7 @@ class Contact // Add internal fields $removal = []; if (!empty($fields)) { - foreach (['id', 'next-update', 'network'] as $internal) { + foreach (['id', 'next-update', 'network', 'local-data'] as $internal) { if (!in_array($internal, $fields)) { $fields[] = $internal; $removal[] = $internal; @@ -358,8 +358,10 @@ class Contact return []; } + $background_update = DI::config()->get('system', 'update_active_contacts') ? $contact['local-data'] : true; + // Update the contact in the background if needed - if (Probe::isProbable($contact['network']) && ($contact['next-update'] < DateTimeFormat::utcNow())) { + if ($background_update && !self::isLocal($url) && Probe::isProbable($contact['network']) && ($contact['next-update'] < DateTimeFormat::utcNow())) { Worker::add(['priority' => Worker::PRIORITY_LOW, 'dont_fork' => true], 'UpdateContact', $contact['id']); } @@ -1266,12 +1268,14 @@ class Contact return 0; } - $contact = self::getByURL($url, false, ['id', 'network', 'uri-id', 'next-update'], $uid); + $contact = self::getByURL($url, false, ['id', 'network', 'uri-id', 'next-update', 'local-data'], $uid); if (!empty($contact)) { $contact_id = $contact['id']; - if (Probe::isProbable($contact['network']) && ($contact['next-update'] < DateTimeFormat::utcNow())) { + $background_update = DI::config()->get('system', 'update_active_contacts') ? $contact['local-data'] : true; + + if ($background_update && !self::isLocal($url) && Probe::isProbable($contact['network']) && ($contact['next-update'] < DateTimeFormat::utcNow())) { Worker::add(['priority' => Worker::PRIORITY_LOW, 'dont_fork' => true], 'UpdateContact', $contact['id']); } @@ -2481,6 +2485,44 @@ class Contact return true; } + /** + * Perform a contact update if the contact is outdated + * + * @param integer $id contact id + * @return bool + */ + public static function updateByIdIfNeeded(int $id): bool + { + $contact = self::selectFirst(['url'], ["`id` = ? AND `next-update` < ?", $id, DateTimeFormat::utcNow()]); + if (empty($contact['url'])) { + return false; + } + + if (self::isLocal($contact['url'])) { + return true; + } + + $stamp = (float)microtime(true); + self::updateFromProbe($id); + Logger::debug('Contact data is updated.', ['duration' => round((float)microtime(true) - $stamp, 3), 'id' => $id, 'url' => $contact['url'], 'callstack' => System::callstack(20)]); + return true; + } + + /** + * Perform a contact update if the contact is outdated + * + * @param string $url contact url + * @return bool + */ + public static function updateByUrlIfNeeded(string $url): bool + { + $id = self::getIdForURL($url, 0, false); + if (!empty($id)) { + return self::updateByIdIfNeeded($id); + } + return (bool)self::getIdForURL($url); + } + /** * Updates contact record by provided id and optional network * diff --git a/src/Protocol/ActivityPub/Delivery.php b/src/Protocol/ActivityPub/Delivery.php index add898888..0e014c703 100644 --- a/src/Protocol/ActivityPub/Delivery.php +++ b/src/Protocol/ActivityPub/Delivery.php @@ -205,9 +205,15 @@ class Delivery */ private static function setSuccess(array $receivers, bool $success) { - $gsid = null; + $gsid = null; + $update_counter = 0; foreach ($receivers as $receiver) { + // Only update the first 10 receivers to avoid flooding the remote system with requests + if ($success && ($update_counter < 10) && Contact::updateByIdIfNeeded($receiver)) { + $update_counter++; + } + $contact = Contact::getById($receiver); if (empty($contact)) { continue; diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index e98fa4596..fdf35ff86 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -1530,6 +1530,12 @@ class Processor $ldactivity['recursion-depth'] = !empty($child['recursion-depth']) ? $child['recursion-depth'] + 1 : 0; + if ($object_actor != $actor) { + Contact::updateByUrlIfNeeded($object_actor); + } + + Contact::updateByUrlIfNeeded($actor); + if (!empty($relay_actor)) { $ldactivity['thread-completion'] = $ldactivity['from-relay'] = Contact::getIdForURL($relay_actor); $ldactivity['completion-mode'] = Receiver::COMPLETION_RELAY; diff --git a/src/Protocol/ActivityPub/Receiver.php b/src/Protocol/ActivityPub/Receiver.php index fbe8c32dc..3117a8791 100644 --- a/src/Protocol/ActivityPub/Receiver.php +++ b/src/Protocol/ActivityPub/Receiver.php @@ -315,12 +315,15 @@ class Receiver $object_type = JsonLD::fetchElement($activity['as:object'], '@type'); } + $fetched = false; + if (!empty($id) && !$trust_source) { $fetch_uid = $uid ?: self::getBestUserForActivity($activity); $fetched_activity = Processor::fetchCachedActivity($fetch_id, $fetch_uid); if (!empty($fetched_activity)) { - $object = JsonLD::compact($fetched_activity); + $fetched = true; + $object = JsonLD::compact($fetched_activity); $fetched_id = JsonLD::fetchElement($object, '@id'); $fetched_type = JsonLD::fetchElement($object, '@type'); @@ -351,7 +354,7 @@ class Receiver $type = JsonLD::fetchElement($activity, '@type'); // Fetch all receivers from to, cc, bto and bcc - $receiverdata = self::getReceivers($activity, $actor); + $receiverdata = self::getReceivers($activity, $actor, [], false, $push || $fetched); $receivers = $reception_types = []; foreach ($receiverdata as $key => $data) { $receivers[$key] = $data['uid']; @@ -1029,7 +1032,7 @@ class Receiver $uid = 0; $actor = JsonLD::fetchElement($activity, 'as:actor', '@id') ?? ''; - $receivers = self::getReceivers($activity, $actor); + $receivers = self::getReceivers($activity, $actor, [], false, false); foreach ($receivers as $receiver) { if ($receiver['type'] == self::TARGET_GLOBAL) { return 0; @@ -1076,14 +1079,15 @@ class Receiver * Fetch the receiver list from an activity array * * @param array $activity - * @param string $actor - * @param array $tags - * @param boolean $fetch_unlisted + * @param string $actor + * @param array $tags + * @param bool $fetch_unlisted + * @param bool $push * * @return array with receivers (user id) * @throws \Exception */ - private static function getReceivers(array $activity, string $actor, array $tags = [], bool $fetch_unlisted = false): array + private static function getReceivers(array $activity, string $actor, array $tags, bool $fetch_unlisted, bool $push): array { $reply = $receivers = $profile = []; @@ -1117,6 +1121,9 @@ class Receiver $profile = APContact::getByURL($actor); $followers = $profile['followers'] ?? ''; $is_forum = ($actor['type'] ?? '') == 'Group'; + if ($push) { + Contact::updateByUrlIfNeeded($actor); + } Logger::info('Got actor and followers', ['actor' => $actor, 'followers' => $followers]); } else { Logger::info('Empty actor', ['activity' => $activity]); @@ -2006,7 +2013,7 @@ class Receiver $object_data['question'] = self::processQuestion($object); } - $receiverdata = self::getReceivers($object, $object_data['actor'] ?? '', $object_data['tags'], true); + $receiverdata = self::getReceivers($object, $object_data['actor'] ?? '', $object_data['tags'], true, false); $receivers = $reception_types = []; foreach ($receiverdata as $key => $data) { $receivers[$key] = $data['uid']; diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php index 86d88867b..0cb2a66cf 100644 --- a/src/Protocol/Diaspora.php +++ b/src/Protocol/Diaspora.php @@ -534,7 +534,7 @@ class Diaspora if (is_null($fields)) { $private = true; if (!($fields = self::validPosting($msg))) { - Logger::warning('Invalid posting', ['msg' => $msg]); + Logger::notice('Invalid posting', ['msg' => $msg]); return false; } } else { @@ -812,6 +812,7 @@ class Diaspora */ private static function contactByHandle(int $uid, WebFingerUri $uri): array { + Contact::updateByUrlIfNeeded($uri->getAddr()); return Contact::getByURL($uri->getAddr(), null, [], $uid); } diff --git a/src/Worker/UpdateContacts.php b/src/Worker/UpdateContacts.php index ed4a9932f..3c7e54228 100644 --- a/src/Worker/UpdateContacts.php +++ b/src/Worker/UpdateContacts.php @@ -22,10 +22,11 @@ namespace Friendica\Worker; use Friendica\Core\Logger; -use Friendica\Core\Protocol; use Friendica\Core\Worker; use Friendica\Database\DBA; use Friendica\DI; +use Friendica\Model\Contact; +use Friendica\Model\GServer; use Friendica\Util\DateTimeFormat; /** @@ -56,14 +57,22 @@ class UpdateContacts } $condition = DBA::mergeConditions(["`next-update` < ?", DateTimeFormat::utcNow()], $condition); - $contacts = DBA::select('contact', ['id'], $condition, ['order' => ['next-update'], 'limit' => $limit]); + $contacts = DBA::select('contact', ['id', 'url', 'gsid', 'baseurl'], $condition, ['order' => ['next-update'], 'limit' => $limit]); $count = 0; while ($contact = DBA::fetch($contacts)) { - if (Worker::add(['priority' => Worker::PRIORITY_LOW, 'dont_fork' => true], "UpdateContact", $contact['id'])) { + if (Contact::isLocal($contact['url'])) { + continue; + } + if ((!empty($contact['gsid']) || !empty($contact['baseurl'])) && GServer::reachable($contact)) { + $stamp = (float)microtime(true); + $success = Contact::updateFromProbe($contact['id']); + Logger::debug('Direct update', ['id' => $contact['id'], 'duration' => round((float)microtime(true) - $stamp, 3), 'success' => $success]); + ++$count; + } elseif (Worker::add(['priority' => Worker::PRIORITY_LOW, 'dont_fork' => true], 'UpdateContact', $contact['id'])) { ++$count; } Worker::coolDown(); - } + } DBA::close($contacts); Logger::info('Initiated update for federated contacts', ['count' => $count]);