Merge pull request #12557 from annando/reduce-update-contacts

Reduce the amount of "UpdateContact" worker calls

src/Model/Contact.php

@@ -329,7 +329,7 @@ class Contact
 		// Add internal fields
 		$removal = [];
 		if (!empty($fields)) {
-			foreach (['id', 'next-update', 'network'] as $internal) {
+			foreach (['id', 'next-update', 'network', 'local-data'] as $internal) {
 				if (!in_array($internal, $fields)) {
 					$fields[] = $internal;
 					$removal[] = $internal;
@@ -358,8 +358,10 @@ class Contact
 			return [];
 		}
 
+		$background_update = DI::config()->get('system', 'update_active_contacts') ? $contact['local-data'] : true;
+
 		// Update the contact in the background if needed
-		if (Probe::isProbable($contact['network']) && ($contact['next-update'] < DateTimeFormat::utcNow())) {
+		if ($background_update && !self::isLocal($url) && Probe::isProbable($contact['network']) && ($contact['next-update'] < DateTimeFormat::utcNow())) {
 			Worker::add(['priority' => Worker::PRIORITY_LOW, 'dont_fork' => true], 'UpdateContact', $contact['id']);
 		}
 
@@ -1266,12 +1268,14 @@ class Contact
 			return 0;
 		}
 
-		$contact = self::getByURL($url, false, ['id', 'network', 'uri-id', 'next-update'], $uid);
+		$contact = self::getByURL($url, false, ['id', 'network', 'uri-id', 'next-update', 'local-data'], $uid);
 
 		if (!empty($contact)) {
 			$contact_id = $contact['id'];
 
-			if (Probe::isProbable($contact['network']) && ($contact['next-update'] < DateTimeFormat::utcNow())) {
+			$background_update = DI::config()->get('system', 'update_active_contacts') ? $contact['local-data'] : true;
+
+			if ($background_update && !self::isLocal($url) && Probe::isProbable($contact['network']) && ($contact['next-update'] < DateTimeFormat::utcNow())) {
 				Worker::add(['priority' => Worker::PRIORITY_LOW, 'dont_fork' => true], 'UpdateContact', $contact['id']);
 			}
 
@@ -2481,6 +2485,44 @@ class Contact
 		return true;
 	}
 
+	/**
+	 * Perform a contact update if the contact is outdated
+	 *
+	 * @param integer $id contact id
+	 * @return bool
+	 */
+	public static function updateByIdIfNeeded(int $id): bool
+	{
+		$contact = self::selectFirst(['url'], ["`id` = ? AND `next-update` < ?", $id, DateTimeFormat::utcNow()]);
+		if (empty($contact['url'])) {
+			return false;
+		}
+
+		if (self::isLocal($contact['url'])) {
+			return true;
+		}
+
+		$stamp = (float)microtime(true);
+		self::updateFromProbe($id);
+		Logger::debug('Contact data is updated.', ['duration' => round((float)microtime(true) - $stamp, 3), 'id' => $id, 'url' => $contact['url'], 'callstack' => System::callstack(20)]);
+		return true;
+	}
+
+	/**
+	 * Perform a contact update if the contact is outdated
+	 *
+	 * @param string $url contact url
+	 * @return bool
+	 */
+	public static function updateByUrlIfNeeded(string $url): bool
+	{
+		$id = self::getIdForURL($url, 0, false);
+		if (!empty($id)) {
+			return self::updateByIdIfNeeded($id);
+		}
+		return (bool)self::getIdForURL($url);
+	}
+
 	/**
 	 * Updates contact record by provided id and optional network
 	 *

src/Protocol/ActivityPub/Delivery.php

@@ -206,8 +206,14 @@ class Delivery
 	private static function setSuccess(array $receivers, bool $success)
 	{
 		$gsid           = null;
+		$update_counter = 0;
 
 		foreach ($receivers as $receiver) {
+			// Only update the first 10 receivers to avoid flooding the remote system with requests
+			if ($success && ($update_counter < 10) && Contact::updateByIdIfNeeded($receiver)) {
+				$update_counter++;
+			}
+
 			$contact = Contact::getById($receiver);
 			if (empty($contact)) {
 				continue;

src/Protocol/ActivityPub/Processor.php

@@ -1530,6 +1530,12 @@ class Processor
 
 		$ldactivity['recursion-depth'] = !empty($child['recursion-depth']) ? $child['recursion-depth'] + 1 : 0;
 
+		if ($object_actor != $actor) {
+			Contact::updateByUrlIfNeeded($object_actor);
+		}
+
+		Contact::updateByUrlIfNeeded($actor);
+
 		if (!empty($relay_actor)) {
 			$ldactivity['thread-completion'] = $ldactivity['from-relay'] = Contact::getIdForURL($relay_actor);
 			$ldactivity['completion-mode']   = Receiver::COMPLETION_RELAY;

src/Protocol/ActivityPub/Receiver.php

@@ -315,11 +315,14 @@ class Receiver
 			$object_type = JsonLD::fetchElement($activity['as:object'], '@type');
 		}
 
+		$fetched = false;
+
 		if (!empty($id) && !$trust_source) {
 			$fetch_uid = $uid ?: self::getBestUserForActivity($activity);
 
 			$fetched_activity = Processor::fetchCachedActivity($fetch_id, $fetch_uid);
 			if (!empty($fetched_activity)) {
+				$fetched = true;
 				$object  = JsonLD::compact($fetched_activity);
 
 				$fetched_id   = JsonLD::fetchElement($object, '@id');
@@ -351,7 +354,7 @@ class Receiver
 		$type = JsonLD::fetchElement($activity, '@type');
 
 		// Fetch all receivers from to, cc, bto and bcc
-		$receiverdata = self::getReceivers($activity, $actor);
+		$receiverdata = self::getReceivers($activity, $actor, [], false, $push || $fetched);
 		$receivers = $reception_types = [];
 		foreach ($receiverdata as $key => $data) {
 			$receivers[$key] = $data['uid'];
@@ -1029,7 +1032,7 @@ class Receiver
 		$uid = 0;
 		$actor = JsonLD::fetchElement($activity, 'as:actor', '@id') ?? '';
 
-		$receivers = self::getReceivers($activity, $actor);
+		$receivers = self::getReceivers($activity, $actor, [], false, false);
 		foreach ($receivers as $receiver) {
 			if ($receiver['type'] == self::TARGET_GLOBAL) {
 				return 0;
@@ -1078,12 +1081,13 @@ class Receiver
 	 * @param array   $activity
 	 * @param string $actor
 	 * @param array  $tags
-	 * @param boolean $fetch_unlisted
+	 * @param bool   $fetch_unlisted
+	 * @param bool   $push
 	 *
 	 * @return array with receivers (user id)
 	 * @throws \Exception
 	 */
-	private static function getReceivers(array $activity, string $actor, array $tags = [], bool $fetch_unlisted = false): array
+	private static function getReceivers(array $activity, string $actor, array $tags, bool $fetch_unlisted, bool $push): array
 	{
 		$reply = $receivers = $profile = [];
 
@@ -1117,6 +1121,9 @@ class Receiver
 			$profile   = APContact::getByURL($actor);
 			$followers = $profile['followers'] ?? '';
 			$is_forum  = ($actor['type'] ?? '') == 'Group';
+			if ($push) {
+				Contact::updateByUrlIfNeeded($actor);
+			}
 			Logger::info('Got actor and followers', ['actor' => $actor, 'followers' => $followers]);
 		} else {
 			Logger::info('Empty actor', ['activity' => $activity]);
@@ -2006,7 +2013,7 @@ class Receiver
 			$object_data['question'] = self::processQuestion($object);
 		}
 
-		$receiverdata = self::getReceivers($object, $object_data['actor'] ?? '', $object_data['tags'], true);
+		$receiverdata = self::getReceivers($object, $object_data['actor'] ?? '', $object_data['tags'], true, false);
 		$receivers = $reception_types = [];
 		foreach ($receiverdata as $key => $data) {
 			$receivers[$key] = $data['uid'];

src/Protocol/Diaspora.php

@@ -534,7 +534,7 @@ class Diaspora
 		if (is_null($fields)) {
 			$private = true;
 			if (!($fields = self::validPosting($msg))) {
-				Logger::warning('Invalid posting', ['msg' => $msg]);
+				Logger::notice('Invalid posting', ['msg' => $msg]);
 				return false;
 			}
 		} else {
@@ -812,6 +812,7 @@ class Diaspora
 	 */
 	private static function contactByHandle(int $uid, WebFingerUri $uri): array
 	{
+		Contact::updateByUrlIfNeeded($uri->getAddr());
 		return Contact::getByURL($uri->getAddr(), null, [], $uid);
 	}
 

src/Worker/UpdateContacts.php

@@ -22,10 +22,11 @@
 namespace Friendica\Worker;
 
 use Friendica\Core\Logger;
-use Friendica\Core\Protocol;
 use Friendica\Core\Worker;
 use Friendica\Database\DBA;
 use Friendica\DI;
+use Friendica\Model\Contact;
+use Friendica\Model\GServer;
 use Friendica\Util\DateTimeFormat;
 
 /**
@@ -56,10 +57,19 @@ class UpdateContacts
 		}
 
 		$condition = DBA::mergeConditions(["`next-update` < ?", DateTimeFormat::utcNow()], $condition);
-		$contacts = DBA::select('contact', ['id'], $condition, ['order' => ['next-update'], 'limit' => $limit]);
+		$contacts = DBA::select('contact', ['id', 'url', 'gsid', 'baseurl'], $condition, ['order' => ['next-update'], 'limit' => $limit]);
 		$count = 0;
 		while ($contact = DBA::fetch($contacts)) {
-			if (Worker::add(['priority' => Worker::PRIORITY_LOW, 'dont_fork' => true], "UpdateContact", $contact['id'])) {
+			if (Contact::isLocal($contact['url'])) {
+				continue;
+			}
+			if ((!empty($contact['gsid']) || !empty($contact['baseurl'])) && GServer::reachable($contact)) {
+				$stamp = (float)microtime(true);
+				$success = Contact::updateFromProbe($contact['id']);
+				Logger::debug('Direct update', ['id' => $contact['id'], 'count' => $count, 'duration' => round((float)microtime(true) - $stamp, 3), 'success' => $success]);
+				++$count;
+			} elseif (Worker::add(['priority' => Worker::PRIORITY_LOW, 'dont_fork' => true], 'UpdateContact', $contact['id'])) {
+				Logger::debug('Update by worker', ['id' => $contact['id'], 'count' => $count]);
 				++$count;
 			}
 			Worker::coolDown();