Merge pull request #11905 from annando/fcontact-noscrape

The fcontact table is now updated in the background to improve performance
This commit is contained in:
Hypolite Petovan 2022-09-19 09:10:32 -04:00 committed by GitHub
commit 0747eb4961
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 47 additions and 47 deletions

View file

@ -1,6 +1,6 @@
-- ------------------------------------------ -- ------------------------------------------
-- Friendica 2022.09-rc (Giant Rhubarb) -- Friendica 2022.09-rc (Giant Rhubarb)
-- DB_UPDATE_VERSION 1482 -- DB_UPDATE_VERSION 1483
-- ------------------------------------------ -- ------------------------------------------
@ -625,6 +625,7 @@ CREATE TABLE IF NOT EXISTS `fcontact` (
`network` char(4) NOT NULL DEFAULT '' COMMENT '', `network` char(4) NOT NULL DEFAULT '' COMMENT '',
`alias` varbinary(383) NOT NULL DEFAULT '' COMMENT '', `alias` varbinary(383) NOT NULL DEFAULT '' COMMENT '',
`pubkey` text COMMENT '', `pubkey` text COMMENT '',
`created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
`updated` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '', `updated` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
`interacting_count` int unsigned DEFAULT 0 COMMENT 'Number of contacts this contact interactes with', `interacting_count` int unsigned DEFAULT 0 COMMENT 'Number of contacts this contact interactes with',
`interacted_count` int unsigned DEFAULT 0 COMMENT 'Number of contacts that interacted with this contact', `interacted_count` int unsigned DEFAULT 0 COMMENT 'Number of contacts that interacted with this contact',

View file

@ -25,6 +25,7 @@ Fields
| network | | char(4) | NO | | | | | network | | char(4) | NO | | | |
| alias | | varbinary(383) | NO | | | | | alias | | varbinary(383) | NO | | | |
| pubkey | | text | YES | | NULL | | | pubkey | | text | YES | | NULL | |
| created | | datetime | NO | | 0001-01-01 00:00:00 | |
| updated | | datetime | NO | | 0001-01-01 00:00:00 | | | updated | | datetime | NO | | 0001-01-01 00:00:00 | |
| interacting_count | Number of contacts this contact interactes with | int unsigned | YES | | 0 | | | interacting_count | Number of contacts this contact interactes with | int unsigned | YES | | 0 | |
| interacted_count | Number of contacts that interacted with this contact | int unsigned | YES | | 0 | | | interacted_count | Number of contacts that interacted with this contact | int unsigned | YES | | 0 | |

View file

@ -252,7 +252,7 @@ class Contact
// Add internal fields // Add internal fields
$removal = []; $removal = [];
if (!empty($fields)) { if (!empty($fields)) {
foreach (['id', 'avatar', 'created', 'updated', 'last-update', 'success_update', 'failure_update', 'network'] as $internal) { foreach (['id', 'next-update', 'network'] as $internal) {
if (!in_array($internal, $fields)) { if (!in_array($internal, $fields)) {
$fields[] = $internal; $fields[] = $internal;
$removal[] = $internal; $removal[] = $internal;
@ -282,9 +282,8 @@ class Contact
} }
// Update the contact in the background if needed // Update the contact in the background if needed
$updated = max($contact['success_update'], $contact['created'], $contact['updated'], $contact['last-update'], $contact['failure_update']); if (Probe::isProbable($contact['network']) && ($contact['next-update'] < DateTimeFormat::utcNow())) {
if (($updated < DateTimeFormat::utc('now -7 days')) && in_array($contact['network'], Protocol::FEDERATED) && !self::isLocalById($contact['id'])) { Worker::add(['priority' => PRIORITY_LOW, 'dont_fork' => true], 'UpdateContact', $contact['id']);
Worker::add(PRIORITY_LOW, "UpdateContact", $contact['id']);
} }
// Remove the internal fields // Remove the internal fields
@ -935,7 +934,6 @@ class Contact
DI::cache()->delete(ActivityPub\Transmitter::CACHEKEY_CONTACTS . 'followers:' . $uid); DI::cache()->delete(ActivityPub\Transmitter::CACHEKEY_CONTACTS . 'followers:' . $uid);
DI::cache()->delete(ActivityPub\Transmitter::CACHEKEY_CONTACTS . 'following:' . $uid); DI::cache()->delete(ActivityPub\Transmitter::CACHEKEY_CONTACTS . 'following:' . $uid);
DI::cache()->delete(NoScrape::CACHEKEY . $uid);
} }
/** /**
@ -1194,11 +1192,15 @@ class Contact
return 0; return 0;
} }
$contact = self::getByURL($url, false, ['id', 'network', 'uri-id'], $uid); $contact = self::getByURL($url, false, ['id', 'network', 'uri-id', 'next-update'], $uid);
if (!empty($contact)) { if (!empty($contact)) {
$contact_id = $contact['id']; $contact_id = $contact['id'];
if (Probe::isProbable($contact['network']) && ($contact['next-update'] < DateTimeFormat::utcNow())) {
Worker::add(['priority' => PRIORITY_LOW, 'dont_fork' => true], 'UpdateContact', $contact['id']);
}
if (empty($update) && (!empty($contact['uri-id']) || is_bool($update))) { if (empty($update) && (!empty($contact['uri-id']) || is_bool($update))) {
Logger::debug('Contact found', ['url' => $url, 'uid' => $uid, 'update' => $update, 'cid' => $contact_id]); Logger::debug('Contact found', ['url' => $url, 'uid' => $uid, 'update' => $update, 'cid' => $contact_id]);
return $contact_id; return $contact_id;
@ -2460,7 +2462,7 @@ class Contact
$has_local_data = self::hasLocalData($id, $contact); $has_local_data = self::hasLocalData($id, $contact);
if (!in_array($ret['network'], array_merge(Protocol::FEDERATED, [Protocol::ZOT, Protocol::PHANTOM]))) { if (!Probe::isProbable($ret['network'])) {
// Periodical checks are only done on federated contacts // Periodical checks are only done on federated contacts
$failed_next_update = null; $failed_next_update = null;
$success_next_update = null; $success_next_update = null;
@ -3320,12 +3322,12 @@ class Contact
if (empty($url) || !is_string($url)) { if (empty($url) || !is_string($url)) {
continue; continue;
} }
$contact = self::getByURL($url, false, ['id', 'updated']); $contact = self::getByURL($url, false, ['id', 'network', 'next-update']);
if (empty($contact['id']) && Network::isValidHttpUrl($url)) { if (empty($contact['id']) && Network::isValidHttpUrl($url)) {
Worker::add(PRIORITY_LOW, 'AddContact', 0, $url); Worker::add(PRIORITY_LOW, 'AddContact', 0, $url);
++$added; ++$added;
} elseif ($contact['updated'] < DateTimeFormat::utc('now -7 days')) { } elseif (Probe::isProbable($contact['network']) && ($contact['next-update'] < DateTimeFormat::utcNow())) {
Worker::add(PRIORITY_LOW, 'UpdateContact', $contact['id']); Worker::add(['priority' => PRIORITY_LOW, 'dont_fork' => true], 'UpdateContact', $contact['id']);
++$updated; ++$updated;
} else { } else {
++$unchanged; ++$unchanged;

View file

@ -23,6 +23,7 @@ namespace Friendica\Model;
use Friendica\Core\Logger; use Friendica\Core\Logger;
use Friendica\Core\Protocol; use Friendica\Core\Protocol;
use Friendica\Core\Worker;
use Friendica\Database\DBA; use Friendica\Database\DBA;
use Friendica\DI; use Friendica\DI;
use Friendica\Network\Probe; use Friendica\Network\Probe;
@ -43,6 +44,7 @@ class FContact
*/ */
public static function getByURL(string $handle, $update = null): array public static function getByURL(string $handle, $update = null): array
{ {
Logger::debug('Fetch fcontact', ['handle' => $handle, 'update' => $update]);
$person = DBA::selectFirst('fcontact', [], ['network' => Protocol::DIASPORA, 'addr' => $handle]); $person = DBA::selectFirst('fcontact', [], ['network' => Protocol::DIASPORA, 'addr' => $handle]);
if (!DBA::isResult($person)) { if (!DBA::isResult($person)) {
$urls = [$handle, str_replace('http://', 'https://', $handle), Strings::normaliseLink($handle)]; $urls = [$handle, str_replace('http://', 'https://', $handle), Strings::normaliseLink($handle)];
@ -50,21 +52,17 @@ class FContact
} }
if (DBA::isResult($person)) { if (DBA::isResult($person)) {
Logger::debug('In cache', ['person' => $person]); Logger::debug('In cache', ['handle' => $handle]);
if (is_null($update)) { if (is_null($update)) {
// update record occasionally so it doesn't get stale $update = empty($person['guid']) || empty($person['uri-id']) || ($person['created'] <= DBA::NULL_DATETIME);
$d = strtotime($person['updated'] . ' +00:00'); if (GServer::getNextUpdateDate(true, $person['created'], $person['updated'], false) < DateTimeFormat::utcNow()) {
if ($d < strtotime('now - 14 days')) { Logger::debug('Start background update', ['handle' => $handle]);
$update = true; Worker::add(['priority' => PRIORITY_LOW, 'dont_fork' => true], 'UpdateFContact', $handle);
}
if (empty($person['guid']) || empty($person['uri-id'])) {
$update = true;
} }
} }
} elseif (is_null($update)) { } elseif (is_null($update)) {
$update = !DBA::isResult($person); $update = true;
} else { } else {
$person = []; $person = [];
} }
@ -95,7 +93,8 @@ class FContact
{ {
$uriid = ItemURI::insert(['uri' => $arr['url'], 'guid' => $arr['guid']]); $uriid = ItemURI::insert(['uri' => $arr['url'], 'guid' => $arr['guid']]);
$contact = Contact::getByUriId($uriid, ['id']); $fcontact = DBA::selectFirst('fcontact', ['created'], ['url' => $arr['url'], 'network' => $arr['network']]);
$contact = Contact::getByUriId($uriid, ['id', 'created']);
$apcontact = APContact::getByURL($arr['url'], false); $apcontact = APContact::getByURL($arr['url'], false);
if (!empty($apcontact)) { if (!empty($apcontact)) {
$interacted = $apcontact['following_count']; $interacted = $apcontact['following_count'];
@ -129,10 +128,14 @@ class FContact
'updated' => DateTimeFormat::utcNow(), 'updated' => DateTimeFormat::utcNow(),
]; ];
$condition = ['url' => $arr['url'], 'network' => $arr['network']]; if (empty($fcontact['created'])) {
$fields['created'] = $fields['updated'];
} elseif (!empty($contact['created']) && ($fcontact['created'] <= DBA::NULL_DATETIME)) {
$fields['created'] = $contact['created'];
}
$fields = DI::dbaDefinition()->truncateFieldsForTable('fcontact', $fields); $fields = DI::dbaDefinition()->truncateFieldsForTable('fcontact', $fields);
DBA::update('fcontact', $fields, $condition, true); DBA::update('fcontact', $fields, ['url' => $arr['url'], 'network' => $arr['network']], true);
} }
/** /**

View file

@ -22,11 +22,10 @@
namespace Friendica\Module; namespace Friendica\Module;
use Friendica\BaseModule; use Friendica\BaseModule;
use Friendica\Core\Cache\Enum\Duration;
use Friendica\Core\Protocol;
use Friendica\Core\System; use Friendica\Core\System;
use Friendica\Database\DBA; use Friendica\Database\DBA;
use Friendica\DI; use Friendica\DI;
use Friendica\Model\APContact;
use Friendica\Model\User; use Friendica\Model\User;
/** /**
@ -36,8 +35,6 @@ use Friendica\Model\User;
*/ */
class NoScrape extends BaseModule class NoScrape extends BaseModule
{ {
const CACHEKEY = 'noscrape:';
protected function rawContent(array $request = []) protected function rawContent(array $request = [])
{ {
$a = DI::app(); $a = DI::app();
@ -58,12 +55,6 @@ class NoScrape extends BaseModule
System::jsonError(404, 'Profile not found'); System::jsonError(404, 'Profile not found');
} }
$cachekey = self::CACHEKEY . $owner['uid'];
$result = DI::cache()->get($cachekey);
if (!is_null($result)) {
System::jsonExit($result);
}
$json_info = [ $json_info = [
'addr' => $owner['addr'], 'addr' => $owner['addr'],
'nick' => $which, 'nick' => $which,
@ -98,16 +89,8 @@ class NoScrape extends BaseModule
} }
if (!($owner['hide-friends'] ?? false)) { if (!($owner['hide-friends'] ?? false)) {
$json_info['contacts'] = DBA::count('contact', $apcontact = APContact::getByURL($owner['url']);
[ $json_info['contacts'] = max($apcontact['following_count'], $apcontact['followers_count']);
'uid' => $owner['uid'],
'self' => 0,
'blocked' => 0,
'pending' => 0,
'hidden' => 0,
'archive' => 0,
'network' => [Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS]
]);
} }
// We display the last activity (post or login), reduced to year and week number // We display the last activity (post or login), reduced to year and week number
@ -135,8 +118,6 @@ class NoScrape extends BaseModule
} }
} }
DI::cache()->set($cachekey, $json_info, Duration::DAY);
System::jsonExit($json_info); System::jsonExit($json_info);
} }
} }

View file

@ -57,6 +57,17 @@ class Probe
private static $baseurl; private static $baseurl;
private static $istimeout; private static $istimeout;
/**
* Checks if the provided network can be probed
*
* @param string $network
* @return boolean
*/
public static function isProbable(string $network): bool
{
return (in_array($network, array_merge(Protocol::FEDERATED, [Protocol::ZOT, Protocol::PHANTOM])));
}
/** /**
* Remove stuff from an URI that doesn't belong there * Remove stuff from an URI that doesn't belong there
* *

View file

@ -55,7 +55,7 @@
use Friendica\Database\DBA; use Friendica\Database\DBA;
if (!defined('DB_UPDATE_VERSION')) { if (!defined('DB_UPDATE_VERSION')) {
define('DB_UPDATE_VERSION', 1482); define('DB_UPDATE_VERSION', 1483);
} }
return [ return [
@ -684,6 +684,7 @@ return [
"network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => ""], "network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => ""],
"alias" => ["type" => "varbinary(383)", "not null" => "1", "default" => "", "comment" => ""], "alias" => ["type" => "varbinary(383)", "not null" => "1", "default" => "", "comment" => ""],
"pubkey" => ["type" => "text", "comment" => ""], "pubkey" => ["type" => "text", "comment" => ""],
"created" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => ""],
"updated" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => ""], "updated" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => ""],
"interacting_count" => ["type" => "int unsigned", "default" => 0, "comment" => "Number of contacts this contact interactes with"], "interacting_count" => ["type" => "int unsigned", "default" => 0, "comment" => "Number of contacts this contact interactes with"],
"interacted_count" => ["type" => "int unsigned", "default" => 0, "comment" => "Number of contacts that interacted with this contact"], "interacted_count" => ["type" => "int unsigned", "default" => 0, "comment" => "Number of contacts that interacted with this contact"],