From 88a39ea99936992d73b8aafdd95babd4bd864fe0 Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 20 Dec 2019 06:47:18 +0000 Subject: [PATCH 01/17] "gprobe" removed --- src/Model/GContact.php | 18 ----------- src/Model/Profile.php | 2 -- src/Worker/GProbe.php | 68 ------------------------------------------ 3 files changed, 88 deletions(-) delete mode 100644 src/Worker/GProbe.php diff --git a/src/Model/GContact.php b/src/Model/GContact.php index 2caad9459..0ad6df688 100644 --- a/src/Model/GContact.php +++ b/src/Model/GContact.php @@ -580,9 +580,6 @@ class GContact public static function getId($contact) { $gcontact_id = 0; - $doprobing = false; - $last_failure_str = ''; - $last_contact_str = ''; if (empty($contact['network'])) { Logger::notice('Empty network', ['url' => $contact['url'], 'callstack' => System::callstack()]); @@ -613,15 +610,6 @@ class GContact $gcnt = DBA::selectFirst('gcontact', $fields, ['nurl' => Strings::normaliseLink($contact['url'])]); if (DBA::isResult($gcnt)) { $gcontact_id = $gcnt['id']; - - // Update every 90 days - if (empty($gcnt['network']) || in_array($gcnt['network'], Protocol::FEDERATED)) { - $last_failure_str = $gcnt['last_failure']; - $last_failure = strtotime($gcnt['last_failure']); - $last_contact_str = $gcnt['last_contact']; - $last_contact = strtotime($gcnt['last_contact']); - $doprobing = (((time() - $last_contact) > (90 * 86400)) && ((time() - $last_failure) > (90 * 86400))); - } } else { $contact['location'] = $contact['location'] ?? ''; $contact['about'] = $contact['about'] ?? ''; @@ -638,16 +626,10 @@ class GContact $cnt = DBA::selectFirst('gcontact', ['id', 'network'], $condition, ['order' => ['id']]); if (DBA::isResult($cnt)) { $gcontact_id = $cnt['id']; - $doprobing = (empty($cnt['network']) || in_array($cnt['network'], Protocol::FEDERATED)); } } DBA::unlock(); - if ($doprobing) { - Logger::notice('Probing', ['contact' => $last_contact_str, "failure" => $last_failure_str, "checking" => $contact['url']]); - Worker::add(PRIORITY_LOW, 'GProbe', $contact['url']); - } - return $gcontact_id; } diff --git a/src/Model/Profile.php b/src/Model/Profile.php index eb274a640..8659506cb 100644 --- a/src/Model/Profile.php +++ b/src/Model/Profile.php @@ -1074,8 +1074,6 @@ class Profile Logger::log('Not authenticated. Invoking reverse magic-auth for ' . $my_url, Logger::DEBUG); - Worker::add(PRIORITY_LOW, 'GProbe', $my_url); - // Remove the "addr" parameter from the destination. It is later added as separate parameter again. $addr_request = 'addr=' . urlencode($addr); $query = rtrim(str_replace($addr_request, '', $a->query_string), '?&'); diff --git a/src/Worker/GProbe.php b/src/Worker/GProbe.php deleted file mode 100644 index 415c8f3ae..000000000 --- a/src/Worker/GProbe.php +++ /dev/null @@ -1,68 +0,0 @@ - $_SERVER]); - return; - } - } - - $arr = Probe::uri($url); - - if (is_null($result)) { - Cache::set("gprobe:".$urlparts["host"], $arr); - } - - if (!in_array($arr["network"], [Protocol::FEED, Protocol::PHANTOM])) { - GContact::update($arr); - } - - $r = q( - "SELECT `id`, `url`, `network` FROM `gcontact` WHERE `nurl` = '%s' ORDER BY `id` LIMIT 1", - DBA::escape(Strings::normaliseLink($url)) - ); - } - if (DBA::isResult($r)) { - // Check for accessibility and do a poco discovery - if (GContact::updateFromProbe($r[0]['url'], true) && ($r[0]["network"] == Protocol::DFRN)) { - PortableContact::loadWorker(0, 0, $r[0]['id'], str_replace('/profile/', '/poco/', $r[0]['url'])); - } - } - - Logger::log("gprobe end for ".Strings::normaliseLink($url), Logger::DEBUG); - return; - } -} From 37f05195140e09a5897f931c2955ff2a77b09d4f Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 20 Dec 2019 20:30:13 +0000 Subject: [PATCH 02/17] 'SearchDirectory' created (moved out of 'DiscoverPoco' mess) --- src/Core/Search.php | 2 +- src/Worker/DiscoverPoCo.php | 169 +++++---------------------------- src/Worker/SearchDirectory.php | 91 ++++++++++++++++++ 3 files changed, 115 insertions(+), 147 deletions(-) create mode 100644 src/Worker/SearchDirectory.php diff --git a/src/Core/Search.php b/src/Core/Search.php index 9700c6472..e43e621d2 100644 --- a/src/Core/Search.php +++ b/src/Core/Search.php @@ -242,7 +242,7 @@ class Search extends BaseObject DBA::close($data); // Add found profiles from the global directory to the local directory - Worker::add(PRIORITY_LOW, 'DiscoverPoCo', "dirsearch", urlencode($search)); + Worker::add(PRIORITY_LOW, 'SearchDirectory', $search); return $resultList; } diff --git a/src/Worker/DiscoverPoCo.php b/src/Worker/DiscoverPoCo.php index d495b4c2a..2c3536bf8 100644 --- a/src/Worker/DiscoverPoCo.php +++ b/src/Worker/DiscoverPoCo.php @@ -26,58 +26,21 @@ class DiscoverPoCo { /* This function can be called in these ways: - - dirsearch : Searches for "search pattern" in the directory. "search pattern" is url encoded. - checkcontact: Updates gcontact entries - suggestions: Discover other servers for their contacts. - server : Searches for the poco server list. "poco url" is base64 encoded. - update_server: Frequently check the first 250 servers for vitality. - update_server_directory: Discover the given server id for their contacts - PortableContact::load: Load POCO data from a given POCO address - - check_profile: Update remote profile data */ $search = ""; $mode = 0; - if ($command == "dirsearch") { - $search = urldecode($param1); - $mode = 1; - } elseif ($command == "checkcontact") { - $mode = 2; + if (($command == "checkcontact") && Config::get('system', 'poco_completion')) { + self::discoverUsers(); } elseif ($command == "suggestions") { - $mode = 3; + GContact::updateSuggestions(); } elseif ($command == "server") { - $mode = 4; - } elseif ($command == "update_server") { - $mode = 5; - } elseif ($command == "update_server_directory") { - $mode = 6; - } elseif ($command == "load") { - $mode = 7; - } elseif ($command == "check_profile") { - $mode = 8; - } elseif ($command !== "") { - Logger::log("Unknown or missing parameter ".$command."\n"); - return; - } - - Logger::log('start '.$search); - - if ($mode == 8) { - if ($param1 != "") { - GContact::updateFromProbe($param1, true); - } - } elseif ($mode == 7) { - if (!empty($param4)) { - $url = $param4; - } else { - $url = ''; - } - PortableContact::load(intval($param1), intval($param2), intval($param3), $url); - } elseif ($mode == 6) { - PortableContact::discoverSingleServer(intval($param1)); - } elseif ($mode == 5) { - self::updateServer(); - } elseif ($mode == 4) { $server_url = $param1; if ($server_url == "") { return; @@ -94,14 +57,25 @@ class DiscoverPoCo $result .= "failed"; } Logger::log($result, Logger::DEBUG); - } elseif ($mode == 3) { - GContact::updateSuggestions(); - } elseif (($mode == 2) && Config::get('system', 'poco_completion')) { - self::discoverUsers(); - } elseif (($mode == 1) && ($search != "") && Config::get('system', 'poco_local_search')) { - self::discoverDirectory($search); - self::gsSearchUser($search); - } elseif (($mode == 0) && ($search == "") && (Config::get('system', 'poco_discovery') != PortableContact::DISABLED)) { + } elseif ($command == "update_server") { + self::updateServer(); + } elseif ($command == "update_server_directory") { + PortableContact::discoverSingleServer(intval($param1)); + } elseif ($command == "load") { + if (!empty($param4)) { + $url = $param4; + } else { + $url = ''; + } + PortableContact::load(intval($param1), intval($param2), intval($param3), $url); + } elseif ($command !== "") { + Logger::log("Unknown or missing parameter ".$command."\n"); + return; + } + + Logger::log('start '.$search); + + if (($mode == 0) && ($search == "") && (Config::get('system', 'poco_discovery') != PortableContact::DISABLED)) { // Query Friendica and Hubzilla servers for their users PortableContact::discover(); @@ -189,7 +163,7 @@ class DiscoverPoCo if ((($server_url == "") && ($user["network"] == Protocol::FEED)) || $force_update || GServer::check($server_url, $user["network"])) { Logger::log('Check profile '.$user["url"]); - Worker::add(PRIORITY_LOW, "DiscoverPoCo", "check_profile", $user["url"]); + Worker::add(PRIORITY_LOW, 'UpdateGContact', $user['url'], 'force'); if (++$checked > 100) { return; @@ -205,101 +179,4 @@ class DiscoverPoCo } } } - - private static function discoverDirectory($search) { - - $data = Cache::get("dirsearch:".$search); - if (!is_null($data)) { - // Only search for the same item every 24 hours - if (time() < $data + (60 * 60 * 24)) { - Logger::log("Already searched for ".$search." in the last 24 hours", Logger::DEBUG); - return; - } - } - - $x = Network::fetchUrl(get_server()."/lsearch?p=1&n=500&search=".urlencode($search)); - $j = json_decode($x); - - if (!empty($j->results)) { - foreach ($j->results as $jj) { - // Check if the contact already exists - $exists = q("SELECT `id`, `last_contact`, `last_failure`, `updated` FROM `gcontact` WHERE `nurl` = '%s'", Strings::normaliseLink($jj->url)); - if (DBA::isResult($exists)) { - Logger::log("Profile ".$jj->url." already exists (".$search.")", Logger::DEBUG); - - if (($exists[0]["last_contact"] < $exists[0]["last_failure"]) && - ($exists[0]["updated"] < $exists[0]["last_failure"])) { - continue; - } - // Update the contact - GContact::updateFromProbe($jj->url); - continue; - } - - $server_url = Contact::getBasepath($jj->url); - if ($server_url != '') { - if (!GServer::check($server_url)) { - Logger::log("Friendica server ".$server_url." doesn't answer.", Logger::DEBUG); - continue; - } - Logger::log("Friendica server ".$server_url." seems to be okay.", Logger::DEBUG); - } - - $data = Probe::uri($jj->url); - if ($data["network"] == Protocol::DFRN) { - Logger::log("Profile ".$jj->url." is reachable (".$search.")", Logger::DEBUG); - Logger::log("Add profile ".$jj->url." to local directory (".$search.")", Logger::DEBUG); - - if ($jj->tags != "") { - $data["keywords"] = $jj->tags; - } - - $data["server_url"] = $data["baseurl"]; - - GContact::update($data); - } else { - Logger::log("Profile ".$jj->url." is not responding or no Friendica contact - but network ".$data["network"], Logger::DEBUG); - } - } - } - Cache::set("dirsearch:".$search, time(), Cache::DAY); - } - - /** - * @brief Search for GNU Social user with gstools.org - * - * @param string $search User name - * @return bool - * @throws \Friendica\Network\HTTPException\InternalServerErrorException - * @throws \ImagickException - */ - private static function gsSearchUser($search) { - - // Currently disabled, since the service isn't available anymore. - // It is not removed since I hope that there will be a successor. - return false; - - $url = "http://gstools.org/api/users_search/".urlencode($search); - - $curlResult = Network::curl($url); - if (!$curlResult->isSuccess()) { - return false; - } - - $contacts = json_decode($curlResult->getBody()); - - if ($contacts->status == 'ERROR') { - return false; - } - - /// @TODO AS is considered as a notation for constants (as they usually being written all upper-case) - /// @TODO find all those and convert to all lower-case which is a keyword then - foreach ($contacts->data AS $user) { - $contact = Probe::uri($user->site_address."/".$user->name); - if ($contact["network"] != Protocol::PHANTOM) { - $contact["about"] = $user->description; - GContact::update($contact); - } - } - } } diff --git a/src/Worker/SearchDirectory.php b/src/Worker/SearchDirectory.php new file mode 100644 index 000000000..0f5782db5 --- /dev/null +++ b/src/Worker/SearchDirectory.php @@ -0,0 +1,91 @@ +: Searches for "search pattern" in the directory. + public static function execute($search) + { + if (!Config::get('system', 'poco_local_search')) { + Logger::info('Local search is not enabled'); + return; + } + + self::discoverDirectory($search); + return; + } + + private static function discoverDirectory($search) + { + $data = Cache::get('discoverDirectory' . $search); + if (!is_null($data)) { + // Only search for the same item every 24 hours + if (time() < $data + (60 * 60 * 24)) { + Logger::info('Already searched this in the last 24 hours', ['search' => $search]); + return; + } + } + + $x = Network::fetchUrl(get_server() . '/lsearch?p=1&n=500&search=' . urlencode($search)); + $j = json_decode($x); + + if (!empty($j->results)) { + foreach ($j->results as $jj) { + // Check if the contact already exists + $gcontact = DBA::selectFirst('gcontact', ['id', 'last_contact', 'last_failure', 'updated'], ['nurl' => Strings::normaliseLink($jj->url)]); + if (DBA::isResult($gcontact)) { + Logger::info('Profile already exists', ['profile' => $jj->url, 'search' => $search]); + + if (($gcontact['last_contact'] < $gcontact['last_failure']) && + ($gcontact['updated'] < $gcontact['last_failure'])) { + continue; + } + + // Update the contact + GContact::updateFromProbe($jj->url); + continue; + } + + $server_url = Contact::getBasepath($jj->url); + if ($server_url != '') { + if (!GServer::check($server_url)) { + Logger::log("Friendica server doesn't answer.", ['server' => $server_url]); + continue; + } + Logger::log('Friendica server seems to be okay.', ['server' => $server_url]); + } + + $data = Probe::uri($jj->url); + if ($data['network'] == Protocol::DFRN) { + Logger::log('Add profile to local directory', ['profile' => $jj->url]); + + if ($jj->tags != '') { + $data['keywords'] = $jj->tags; + } + + $data['server_url'] = $data['baseurl']; + + GContact::update($data); + } else { + Logger::log('Profile is not responding or no Friendica contact', ['profile' => $jj->url, 'network' => $data['network']]); + } + } + } + Cache::set('discoverDirectory' . $search, time(), Cache::DAY); + } +} From a1ad549e0b333fc993b2da006060224ba0dbc77f Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 20 Dec 2019 20:37:21 +0000 Subject: [PATCH 03/17] Added UpdateServerDirectory --- src/Protocol/PortableContact.php | 2 +- src/Worker/DiscoverPoCo.php | 3 --- src/Worker/UpdateServerDirectory.php | 18 ++++++++++++++++++ 3 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 src/Worker/UpdateServerDirectory.php diff --git a/src/Protocol/PortableContact.php b/src/Protocol/PortableContact.php index 228a96762..655929f52 100644 --- a/src/Protocol/PortableContact.php +++ b/src/Protocol/PortableContact.php @@ -488,7 +488,7 @@ class PortableContact } Logger::log('Update directory from server ' . $gserver['url'] . ' with ID ' . $gserver['id'], Logger::DEBUG); - Worker::add(PRIORITY_LOW, 'DiscoverPoCo', 'update_server_directory', (int) $gserver['id']); + Worker::add(PRIORITY_LOW, 'UpdateServerDirectory', (int)$gserver['id']); if (!$complete && ( --$no_of_queries == 0)) { break; diff --git a/src/Worker/DiscoverPoCo.php b/src/Worker/DiscoverPoCo.php index 2c3536bf8..47071e20c 100644 --- a/src/Worker/DiscoverPoCo.php +++ b/src/Worker/DiscoverPoCo.php @@ -30,7 +30,6 @@ class DiscoverPoCo - suggestions: Discover other servers for their contacts. - server : Searches for the poco server list. "poco url" is base64 encoded. - update_server: Frequently check the first 250 servers for vitality. - - update_server_directory: Discover the given server id for their contacts - PortableContact::load: Load POCO data from a given POCO address */ @@ -59,8 +58,6 @@ class DiscoverPoCo Logger::log($result, Logger::DEBUG); } elseif ($command == "update_server") { self::updateServer(); - } elseif ($command == "update_server_directory") { - PortableContact::discoverSingleServer(intval($param1)); } elseif ($command == "load") { if (!empty($param4)) { $url = $param4; diff --git a/src/Worker/UpdateServerDirectory.php b/src/Worker/UpdateServerDirectory.php new file mode 100644 index 000000000..df87b20dc --- /dev/null +++ b/src/Worker/UpdateServerDirectory.php @@ -0,0 +1,18 @@ + Date: Fri, 20 Dec 2019 20:49:09 +0000 Subject: [PATCH 04/17] "UpdateSuggestions" added --- src/Worker/Cron.php | 2 +- src/Worker/DiscoverPoCo.php | 3 --- src/Worker/UpdateSuggestions.php | 17 +++++++++++++++++ 3 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 src/Worker/UpdateSuggestions.php diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php index e43fde28e..42a621976 100644 --- a/src/Worker/Cron.php +++ b/src/Worker/Cron.php @@ -75,7 +75,7 @@ class Cron Worker::add(PRIORITY_LOW, "DiscoverPoCo", "update_server"); - Worker::add(PRIORITY_LOW, "DiscoverPoCo", "suggestions"); + Worker::add(PRIORITY_LOW, 'UpdateSuggestions'); Worker::add(PRIORITY_LOW, 'Expire'); diff --git a/src/Worker/DiscoverPoCo.php b/src/Worker/DiscoverPoCo.php index 47071e20c..44b221920 100644 --- a/src/Worker/DiscoverPoCo.php +++ b/src/Worker/DiscoverPoCo.php @@ -27,7 +27,6 @@ class DiscoverPoCo /* This function can be called in these ways: - checkcontact: Updates gcontact entries - - suggestions: Discover other servers for their contacts. - server : Searches for the poco server list. "poco url" is base64 encoded. - update_server: Frequently check the first 250 servers for vitality. - PortableContact::load: Load POCO data from a given POCO address @@ -37,8 +36,6 @@ class DiscoverPoCo $mode = 0; if (($command == "checkcontact") && Config::get('system', 'poco_completion')) { self::discoverUsers(); - } elseif ($command == "suggestions") { - GContact::updateSuggestions(); } elseif ($command == "server") { $server_url = $param1; if ($server_url == "") { diff --git a/src/Worker/UpdateSuggestions.php b/src/Worker/UpdateSuggestions.php new file mode 100644 index 000000000..222244c1c --- /dev/null +++ b/src/Worker/UpdateSuggestions.php @@ -0,0 +1,17 @@ + Date: Fri, 20 Dec 2019 21:04:38 +0000 Subject: [PATCH 05/17] Added "DiscoverContacts" --- src/Worker/Cron.php | 2 +- src/Worker/DiscoverContacts.php | 81 +++++++++++++++++++++++++++++++++ src/Worker/DiscoverPoCo.php | 67 +-------------------------- 3 files changed, 83 insertions(+), 67 deletions(-) create mode 100644 src/Worker/DiscoverContacts.php diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php index 42a621976..011fa4629 100644 --- a/src/Worker/Cron.php +++ b/src/Worker/Cron.php @@ -42,7 +42,7 @@ class Cron Worker::add(PRIORITY_LOW, "DiscoverPoCo"); // run the process to update locally stored global contacts in the background - Worker::add(PRIORITY_LOW, "DiscoverPoCo", "checkcontact"); + Worker::add(PRIORITY_LOW, 'DiscoverContacts'); // Expire and remove user entries Worker::add(PRIORITY_MEDIUM, "CronJobs", "expire_and_remove_users"); diff --git a/src/Worker/DiscoverContacts.php b/src/Worker/DiscoverContacts.php new file mode 100644 index 000000000..88023bccc --- /dev/null +++ b/src/Worker/DiscoverContacts.php @@ -0,0 +1,81 @@ + Protocol::PHANTOM], + ['nurl' => Strings::normaliseLink($contact['url'])]); + continue; + } + + if (in_array($urlparts['host'], ['twitter.com', 'identi.ca'])) { + $networks = ['twitter.com' => Protocol::TWITTER, 'identi.ca' => Protocol::PUMPIO]; + + DBA::update('gcontact', ['network' => $networks[$urlparts['host']]], + ['nurl' => Strings::normaliseLink($contact['url'])]); + continue; + } + + $server_url = Contact::getBasepath($contact['url']); + $force_update = false; + + if (!empty($contact['server_url'])) { + $force_update = (Strings::normaliseLink($contact['server_url']) != Strings::normaliseLink($server_url)); + + $server_url = $contact['server_url']; + } + + if ((empty($server_url) && ($contact['network'] == Protocol::FEED)) || $force_update || GServer::check($server_url, $contact['network'])) { + Logger::info('Check profile', ['profile' => $contact['url']]); + Worker::add(PRIORITY_LOW, 'UpdateGContact', $contact['url'], 'force'); + + if (++$checked > 100) { + return; + } + } else { + DBA::update('gcontact', ['last_failure' => DateTimeFormat::utcNow()], + ['nurl' => Strings::normaliseLink($contact['url'])]); + } + + // Quit the loop after 3 minutes + if (time() > ($starttime + 180)) { + return; + } + } + } +} diff --git a/src/Worker/DiscoverPoCo.php b/src/Worker/DiscoverPoCo.php index 44b221920..508914b48 100644 --- a/src/Worker/DiscoverPoCo.php +++ b/src/Worker/DiscoverPoCo.php @@ -34,9 +34,7 @@ class DiscoverPoCo $search = ""; $mode = 0; - if (($command == "checkcontact") && Config::get('system', 'poco_completion')) { - self::discoverUsers(); - } elseif ($command == "server") { + if ($command == "server") { $server_url = $param1; if ($server_url == "") { return; @@ -110,67 +108,4 @@ class DiscoverPoCo } } } - - private static function discoverUsers() { - Logger::log("Discover users", Logger::DEBUG); - - $starttime = time(); - - $users = q("SELECT `url`, `created`, `updated`, `last_failure`, `last_contact`, `server_url`, `network` FROM `gcontact` - WHERE `last_contact` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND - `last_failure` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND - `network` IN ('%s', '%s', '%s', '%s', '') ORDER BY rand()", - DBA::escape(Protocol::DFRN), DBA::escape(Protocol::DIASPORA), - DBA::escape(Protocol::OSTATUS), DBA::escape(Protocol::FEED)); - - if (!$users) { - return; - } - $checked = 0; - - foreach ($users AS $user) { - - $urlparts = parse_url($user["url"]); - if (!isset($urlparts["scheme"])) { - DBA::update('gcontact', ['network' => Protocol::PHANTOM], - ['nurl' => Strings::normaliseLink($user["url"])]); - continue; - } - - if (in_array($urlparts["host"], ["twitter.com", "identi.ca"])) { - $networks = ["twitter.com" => Protocol::TWITTER, "identi.ca" => Protocol::PUMPIO]; - - DBA::update('gcontact', ['network' => $networks[$urlparts["host"]]], - ['nurl' => Strings::normaliseLink($user["url"])]); - continue; - } - - $server_url = Contact::getBasepath($user["url"]); - $force_update = false; - - if ($user["server_url"] != "") { - - $force_update = (Strings::normaliseLink($user["server_url"]) != Strings::normaliseLink($server_url)); - - $server_url = $user["server_url"]; - } - - if ((($server_url == "") && ($user["network"] == Protocol::FEED)) || $force_update || GServer::check($server_url, $user["network"])) { - Logger::log('Check profile '.$user["url"]); - Worker::add(PRIORITY_LOW, 'UpdateGContact', $user['url'], 'force'); - - if (++$checked > 100) { - return; - } - } else { - DBA::update('gcontact', ['last_failure' => DateTimeFormat::utcNow()], - ['nurl' => Strings::normaliseLink($user["url"])]); - } - - // Quit the loop after 3 minutes - if (time() > ($starttime + 180)) { - return; - } - } - } } From e5546dd40ae6a78f4b5e213941884885e7c48eb7 Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 20 Dec 2019 21:12:44 +0000 Subject: [PATCH 06/17] "UpdateServers" added --- src/Worker/Cron.php | 2 +- src/Worker/DiscoverPoCo.php | 30 --------------------------- src/Worker/UpdateServers.php | 40 ++++++++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 31 deletions(-) create mode 100644 src/Worker/UpdateServers.php diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php index 011fa4629..356be46e4 100644 --- a/src/Worker/Cron.php +++ b/src/Worker/Cron.php @@ -73,7 +73,7 @@ class Cron // update nodeinfo data Worker::add(PRIORITY_LOW, "CronJobs", "nodeinfo"); - Worker::add(PRIORITY_LOW, "DiscoverPoCo", "update_server"); + Worker::add(PRIORITY_LOW, 'UpdateServers'); Worker::add(PRIORITY_LOW, 'UpdateSuggestions'); diff --git a/src/Worker/DiscoverPoCo.php b/src/Worker/DiscoverPoCo.php index 508914b48..cd0968fd8 100644 --- a/src/Worker/DiscoverPoCo.php +++ b/src/Worker/DiscoverPoCo.php @@ -28,7 +28,6 @@ class DiscoverPoCo This function can be called in these ways: - checkcontact: Updates gcontact entries - server : Searches for the poco server list. "poco url" is base64 encoded. - - update_server: Frequently check the first 250 servers for vitality. - PortableContact::load: Load POCO data from a given POCO address */ @@ -51,8 +50,6 @@ class DiscoverPoCo $result .= "failed"; } Logger::log($result, Logger::DEBUG); - } elseif ($command == "update_server") { - self::updateServer(); } elseif ($command == "load") { if (!empty($param4)) { $url = $param4; @@ -81,31 +78,4 @@ class DiscoverPoCo return; } - - /** - * @brief Updates the first 250 servers - * - */ - private static function updateServer() { - $r = q("SELECT `url`, `created`, `last_failure`, `last_contact` FROM `gserver` ORDER BY rand()"); - - if (!DBA::isResult($r)) { - return; - } - - $updated = 0; - - foreach ($r AS $server) { - if (!PortableContact::updateNeeded($server["created"], "", $server["last_failure"], $server["last_contact"])) { - continue; - } - Logger::log('Update server status for server '.$server["url"], Logger::DEBUG); - - Worker::add(PRIORITY_LOW, "DiscoverPoCo", "server", $server["url"]); - - if (++$updated > 250) { - return; - } - } - } } diff --git a/src/Worker/UpdateServers.php b/src/Worker/UpdateServers.php new file mode 100644 index 000000000..a8248db45 --- /dev/null +++ b/src/Worker/UpdateServers.php @@ -0,0 +1,40 @@ + $gserver['url']]); + + Worker::add(PRIORITY_LOW, 'DiscoverPoCo', 'server', $gserver['url']); + + if (++$updated > 250) { + return; + } + } + } +} From 3c4f44a4ab791e134122d1f1fbf259c1fdea164a Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 20 Dec 2019 21:21:25 +0000 Subject: [PATCH 07/17] "FetchPoCo" added --- src/Protocol/PortableContact.php | 2 +- src/Worker/DiscoverPoCo.php | 13 ------------- src/Worker/FetchPoCo.php | 17 +++++++++++++++++ 3 files changed, 18 insertions(+), 14 deletions(-) create mode 100644 src/Worker/FetchPoCo.php diff --git a/src/Protocol/PortableContact.php b/src/Protocol/PortableContact.php index 655929f52..60844df9f 100644 --- a/src/Protocol/PortableContact.php +++ b/src/Protocol/PortableContact.php @@ -57,7 +57,7 @@ class PortableContact public static function loadWorker($cid, $uid = 0, $zcid = 0, $url = null) { // Call the function "load" via the worker - Worker::add(PRIORITY_LOW, "DiscoverPoCo", "load", (int)$cid, (int)$uid, (int)$zcid, $url); + Worker::add(PRIORITY_LOW, 'FetchPoCo', (int)$cid, (int)$uid, (int)$zcid, $url); } /** diff --git a/src/Worker/DiscoverPoCo.php b/src/Worker/DiscoverPoCo.php index cd0968fd8..af486a644 100644 --- a/src/Worker/DiscoverPoCo.php +++ b/src/Worker/DiscoverPoCo.php @@ -4,7 +4,6 @@ */ namespace Friendica\Worker; -use Friendica\Core\Cache; use Friendica\Core\Config; use Friendica\Core\Logger; use Friendica\Core\Protocol; @@ -13,10 +12,7 @@ use Friendica\Database\DBA; use Friendica\Model\GContact; use Friendica\Model\Contact; use Friendica\Model\GServer; -use Friendica\Network\Probe; use Friendica\Protocol\PortableContact; -use Friendica\Util\DateTimeFormat; -use Friendica\Util\Network; use Friendica\Util\Strings; class DiscoverPoCo @@ -26,9 +22,7 @@ class DiscoverPoCo { /* This function can be called in these ways: - - checkcontact: Updates gcontact entries - server : Searches for the poco server list. "poco url" is base64 encoded. - - PortableContact::load: Load POCO data from a given POCO address */ $search = ""; @@ -50,13 +44,6 @@ class DiscoverPoCo $result .= "failed"; } Logger::log($result, Logger::DEBUG); - } elseif ($command == "load") { - if (!empty($param4)) { - $url = $param4; - } else { - $url = ''; - } - PortableContact::load(intval($param1), intval($param2), intval($param3), $url); } elseif ($command !== "") { Logger::log("Unknown or missing parameter ".$command."\n"); return; diff --git a/src/Worker/FetchPoCo.php b/src/Worker/FetchPoCo.php new file mode 100644 index 000000000..372bd3733 --- /dev/null +++ b/src/Worker/FetchPoCo.php @@ -0,0 +1,17 @@ + Date: Fri, 20 Dec 2019 21:27:49 +0000 Subject: [PATCH 08/17] Fix some errors --- src/Worker/DiscoverContacts.php | 2 +- src/Worker/UpdateServerDirectory.php | 4 ++-- src/Worker/UpdateServers.php | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Worker/DiscoverContacts.php b/src/Worker/DiscoverContacts.php index 88023bccc..7af170b3b 100644 --- a/src/Worker/DiscoverContacts.php +++ b/src/Worker/DiscoverContacts.php @@ -27,7 +27,7 @@ class DiscoverContacts $starttime = time(); - $contacts = DBA::select("SELECT `url`, `created`, `updated`, `last_failure`, `last_contact`, `server_url`, `network` FROM `gcontact` + $contacts = DBA::p("SELECT `url`, `created`, `updated`, `last_failure`, `last_contact`, `server_url`, `network` FROM `gcontact` WHERE `last_contact` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND `last_failure` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND `network` IN (?, ?, ?, ?, '') ORDER BY rand()", diff --git a/src/Worker/UpdateServerDirectory.php b/src/Worker/UpdateServerDirectory.php index df87b20dc..e2621073a 100644 --- a/src/Worker/UpdateServerDirectory.php +++ b/src/Worker/UpdateServerDirectory.php @@ -7,12 +7,12 @@ namespace Friendica\Worker; use Friendica\Core\Logger; use Friendica\Protocol\PortableContact; -class UpdateServerDirectory.php +class UpdateServerDirectory { // Discover the given server id for their contacts public static function execute($gserverid) { - PortableContact::discoverSingleServer(gserverid); + PortableContact::discoverSingleServer($gserverid); return; } } diff --git a/src/Worker/UpdateServers.php b/src/Worker/UpdateServers.php index a8248db45..4d288f8fe 100644 --- a/src/Worker/UpdateServers.php +++ b/src/Worker/UpdateServers.php @@ -17,7 +17,7 @@ class UpdateServers */ public static function execute() { - $gservers = DBA::select("SELECT `url`, `created`, `last_failure`, `last_contact` FROM `gserver` ORDER BY rand()"); + $gservers = DBA::p("SELECT `url`, `created`, `last_failure`, `last_contact` FROM `gserver` ORDER BY rand()"); if (!DBA::isResult($gservers)) { return; } From b4f690839a0ef1d560c0e6b3a5b53aa6374495cf Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 20 Dec 2019 21:46:30 +0000 Subject: [PATCH 09/17] "CheckServer" added --- src/Protocol/PortableContact.php | 6 ++-- src/Worker/CheckServer.php | 28 +++++++++++++++++ src/Worker/DiscoverPoCo.php | 54 +++++--------------------------- src/Worker/UpdateServers.php | 2 +- 4 files changed, 39 insertions(+), 51 deletions(-) create mode 100644 src/Worker/CheckServer.php diff --git a/src/Protocol/PortableContact.php b/src/Protocol/PortableContact.php index 60844df9f..1870f8f9b 100644 --- a/src/Protocol/PortableContact.php +++ b/src/Protocol/PortableContact.php @@ -315,7 +315,7 @@ class PortableContact if (!DBA::isResult($r)) { Logger::log("Call server check for server ".$server_url, Logger::DEBUG); - Worker::add(PRIORITY_LOW, "DiscoverPoCo", "server", $server_url); + Worker::add(PRIORITY_LOW, 'CheckServer', $server_url); } } } @@ -340,7 +340,7 @@ class PortableContact if (!empty($servers['pods'])) { foreach ($servers['pods'] as $server) { - Worker::add(PRIORITY_LOW, "DiscoverPoCo", "server", "https://" . $server['host']); + Worker::add(PRIORITY_LOW, 'CheckServer', 'https://' . $server['host']); } } } @@ -359,7 +359,7 @@ class PortableContact foreach ($servers['instances'] as $server) { $url = (is_null($server['https_score']) ? 'http' : 'https') . '://' . $server['name']; - Worker::add(PRIORITY_LOW, "DiscoverPoCo", "server", $url); + Worker::add(PRIORITY_LOW, 'CheckServer', $url); } } } diff --git a/src/Worker/CheckServer.php b/src/Worker/CheckServer.php new file mode 100644 index 000000000..d51b72888 --- /dev/null +++ b/src/Worker/CheckServer.php @@ -0,0 +1,28 @@ + $server_url, 'result' => $ret]); + } +} diff --git a/src/Worker/DiscoverPoCo.php b/src/Worker/DiscoverPoCo.php index af486a644..29b5d4f65 100644 --- a/src/Worker/DiscoverPoCo.php +++ b/src/Worker/DiscoverPoCo.php @@ -6,63 +6,23 @@ namespace Friendica\Worker; use Friendica\Core\Config; use Friendica\Core\Logger; -use Friendica\Core\Protocol; -use Friendica\Core\Worker; -use Friendica\Database\DBA; use Friendica\Model\GContact; -use Friendica\Model\Contact; -use Friendica\Model\GServer; use Friendica\Protocol\PortableContact; -use Friendica\Util\Strings; class DiscoverPoCo { - /// @todo Clean up this mess of a parameter hell and split it in several classes - public static function execute($command = '', $param1 = '', $param2 = '', $param3 = '', $param4 = '') + public static function execute() { - /* - This function can be called in these ways: - - server : Searches for the poco server list. "poco url" is base64 encoded. - */ - - $search = ""; - $mode = 0; - if ($command == "server") { - $server_url = $param1; - if ($server_url == "") { - return; - } - $server_url = filter_var($server_url, FILTER_SANITIZE_URL); - if (substr(Strings::normaliseLink($server_url), 0, 7) != "http://") { - return; - } - $result = "Checking server ".$server_url." - "; - $ret = GServer::check($server_url); - if ($ret) { - $result .= "success"; - } else { - $result .= "failed"; - } - Logger::log($result, Logger::DEBUG); - } elseif ($command !== "") { - Logger::log("Unknown or missing parameter ".$command."\n"); + if (Config::get('system', 'poco_discovery') == PortableContact::DISABLED) { return; } - Logger::log('start '.$search); + // Query Friendica and Hubzilla servers for their users + PortableContact::discover(); - if (($mode == 0) && ($search == "") && (Config::get('system', 'poco_discovery') != PortableContact::DISABLED)) { - // Query Friendica and Hubzilla servers for their users - PortableContact::discover(); - - // Query GNU Social servers for their users ("statistics" addon has to be enabled on the GS server) - if (!Config::get('system', 'ostatus_disabled')) { - GContact::discoverGsUsers(); - } + // Query GNU Social servers for their users ("statistics" addon has to be enabled on the GS server) + if (!Config::get('system', 'ostatus_disabled')) { + GContact::discoverGsUsers(); } - - Logger::log('end '.$search); - - return; } } diff --git a/src/Worker/UpdateServers.php b/src/Worker/UpdateServers.php index 4d288f8fe..d879c5f20 100644 --- a/src/Worker/UpdateServers.php +++ b/src/Worker/UpdateServers.php @@ -30,7 +30,7 @@ class UpdateServers } Logger::info('Update server status', ['server' => $gserver['url']]); - Worker::add(PRIORITY_LOW, 'DiscoverPoCo', 'server', $gserver['url']); + Worker::add(PRIORITY_LOW, 'CheckServer', $gserver['url']); if (++$updated > 250) { return; From 4024d725dbe35425fdadbf760742847a5a9183bd Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 21 Dec 2019 04:23:26 +0000 Subject: [PATCH 10/17] Fixed logger call --- src/Worker/CheckServer.php | 2 +- src/Worker/SearchDirectory.php | 8 ++++---- src/Worker/UpdateServers.php | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Worker/CheckServer.php b/src/Worker/CheckServer.php index d51b72888..ad78d7d25 100644 --- a/src/Worker/CheckServer.php +++ b/src/Worker/CheckServer.php @@ -23,6 +23,6 @@ class CheckServer } $ret = GServer::check($server_url); - Logger::log('Checking server', ['url' => $server_url, 'result' => $ret]); + Logger::info('Checked server', ['url' => $server_url, 'result' => $ret]); } } diff --git a/src/Worker/SearchDirectory.php b/src/Worker/SearchDirectory.php index 0f5782db5..63bc4e0d8 100644 --- a/src/Worker/SearchDirectory.php +++ b/src/Worker/SearchDirectory.php @@ -64,15 +64,15 @@ class SearchDirectory $server_url = Contact::getBasepath($jj->url); if ($server_url != '') { if (!GServer::check($server_url)) { - Logger::log("Friendica server doesn't answer.", ['server' => $server_url]); + Logger::info("Friendica server doesn't answer.", ['server' => $server_url]); continue; } - Logger::log('Friendica server seems to be okay.', ['server' => $server_url]); + Logger::info('Friendica server seems to be okay.', ['server' => $server_url]); } $data = Probe::uri($jj->url); if ($data['network'] == Protocol::DFRN) { - Logger::log('Add profile to local directory', ['profile' => $jj->url]); + Logger::info('Add profile to local directory', ['profile' => $jj->url]); if ($jj->tags != '') { $data['keywords'] = $jj->tags; @@ -82,7 +82,7 @@ class SearchDirectory GContact::update($data); } else { - Logger::log('Profile is not responding or no Friendica contact', ['profile' => $jj->url, 'network' => $data['network']]); + Logger::info('Profile is not responding or no Friendica contact', ['profile' => $jj->url, 'network' => $data['network']]); } } } diff --git a/src/Worker/UpdateServers.php b/src/Worker/UpdateServers.php index d879c5f20..409783298 100644 --- a/src/Worker/UpdateServers.php +++ b/src/Worker/UpdateServers.php @@ -24,7 +24,7 @@ class UpdateServers $updated = 0; - while ($gserver == DBA::fetch($gservers)) { + while ($gserver = DBA::fetch($gservers)) { if (!PortableContact::updateNeeded($gserver['created'], '', $gserver['last_failure'], $gserver['last_contact'])) { continue; } From fab85255b673ef04921db88d721fe0cbda2aeb14 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 21 Dec 2019 06:39:22 +0000 Subject: [PATCH 11/17] Relocated functions --- src/Model/GContact.php | 2 +- src/Model/GServer.php | 71 +++++++++++++++++++++++++++- src/Protocol/PortableContact.php | 49 +------------------ src/Worker/UpdateServerDirectory.php | 7 ++- src/Worker/UpdateServers.php | 4 +- 5 files changed, 78 insertions(+), 55 deletions(-) diff --git a/src/Model/GContact.php b/src/Model/GContact.php index 0ad6df688..61e44b410 100644 --- a/src/Model/GContact.php +++ b/src/Model/GContact.php @@ -783,7 +783,7 @@ class GContact return; } - if (!$force && !PortableContact::updateNeeded($gcontact['created'], $gcontact['updated'], $gcontact['last_failure'], $gcontact['last_contact'])) { + if (!$force && !GServer::updateNeeded($gcontact['created'], $gcontact['updated'], $gcontact['last_failure'], $gcontact['last_contact'])) { Logger::info("Don't update profile", ['url' => $data['url'], 'updated' => $gcontact['updated']]); return; } diff --git a/src/Model/GServer.php b/src/Model/GServer.php index 43aa92f33..7f72302c9 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -49,6 +49,61 @@ class GServer return self::check($server, $network, $force); } + /** + * Decides if a server needs to be updated, based upon several date fields + * + * @param date $created Creation date of that server entry + * @param date $updated When had the server entry be updated + * @param date $last_failure Last failure when contacting that server + * @param date $last_contact Last time the server had been contacted + * + * @return boolean Does the server record needs an update? + */ + public static function updateNeeded($created, $updated, $last_failure, $last_contact) + { + $now = strtotime(DateTimeFormat::utcNow()); + + if ($updated > $last_contact) { + $contact_time = strtotime($updated); + } else { + $contact_time = strtotime($last_contact); + } + + $failure_time = strtotime($last_failure); + $created_time = strtotime($created); + + // If there is no "created" time then use the current time + if ($created_time <= 0) { + $created_time = $now; + } + + // If the last contact was less than 24 hours then don't update + if (($now - $contact_time) < (60 * 60 * 24)) { + return false; + } + + // If the last failure was less than 24 hours then don't update + if (($now - $failure_time) < (60 * 60 * 24)) { + return false; + } + + // If the last contact was less than a week ago and the last failure is older than a week then don't update + //if ((($now - $contact_time) < (60 * 60 * 24 * 7)) && ($contact_time > $failure_time)) + // return false; + + // If the last contact time was more than a week ago and the contact was created more than a week ago, then only try once a week + if ((($now - $contact_time) > (60 * 60 * 24 * 7)) && (($now - $created_time) > (60 * 60 * 24 * 7)) && (($now - $failure_time) < (60 * 60 * 24 * 7))) { + return false; + } + + // If the last contact time was more than a month ago and the contact was created more than a month ago, then only try once a month + if ((($now - $contact_time) > (60 * 60 * 24 * 30)) && (($now - $created_time) > (60 * 60 * 24 * 30)) && (($now - $failure_time) < (60 * 60 * 24 * 30))) { + return false; + } + + return true; + } + /** * Checks the state of the given server. * @@ -89,7 +144,7 @@ class GServer $last_failure = DBA::NULL_DATETIME; } - if (!$force && !PortableContact::updateNeeded($gserver['created'], '', $last_failure, $last_contact)) { + if (!$force && !self::updateNeeded($gserver['created'], '', $last_failure, $last_contact)) { Logger::info('No update needed', ['server' => $server_url]); return ($last_contact >= $last_failure); } @@ -1184,4 +1239,18 @@ class GServer } return $serverdata; } + + /** + * Update the user directory of a given gserver record + * + * @param array $gserver gserver record + */ + public static function updateDirectory(array $gserver) + { + /// @todo Add Mastodon API directory + + if (!empty($gserver['poco'])) { + PortableContact::discoverSingleServer($gserver['id']); + } + } } diff --git a/src/Protocol/PortableContact.php b/src/Protocol/PortableContact.php index 1870f8f9b..d720910c5 100644 --- a/src/Protocol/PortableContact.php +++ b/src/Protocol/PortableContact.php @@ -219,51 +219,6 @@ class PortableContact return(preg_match("=https?://.+/user/\d+=ism", $url, $matches)); } - public static function updateNeeded($created, $updated, $last_failure, $last_contact) - { - $now = strtotime(DateTimeFormat::utcNow()); - - if ($updated > $last_contact) { - $contact_time = strtotime($updated); - } else { - $contact_time = strtotime($last_contact); - } - - $failure_time = strtotime($last_failure); - $created_time = strtotime($created); - - // If there is no "created" time then use the current time - if ($created_time <= 0) { - $created_time = $now; - } - - // If the last contact was less than 24 hours then don't update - if (($now - $contact_time) < (60 * 60 * 24)) { - return false; - } - - // If the last failure was less than 24 hours then don't update - if (($now - $failure_time) < (60 * 60 * 24)) { - return false; - } - - // If the last contact was less than a week ago and the last failure is older than a week then don't update - //if ((($now - $contact_time) < (60 * 60 * 24 * 7)) && ($contact_time > $failure_time)) - // return false; - - // If the last contact time was more than a week ago and the contact was created more than a week ago, then only try once a week - if ((($now - $contact_time) > (60 * 60 * 24 * 7)) && (($now - $created_time) > (60 * 60 * 24 * 7)) && (($now - $failure_time) < (60 * 60 * 24 * 7))) { - return false; - } - - // If the last contact time was more than a month ago and the contact was created more than a month ago, then only try once a month - if ((($now - $contact_time) > (60 * 60 * 24 * 30)) && (($now - $created_time) > (60 * 60 * 24 * 30)) && (($now - $failure_time) < (60 * 60 * 24 * 30))) { - return false; - } - - return true; - } - /** * @brief Returns a list of all known servers * @return array List of server urls @@ -470,7 +425,7 @@ class PortableContact $last_update = date('c', time() - (60 * 60 * 24 * $requery_days)); - $gservers = q("SELECT `id`, `url`, `nurl`, `network` + $gservers = q("SELECT `id`, `url`, `nurl`, `network`, `poco` FROM `gserver` WHERE `last_contact` >= `last_failure` AND `poco` != '' @@ -488,7 +443,7 @@ class PortableContact } Logger::log('Update directory from server ' . $gserver['url'] . ' with ID ' . $gserver['id'], Logger::DEBUG); - Worker::add(PRIORITY_LOW, 'UpdateServerDirectory', (int)$gserver['id']); + Worker::add(PRIORITY_LOW, 'UpdateServerDirectory', $gserver); if (!$complete && ( --$no_of_queries == 0)) { break; diff --git a/src/Worker/UpdateServerDirectory.php b/src/Worker/UpdateServerDirectory.php index e2621073a..262da2d30 100644 --- a/src/Worker/UpdateServerDirectory.php +++ b/src/Worker/UpdateServerDirectory.php @@ -5,14 +5,13 @@ namespace Friendica\Worker; use Friendica\Core\Logger; -use Friendica\Protocol\PortableContact; +use Friendica\Model\GServer; class UpdateServerDirectory { - // Discover the given server id for their contacts - public static function execute($gserverid) + public static function execute($gserver) { - PortableContact::discoverSingleServer($gserverid); + GServer::updateDirectory($gserver); return; } } diff --git a/src/Worker/UpdateServers.php b/src/Worker/UpdateServers.php index 409783298..d3ce3fca7 100644 --- a/src/Worker/UpdateServers.php +++ b/src/Worker/UpdateServers.php @@ -7,7 +7,7 @@ namespace Friendica\Worker; use Friendica\Core\Logger; use Friendica\Core\Worker; use Friendica\Database\DBA; -use Friendica\Protocol\PortableContact; +use Friendica\Model\GServer; class UpdateServers { @@ -25,7 +25,7 @@ class UpdateServers $updated = 0; while ($gserver = DBA::fetch($gservers)) { - if (!PortableContact::updateNeeded($gserver['created'], '', $gserver['last_failure'], $gserver['last_contact'])) { + if (!GServer::updateNeeded($gserver['created'], '', $gserver['last_failure'], $gserver['last_contact'])) { continue; } Logger::info('Update server status', ['server' => $gserver['url']]); From 90408b9d49d11b08721ab742b256e820cc622b2a Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 21 Dec 2019 13:48:20 +0000 Subject: [PATCH 12/17] Discover directory type --- database.sql | 3 +- src/Model/GServer.php | 67 ++++++++++++++++++++++++++++++----- static/dbstructure.config.php | 3 +- 3 files changed, 63 insertions(+), 10 deletions(-) diff --git a/database.sql b/database.sql index cbb724d18..6a55db526 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 2019.12-rc (Dalmatian Bellflower) --- DB_UPDATE_VERSION 1326 +-- DB_UPDATE_VERSION 1328 -- ------------------------------------------ @@ -470,6 +470,7 @@ CREATE TABLE IF NOT EXISTS `gserver` ( `info` text COMMENT '', `register_policy` tinyint NOT NULL DEFAULT 0 COMMENT '', `registered-users` int unsigned NOT NULL DEFAULT 0 COMMENT 'Number of registered users', + `directory-type` tinyint DEFAULT 0 COMMENT 'Type of directory service (Poco, Mastodon)', `poco` varchar(255) NOT NULL DEFAULT '' COMMENT '', `noscrape` varchar(255) NOT NULL DEFAULT '' COMMENT '', `network` char(4) NOT NULL DEFAULT '' COMMENT '', diff --git a/src/Model/GServer.php b/src/Model/GServer.php index 7f72302c9..ff9be47a9 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -26,6 +26,10 @@ use Friendica\Network\Probe; */ class GServer { + // Directory types + const DT_NONE = 0; + const DT_POCO = 1; + const DT_MASTODON = 2; /** * Checks if the given server is reachable * @@ -51,12 +55,12 @@ class GServer /** * Decides if a server needs to be updated, based upon several date fields - * + * * @param date $created Creation date of that server entry * @param date $updated When had the server entry be updated * @param date $last_failure Last failure when contacting that server * @param date $last_contact Last time the server had been contacted - * + * * @return boolean Does the server record needs an update? */ public static function updateNeeded($created, $updated, $last_failure, $last_contact) @@ -167,8 +171,24 @@ class GServer */ public static function detect(string $url, string $network = '') { + Logger::info('Detect server type', ['server' => $url]); $serverdata = []; + $original_url = $url; + + // Remove URL content that is not supposed to exist for a server url + $urlparts = parse_url($url); + unset($urlparts['user']); + unset($urlparts['pass']); + unset($urlparts['query']); + unset($urlparts['fragment']); + $url = Network::unparseURL($urlparts); + + // If the URL missmatches, then we mark the old entry as failure + if ($url != $original_url) { + DBA::update('gserver', ['last_failure' => DateTimeFormat::utcNow()], ['nurl' => Strings::normaliseLink($original_url)]); + } + // When a nodeinfo is present, we don't need to dig further $xrd_timeout = Config::get('system', 'xrd_timeout'); $curlResult = Network::curl($url . '/.well-known/nodeinfo', false, ['timeout' => $xrd_timeout]); @@ -238,7 +258,10 @@ class GServer $serverdata = $nodeinfo; } + // Detect the directory type + $serverdata['directory-type'] = self::DT_NONE; $serverdata = self::checkPoCo($url, $serverdata); + $serverdata = self::checkMastodonDirectory($url, $serverdata); // We can't detect the network type. Possibly it is some system that we don't know yet if (empty($serverdata['network'])) { @@ -604,7 +627,7 @@ class GServer $protocols[$protocol] = true; } - if (!empty($protocols['friendica'])) { + if (!empty($protocols['dfrn'])) { $server['network'] = Protocol::DFRN; } elseif (!empty($protocols['activitypub'])) { $server['network'] = Protocol::ACTIVITYPUB; @@ -795,6 +818,8 @@ class GServer */ private static function checkPoCo(string $url, array $serverdata) { + $serverdata['poco'] = ''; + $curlResult = Network::curl($url. '/poco'); if (!$curlResult->isSuccess()) { return $serverdata; @@ -808,9 +833,35 @@ class GServer if (!empty($data['totalResults'])) { $registeredUsers = $serverdata['registered-users'] ?? 0; $serverdata['registered-users'] = max($data['totalResults'], $registeredUsers); + $serverdata['directory-type'] = self::DT_POCO; $serverdata['poco'] = $url . '/poco'; - } else { - $serverdata['poco'] = ''; + } + + return $serverdata; + } + + /** + * Checks if the given server does have a Mastodon style directory endpoint. + * + * @param string $url URL of the given server + * @param array $serverdata array with server data + * + * @return array server data + */ + public static function checkMastodonDirectory(string $url, array $serverdata) + { + $curlResult = Network::curl($url. '/api/v1/directory?limit=1'); + if (!$curlResult->isSuccess()) { + return $serverdata; + } + + $data = json_decode($curlResult->getBody(), true); + if (empty($data)) { + return $serverdata; + } + + if (count($data) == 1) { + $serverdata['directory-type'] = self::DT_MASTODON; } return $serverdata; @@ -1242,13 +1293,13 @@ class GServer /** * Update the user directory of a given gserver record - * - * @param array $gserver gserver record + * + * @param array $gserver gserver record */ public static function updateDirectory(array $gserver) { /// @todo Add Mastodon API directory - + if (!empty($gserver['poco'])) { PortableContact::discoverSingleServer($gserver['id']); } diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index bd8da30f6..ada837fb2 100755 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -34,7 +34,7 @@ use Friendica\Database\DBA; if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1327); + define('DB_UPDATE_VERSION', 1328); } return [ @@ -526,6 +526,7 @@ return [ "info" => ["type" => "text", "comment" => ""], "register_policy" => ["type" => "tinyint", "not null" => "1", "default" => "0", "comment" => ""], "registered-users" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "comment" => "Number of registered users"], + "directory-type" => ["type" => "tinyint", "default" => "0", "comment" => "Type of directory service (Poco, Mastodon)"], "poco" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], "noscrape" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], "network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => ""], From 26ac9a0063dba3eeb1a20d4fe8648e20386bf364 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 21 Dec 2019 18:11:51 +0000 Subject: [PATCH 13/17] "CheckServer" is now "UpdateGServer" --- src/Protocol/PortableContact.php | 6 +++--- src/Worker/{CheckServer.php => UpdateGServer.php} | 6 +++--- src/Worker/UpdateServers.php | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) rename src/Worker/{CheckServer.php => UpdateGServer.php} (77%) diff --git a/src/Protocol/PortableContact.php b/src/Protocol/PortableContact.php index d720910c5..49da85e61 100644 --- a/src/Protocol/PortableContact.php +++ b/src/Protocol/PortableContact.php @@ -270,7 +270,7 @@ class PortableContact if (!DBA::isResult($r)) { Logger::log("Call server check for server ".$server_url, Logger::DEBUG); - Worker::add(PRIORITY_LOW, 'CheckServer', $server_url); + Worker::add(PRIORITY_LOW, 'UpdateGServer', $server_url); } } } @@ -295,7 +295,7 @@ class PortableContact if (!empty($servers['pods'])) { foreach ($servers['pods'] as $server) { - Worker::add(PRIORITY_LOW, 'CheckServer', 'https://' . $server['host']); + Worker::add(PRIORITY_LOW, 'UpdateGServer', 'https://' . $server['host']); } } } @@ -314,7 +314,7 @@ class PortableContact foreach ($servers['instances'] as $server) { $url = (is_null($server['https_score']) ? 'http' : 'https') . '://' . $server['name']; - Worker::add(PRIORITY_LOW, 'CheckServer', $url); + Worker::add(PRIORITY_LOW, 'UpdateGServer', $url); } } } diff --git a/src/Worker/CheckServer.php b/src/Worker/UpdateGServer.php similarity index 77% rename from src/Worker/CheckServer.php rename to src/Worker/UpdateGServer.php index ad78d7d25..2e017422f 100644 --- a/src/Worker/CheckServer.php +++ b/src/Worker/UpdateGServer.php @@ -1,6 +1,6 @@ $server_url, 'result' => $ret]); + Logger::info('Updated gserver', ['url' => $server_url, 'result' => $ret]); } } diff --git a/src/Worker/UpdateServers.php b/src/Worker/UpdateServers.php index d3ce3fca7..d1cc04209 100644 --- a/src/Worker/UpdateServers.php +++ b/src/Worker/UpdateServers.php @@ -30,7 +30,7 @@ class UpdateServers } Logger::info('Update server status', ['server' => $gserver['url']]); - Worker::add(PRIORITY_LOW, 'CheckServer', $gserver['url']); + Worker::add(PRIORITY_LOW, 'UpdateGServer', $gserver['url']); if (++$updated > 250) { return; From 33cb241ed3283f18961a867ca6f99f9b8b76ec4a Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 21 Dec 2019 18:24:43 +0000 Subject: [PATCH 14/17] Renamed function --- src/Worker/SearchDirectory.php | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/Worker/SearchDirectory.php b/src/Worker/SearchDirectory.php index 63bc4e0d8..4fd6d44d5 100644 --- a/src/Worker/SearchDirectory.php +++ b/src/Worker/SearchDirectory.php @@ -26,13 +26,7 @@ class SearchDirectory return; } - self::discoverDirectory($search); - return; - } - - private static function discoverDirectory($search) - { - $data = Cache::get('discoverDirectory' . $search); + $data = Cache::get('SearchDirectory:' . $search); if (!is_null($data)) { // Only search for the same item every 24 hours if (time() < $data + (60 * 60 * 24)) { @@ -86,6 +80,6 @@ class SearchDirectory } } } - Cache::set('discoverDirectory' . $search, time(), Cache::DAY); + Cache::set('SearchDirectory:' . $search, time(), Cache::DAY); } } From 537da657cca1460462f91d788857276e1c21a16d Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 21 Dec 2019 18:57:00 +0000 Subject: [PATCH 15/17] Renamed functions --- src/Worker/Cron.php | 8 ++++---- src/Worker/{DiscoverContacts.php => UpdateGContacts.php} | 4 ++-- src/Worker/{UpdateServers.php => UpdateGServers.php} | 4 ++-- .../{DiscoverPoCo.php => UpdateServerDirectories.php} | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) rename src/Worker/{DiscoverContacts.php => UpdateGContacts.php} (97%) rename src/Worker/{UpdateServers.php => UpdateGServers.php} (92%) rename src/Worker/{DiscoverPoCo.php => UpdateServerDirectories.php} (88%) diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php index 356be46e4..6928cb8d7 100644 --- a/src/Worker/Cron.php +++ b/src/Worker/Cron.php @@ -38,11 +38,11 @@ class Cron // Fork the cron jobs in separate parts to avoid problems when one of them is crashing Hook::fork($a->queue['priority'], "cron"); - // run the process to discover global contacts in the background - Worker::add(PRIORITY_LOW, "DiscoverPoCo"); + // run the process to update server directories in the background + Worker::add(PRIORITY_LOW, 'UpdateServerDirectories'); // run the process to update locally stored global contacts in the background - Worker::add(PRIORITY_LOW, 'DiscoverContacts'); + Worker::add(PRIORITY_LOW, 'UpdateGContacts'); // Expire and remove user entries Worker::add(PRIORITY_MEDIUM, "CronJobs", "expire_and_remove_users"); @@ -73,7 +73,7 @@ class Cron // update nodeinfo data Worker::add(PRIORITY_LOW, "CronJobs", "nodeinfo"); - Worker::add(PRIORITY_LOW, 'UpdateServers'); + Worker::add(PRIORITY_LOW, 'UpdateGServers'); Worker::add(PRIORITY_LOW, 'UpdateSuggestions'); diff --git a/src/Worker/DiscoverContacts.php b/src/Worker/UpdateGContacts.php similarity index 97% rename from src/Worker/DiscoverContacts.php rename to src/Worker/UpdateGContacts.php index 7af170b3b..64ffb2a50 100644 --- a/src/Worker/DiscoverContacts.php +++ b/src/Worker/UpdateGContacts.php @@ -1,6 +1,6 @@ Date: Sat, 21 Dec 2019 20:18:44 +0000 Subject: [PATCH 16/17] Added documentation --- src/Worker/FetchPoCo.php | 9 ++++++++- src/Worker/UpdateContact.php | 5 +++++ src/Worker/UpdateGContact.php | 5 +++++ src/Worker/UpdateGContacts.php | 10 ++++++---- src/Worker/UpdateGServer.php | 5 ++++- src/Worker/UpdateGServers.php | 3 +-- src/Worker/UpdateServerDirectories.php | 3 +++ src/Worker/UpdateServerDirectory.php | 4 ++++ src/Worker/UpdateSuggestions.php | 4 +++- 9 files changed, 39 insertions(+), 9 deletions(-) diff --git a/src/Worker/FetchPoCo.php b/src/Worker/FetchPoCo.php index 372bd3733..67f2e548b 100644 --- a/src/Worker/FetchPoCo.php +++ b/src/Worker/FetchPoCo.php @@ -9,7 +9,14 @@ use Friendica\Protocol\PortableContact; class FetchPoCo { - // Load POCO data from a given POCO address + /** + * Fetch PortableContacts from a given PoCo server address + * + * @param integer $cid Contact ID + * @param integer $uid User ID + * @param integer $zcid Global Contact ID + * @param integer $url PoCo address that should be polled + */ public static function execute($cid, $uid, $zcid, $url) { PortableContact::load($cid, $uid, $zcid, $url); diff --git a/src/Worker/UpdateContact.php b/src/Worker/UpdateContact.php index f23c5c0a0..0e5fd1412 100644 --- a/src/Worker/UpdateContact.php +++ b/src/Worker/UpdateContact.php @@ -13,6 +13,11 @@ use Friendica\Database\DBA; class UpdateContact { + /** + * Update contact data via probe + * @param int $contact_id Contact ID + * @param string $command + */ public static function execute($contact_id, $command = '') { $force = ($command == "force"); diff --git a/src/Worker/UpdateGContact.php b/src/Worker/UpdateGContact.php index aacebcb80..1057d0a27 100644 --- a/src/Worker/UpdateGContact.php +++ b/src/Worker/UpdateGContact.php @@ -12,6 +12,11 @@ use Friendica\Database\DBA; class UpdateGContact { + /** + * Update global contact via probe + * @param string $url Global contact url + * @param string $command + */ public static function execute($url, $command = '') { $force = ($command == "force"); diff --git a/src/Worker/UpdateGContacts.php b/src/Worker/UpdateGContacts.php index 64ffb2a50..1d9d86bcf 100644 --- a/src/Worker/UpdateGContacts.php +++ b/src/Worker/UpdateGContacts.php @@ -16,22 +16,24 @@ use Friendica\Util\Strings; class UpdateGContacts { - // Updates gcontact entries + /** + * Updates global contacts + */ public static function execute() { if (!Config::get('system', 'poco_completion')) { return; } - Logger::info('Discover contacts'); + Logger::info('Update global contacts'); $starttime = time(); $contacts = DBA::p("SELECT `url`, `created`, `updated`, `last_failure`, `last_contact`, `server_url`, `network` FROM `gcontact` WHERE `last_contact` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND `last_failure` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND - `network` IN (?, ?, ?, ?, '') ORDER BY rand()", - Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS, Protocol::FEED); + `network` IN (?, ?, ?, ?, ?, '') ORDER BY rand()", + Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS, Protocol::FEED); $checked = 0; diff --git a/src/Worker/UpdateGServer.php b/src/Worker/UpdateGServer.php index 2e017422f..b94c5343a 100644 --- a/src/Worker/UpdateGServer.php +++ b/src/Worker/UpdateGServer.php @@ -10,7 +10,10 @@ use Friendica\Util\Strings; class UpdateGServer { - // Searches for the poco server list. + /** + * Update the given server + * @param string $server_url Server URL + */ public static function execute($server_url) { if (empty($server_url)) { diff --git a/src/Worker/UpdateGServers.php b/src/Worker/UpdateGServers.php index a245c34fc..d55d8df01 100644 --- a/src/Worker/UpdateGServers.php +++ b/src/Worker/UpdateGServers.php @@ -12,8 +12,7 @@ use Friendica\Model\GServer; class UpdateGServers { /** - * @brief Updates the first 250 servers - * + * Updates the first 250 servers */ public static function execute() { diff --git a/src/Worker/UpdateServerDirectories.php b/src/Worker/UpdateServerDirectories.php index e1b12034a..433685cb2 100644 --- a/src/Worker/UpdateServerDirectories.php +++ b/src/Worker/UpdateServerDirectories.php @@ -11,6 +11,9 @@ use Friendica\Protocol\PortableContact; class UpdateServerDirectories { + /** + * Query global servers for their users + */ public static function execute() { if (Config::get('system', 'poco_discovery') == PortableContact::DISABLED) { diff --git a/src/Worker/UpdateServerDirectory.php b/src/Worker/UpdateServerDirectory.php index 262da2d30..fd5ee0534 100644 --- a/src/Worker/UpdateServerDirectory.php +++ b/src/Worker/UpdateServerDirectory.php @@ -9,6 +9,10 @@ use Friendica\Model\GServer; class UpdateServerDirectory { + /** + * Query the given server for their users + * @param string $gserver Server URL + */ public static function execute($gserver) { GServer::updateDirectory($gserver); diff --git a/src/Worker/UpdateSuggestions.php b/src/Worker/UpdateSuggestions.php index 222244c1c..7d4543409 100644 --- a/src/Worker/UpdateSuggestions.php +++ b/src/Worker/UpdateSuggestions.php @@ -9,7 +9,9 @@ use Friendica\Model\GContact; class UpdateSuggestions { - // Discover other servers for their contacts. + /** + * Discover other servers for their contacts. + */ public static function execute() { GContact::updateSuggestions(); From 8608e8ff63d0d3fd53d40fc053227828796567d4 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 24 Dec 2019 05:04:11 +0000 Subject: [PATCH 17/17] Standards --- src/Model/GServer.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Model/GServer.php b/src/Model/GServer.php index ff9be47a9..3fffd4ed4 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -850,7 +850,7 @@ class GServer */ public static function checkMastodonDirectory(string $url, array $serverdata) { - $curlResult = Network::curl($url. '/api/v1/directory?limit=1'); + $curlResult = Network::curl($url . '/api/v1/directory?limit=1'); if (!$curlResult->isSuccess()) { return $serverdata; }