From 356a2a9a5a68f6f66f430f7268429a2c4236e4e0 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 17 Jan 2023 17:25:19 +0000 Subject: [PATCH] News "blocked" field on the gserver table --- database.sql | 3 +- doc/database/db_gserver.md | 1 + src/Console/ServerBlock.php | 4 ++ src/Model/GServer.php | 65 ++++++++++++++++--- src/Module/Api/Mastodon/Instance/Peers.php | 2 +- .../Moderation/Blocklist/Server/Add.php | 2 + .../Moderation/Blocklist/Server/Import.php | 3 + .../Moderation/Blocklist/Server/Index.php | 3 + src/Worker/Cron.php | 5 +- src/Worker/UpdateBlockedServers.php | 53 +++++++++++++++ src/Worker/UpdateGServer.php | 2 + src/Worker/UpdateGServers.php | 3 +- src/Worker/UpdateServerPeers.php | 6 ++ static/dbstructure.config.php | 3 +- 14 files changed, 140 insertions(+), 15 deletions(-) create mode 100644 src/Worker/UpdateBlockedServers.php diff --git a/database.sql b/database.sql index e06454b11..48e0930e6 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 2023.03-dev (Giant Rhubarb) --- DB_UPDATE_VERSION 1510 +-- DB_UPDATE_VERSION 1511 -- ------------------------------------------ @@ -34,6 +34,7 @@ CREATE TABLE IF NOT EXISTS `gserver` ( `last_poco_query` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '', `last_contact` datetime DEFAULT '0001-01-01 00:00:00' COMMENT 'Last successful connection request', `last_failure` datetime DEFAULT '0001-01-01 00:00:00' COMMENT 'Last failed connection request', + `blocked` boolean COMMENT 'Server is blocked', `failed` boolean COMMENT 'Connection failed', `next_contact` datetime DEFAULT '0001-01-01 00:00:00' COMMENT 'Next connection request', PRIMARY KEY(`id`), diff --git a/doc/database/db_gserver.md b/doc/database/db_gserver.md index b75ac0f3b..8ba9e3d9b 100644 --- a/doc/database/db_gserver.md +++ b/doc/database/db_gserver.md @@ -34,6 +34,7 @@ Fields | last_poco_query | | datetime | YES | | 0001-01-01 00:00:00 | | | last_contact | Last successful connection request | datetime | YES | | 0001-01-01 00:00:00 | | | last_failure | Last failed connection request | datetime | YES | | 0001-01-01 00:00:00 | | +| blocked | Server is blocked | boolean | YES | | NULL | | | failed | Connection failed | boolean | YES | | NULL | | | next_contact | Next connection request | datetime | YES | | 0001-01-01 00:00:00 | | diff --git a/src/Console/ServerBlock.php b/src/Console/ServerBlock.php index 46e7b89eb..ed9ff5dbd 100644 --- a/src/Console/ServerBlock.php +++ b/src/Console/ServerBlock.php @@ -24,6 +24,7 @@ namespace Friendica\Console; use Asika\SimpleConsole\CommandArgsException; use Asika\SimpleConsole\Console; use Console_Table; +use Friendica\Core\Worker; use Friendica\Moderation\DomainPatternBlocklist; /** @@ -127,6 +128,7 @@ HELP; if ($this->blocklist->append($newBlockList)) { $this->out(sprintf("Entries from %s that were not blocked before are now blocked", $filename)); + Worker::add(Worker::PRIORITY_LOW, 'UpdateBlockedServers'); return 0; } else { $this->out("Couldn't save the block list"); @@ -169,6 +171,7 @@ HELP; } else { $this->out(sprintf("The domain pattern '%s' is now blocked. (Reason: '%s')", $pattern, $reason)); } + Worker::add(Worker::PRIORITY_LOW, 'UpdateBlockedServers'); return 0; } else { $this->out(sprintf("Couldn't save '%s' as blocked domain pattern", $pattern)); @@ -193,6 +196,7 @@ HELP; if ($result) { if ($result == 2) { $this->out(sprintf("The domain pattern '%s' isn't blocked anymore", $pattern)); + Worker::add(Worker::PRIORITY_LOW, 'UpdateBlockedServers'); return 0; } else { $this->out(sprintf("The domain pattern '%s' wasn't blocked.", $pattern)); diff --git a/src/Model/GServer.php b/src/Model/GServer.php index a3a3d1abb..54e4d7220 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -127,6 +127,13 @@ class GServer $gserver = DBA::selectFirst('gserver', ['id'], ['nurl' => Strings::normaliseLink($url)]); if (DBA::isResult($gserver)) { Logger::debug('Got ID for URL', ['id' => $gserver['id'], 'url' => $url, 'callstack' => System::callstack(20)]); + + if (Network::isUrlBlocked($url)) { + self::setBlockedById($gserver['id']); + } else { + self::setUnblockedById($gserver['id']); + } + return $gserver['id']; } @@ -341,6 +348,12 @@ class GServer return false; } + if (Network::isUrlBlocked($server_url)) { + Logger::info('Server is blocked', ['url' => $server_url]); + self::setBlockedByUrl($server_url); + return false; + } + $gserver = DBA::selectFirst('gserver', [], ['nurl' => Strings::normaliseLink($server_url)]); if (DBA::isResult($gserver)) { if ($gserver['created'] <= DBA::NULL_DATETIME) { @@ -370,8 +383,13 @@ class GServer public static function setReachableById(int $gsid, string $network) { $gserver = DBA::selectFirst('gserver', ['url', 'failed', 'next_contact', 'network'], ['id' => $gsid]); - if (DBA::isResult($gserver) && $gserver['failed']) { - $fields = ['failed' => false, 'last_contact' => DateTimeFormat::utcNow()]; + if (!DBA::isResult($gserver)) { + return; + } + + $blocked = Network::isUrlBlocked($gserver['url']); + if ($gserver['failed']) { + $fields = ['failed' => false, 'blocked' => $blocked, 'last_contact' => DateTimeFormat::utcNow()]; if (!empty($network) && !in_array($gserver['network'], Protocol::FEDERATED)) { $fields['network'] = $network; } @@ -381,6 +399,10 @@ class GServer if (strtotime($gserver['next_contact']) < time()) { UpdateGServer::add(Worker::PRIORITY_LOW, $gserver['url']); } + } elseif ($blocked) { + self::setBlockedById($gsid); + } else { + self::setUnblockedById($gsid); } } @@ -393,7 +415,7 @@ class GServer { $gserver = DBA::selectFirst('gserver', ['url', 'failed', 'next_contact'], ['id' => $gsid]); if (DBA::isResult($gserver) && !$gserver['failed']) { - self::update(['failed' => true, 'last_failure' => DateTimeFormat::utcNow()], ['id' => $gsid]); + self::update(['failed' => true, 'blocked' => Network::isUrlBlocked($gserver['url']), 'last_failure' => DateTimeFormat::utcNow()], ['id' => $gsid]); Logger::info('Set failed status for server', ['url' => $gserver['url']]); if (strtotime($gserver['next_contact']) < time()) { @@ -402,6 +424,33 @@ class GServer } } + public static function setUnblockedById(int $gsid) + { + $gserver = DBA::selectFirst('gserver', ['url'], ["(`blocked` OR `blocked` IS NULL) AND `id` = ?", $gsid]); + if (DBA::isResult($gserver)) { + self::update(['blocked' => false], ['id' => $gsid]); + Logger::info('Set unblocked status for server', ['url' => $gserver['url']]); + } + } + + public static function setBlockedById(int $gsid) + { + $gserver = DBA::selectFirst('gserver', ['url'], ["(NOT `blocked` OR `blocked` IS NULL) AND `id` = ?", $gsid]); + if (DBA::isResult($gserver)) { + self::update(['blocked' => true, 'failed' => true], ['id' => $gsid]); + Logger::info('Set blocked status for server', ['url' => $gserver['url']]); + } + } + + public static function setBlockedByUrl(string $url) + { + $gserver = DBA::selectFirst('gserver', ['url', 'id'], ["(NOT `blocked` OR `blocked` IS NULL) AND `nurl` = ?", Strings::normaliseLink($url)]); + if (DBA::isResult($gserver)) { + self::update(['blocked' => true, 'failed' => true], ['id' => $gserver['id']]); + Logger::info('Set blocked status for server', ['url' => $gserver['url']]); + } + } + /** * Set failed server status * @@ -412,7 +461,7 @@ class GServer $gserver = DBA::selectFirst('gserver', [], ['nurl' => Strings::normaliseLink($url)]); if (DBA::isResult($gserver)) { $next_update = self::getNextUpdateDate(false, $gserver['created'], $gserver['last_contact']); - self::update(['url' => $url, 'failed' => true, 'last_failure' => DateTimeFormat::utcNow(), + self::update(['url' => $url, 'failed' => true, 'blocked' => Network::isUrlBlocked($url), 'last_failure' => DateTimeFormat::utcNow(), 'next_contact' => $next_update, 'network' => Protocol::PHANTOM, 'detection-method' => null], ['nurl' => Strings::normaliseLink($url)]); Logger::info('Set failed status for existing server', ['url' => $url]); @@ -491,7 +540,7 @@ class GServer * * @return boolean 'true' if server could be detected */ - public static function detect(string $url, string $network = '', bool $only_nodeinfo = false): bool + private static function detect(string $url, string $network = '', bool $only_nodeinfo = false): bool { Logger::info('Detect server type', ['server' => $url]); @@ -732,9 +781,9 @@ class GServer } $serverdata['next_contact'] = self::getNextUpdateDate(true, '', '', in_array($serverdata['network'], [Protocol::PHANTOM, Protocol::FEED])); - $serverdata['last_contact'] = DateTimeFormat::utcNow(); - $serverdata['failed'] = false; + $serverdata['failed'] = false; + $serverdata['blocked'] = false; $gserver = DBA::selectFirst('gserver', ['network'], ['nurl' => Strings::normaliseLink($url)]); if (!DBA::isResult($gserver)) { @@ -2259,7 +2308,7 @@ class GServer $last_update = date('c', time() - (60 * 60 * 24 * $requery_days)); $gservers = DBA::select('gserver', ['id', 'url', 'nurl', 'network', 'poco', 'directory-type'], - ["NOT `failed` AND `directory-type` != ? AND `last_poco_query` < ?", GServer::DT_NONE, $last_update], + ["NOT `blocked` AND NOT `failed` AND `directory-type` != ? AND `last_poco_query` < ?", GServer::DT_NONE, $last_update], ['order' => ['RAND()']]); while ($gserver = DBA::fetch($gservers)) { diff --git a/src/Module/Api/Mastodon/Instance/Peers.php b/src/Module/Api/Mastodon/Instance/Peers.php index 689858c61..b33f07b69 100644 --- a/src/Module/Api/Mastodon/Instance/Peers.php +++ b/src/Module/Api/Mastodon/Instance/Peers.php @@ -42,7 +42,7 @@ class Peers extends BaseApi $return = []; // We only select for Friendica and ActivityPub servers, since it is expected to only deliver AP compatible systems here. - $instances = DBA::select('gserver', ['url'], ["`network` in (?, ?) AND NOT `failed` AND NOT `detection-method` IN (?, ?, ?, ?)", + $instances = DBA::select('gserver', ['url'], ["`network` in (?, ?) AND NOT `blocked` AND NOT `failed` AND NOT `detection-method` IN (?, ?, ?, ?)", Protocol::DFRN, Protocol::ACTIVITYPUB, GServer::DETECT_MANUAL, GServer::DETECT_HEADER, GServer::DETECT_BODY, GServer::DETECT_HOST_META]); while ($instance = DBA::fetch($instances)) { diff --git a/src/Module/Moderation/Blocklist/Server/Add.php b/src/Module/Moderation/Blocklist/Server/Add.php index 10d47702d..7a7d3dacf 100644 --- a/src/Module/Moderation/Blocklist/Server/Add.php +++ b/src/Module/Moderation/Blocklist/Server/Add.php @@ -75,6 +75,8 @@ class Add extends BaseModeration // Add new item to blocklist $this->blocklist->addPattern($pattern, trim($request['reason'])); + Worker::add(Worker::PRIORITY_LOW, 'UpdateBlockedServers'); + $this->systemMessages->addInfo($this->t('Server domain pattern added to the blocklist.')); if (!empty($request['purge'])) { diff --git a/src/Module/Moderation/Blocklist/Server/Import.php b/src/Module/Moderation/Blocklist/Server/Import.php index d73889b04..62086e7cc 100644 --- a/src/Module/Moderation/Blocklist/Server/Import.php +++ b/src/Module/Moderation/Blocklist/Server/Import.php @@ -25,6 +25,7 @@ use Friendica\App; use Friendica\Core\L10n; use Friendica\Core\Renderer; use Friendica\Core\Session\Capability\IHandleUserSessions; +use Friendica\Core\Worker; use Friendica\Moderation\DomainPatternBlocklist; use Friendica\Module\Response; use Friendica\Navigation\SystemMessages; @@ -95,6 +96,8 @@ class Import extends \Friendica\Module\BaseModeration } } + Worker::add(Worker::PRIORITY_LOW, 'UpdateBlockedServers'); + $this->baseUrl->redirect('/moderation/blocklist/server'); } } diff --git a/src/Module/Moderation/Blocklist/Server/Index.php b/src/Module/Moderation/Blocklist/Server/Index.php index ae4320fb5..b52641c67 100644 --- a/src/Module/Moderation/Blocklist/Server/Index.php +++ b/src/Module/Moderation/Blocklist/Server/Index.php @@ -25,6 +25,7 @@ use Friendica\App; use Friendica\Core\L10n; use Friendica\Core\Renderer; use Friendica\Core\Session\Capability\IHandleUserSessions; +use Friendica\Core\Worker; use Friendica\Moderation\DomainPatternBlocklist; use Friendica\Module\BaseModeration; use Friendica\Module\Response; @@ -70,6 +71,8 @@ class Index extends BaseModeration $this->blocklist->set($blocklist); + Worker::add(Worker::PRIORITY_LOW, 'UpdateBlockedServers'); + $this->baseUrl->redirect('moderation/blocklist/server'); } diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php index 1fb9a30dd..a232834bc 100644 --- a/src/Worker/Cron.php +++ b/src/Worker/Cron.php @@ -85,8 +85,6 @@ class Cron // Hourly cron calls if ((DI::keyValue()->get('last_cron_hourly') ?? 0) + 3600 < time()) { - - // Update trending tags cache for the community page Tag::setLocalTrendingHashtags(24, 20); Tag::setGlobalTrendingHashtags(24, 20); @@ -145,6 +143,9 @@ class Cron // Resubscribe to relay servers Relay::reSubscribe(); + // Update "blocked" status of servers + Worker::add(Worker::PRIORITY_LOW, 'UpdateBlockedServers'); + DI::keyValue()->set('last_cron_daily', time()); } diff --git a/src/Worker/UpdateBlockedServers.php b/src/Worker/UpdateBlockedServers.php new file mode 100644 index 000000000..9a3a7df3f --- /dev/null +++ b/src/Worker/UpdateBlockedServers.php @@ -0,0 +1,53 @@ +. + * + */ + +namespace Friendica\Worker; + +use Friendica\Core\Logger; +use Friendica\Database\DBA; +use Friendica\Model\GServer; +use Friendica\Util\Network; + +class UpdateBlockedServers +{ + /** + * Updates the server blocked status + */ + public static function execute() + { + Logger::debug('Update blocked servers - start'); + $gservers = DBA::select('gserver', ['id', 'url', 'blocked']); + while ($gserver = DBA::fetch($gservers)) { + $blocked = Network::isUrlBlocked($gserver['url']); + if (!is_null($gserver['blocked']) && ($blocked == $gserver['blocked'])) { + continue; + } + + if ($blocked) { + GServer::setBlockedById($gserver['id']); + } else { + GServer::setUnblockedById($gserver['id']); + } + } + DBA::close($gservers); + Logger::debug('Update blocked servers - done'); + } +} diff --git a/src/Worker/UpdateGServer.php b/src/Worker/UpdateGServer.php index f6f33113f..4b4e40d76 100644 --- a/src/Worker/UpdateGServer.php +++ b/src/Worker/UpdateGServer.php @@ -55,6 +55,7 @@ class UpdateGServer // Silently dropping the worker task if the server domain is blocked if (Network::isUrlBlocked($filtered)) { + GServer::setBlockedByUrl($filtered); return; } @@ -84,6 +85,7 @@ class UpdateGServer { // Dropping the worker task if the server domain is blocked if (Network::isUrlBlocked($serverUrl)) { + GServer::setBlockedByUrl($serverUrl); return 0; } diff --git a/src/Worker/UpdateGServers.php b/src/Worker/UpdateGServers.php index 12f3ff10e..281e97771 100644 --- a/src/Worker/UpdateGServers.php +++ b/src/Worker/UpdateGServers.php @@ -27,7 +27,6 @@ use Friendica\Database\DBA; use Friendica\DI; use Friendica\Util\DateTimeFormat; use Friendica\Util\Strings; -use GuzzleHttp\Psr7\Uri; class UpdateGServers { @@ -49,7 +48,7 @@ class UpdateGServers } $total = DBA::count('gserver'); - $condition = ["`next_contact` < ? AND (`nurl` != ? OR `url` != ?)", DateTimeFormat::utcNow(), '', '']; + $condition = ["NOT `blocked` AND `next_contact` < ? AND (`nurl` != ? OR `url` != ?)", DateTimeFormat::utcNow(), '', '']; $outdated = DBA::count('gserver', $condition); Logger::info('Server status', ['total' => $total, 'outdated' => $outdated, 'updating' => $limit]); diff --git a/src/Worker/UpdateServerPeers.php b/src/Worker/UpdateServerPeers.php index 681c1a231..85a1b61bf 100644 --- a/src/Worker/UpdateServerPeers.php +++ b/src/Worker/UpdateServerPeers.php @@ -27,6 +27,7 @@ use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\GServer; use Friendica\Network\HTTPClient\Client\HttpClientAccept; +use Friendica\Util\Network; use Friendica\Util\Strings; class UpdateServerPeers @@ -56,6 +57,11 @@ class UpdateServerPeers $total = 0; $added = 0; foreach ($peers as $peer) { + if (Network::isUrlBlocked('http://' . $peer)) { + // Ignore blocked systems as soon as possible in the loop to avoid being slowed down by tar pits + continue; + } + ++$total; if (DBA::exists('gserver', ['nurl' => Strings::normaliseLink('http://' . $peer)])) { // We already know this server diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index d25d9cc3f..a6459f16b 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -55,7 +55,7 @@ use Friendica\Database\DBA; if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1510); + define('DB_UPDATE_VERSION', 1511); } return [ @@ -89,6 +89,7 @@ return [ "last_poco_query" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => ""], "last_contact" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => "Last successful connection request"], "last_failure" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => "Last failed connection request"], + "blocked" => ["type" => "boolean", "comment" => "Server is blocked"], "failed" => ["type" => "boolean", "comment" => "Connection failed"], "next_contact" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => "Next connection request"], ],