From 198e150ba8e882e92be4d66581d4dd088fb081f0 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 25 Mar 2019 21:51:32 +0000 Subject: [PATCH 1/4] Archive inboxes after 5 days of delivery failures --- config/dbstructure.config.php | 16 +++++- database.sql | 30 ++++++++---- src/Protocol/ActivityPub/Transmitter.php | 20 +++++++- src/Util/HTTPSignature.php | 62 +++++++++++++++++++++++- 4 files changed, 115 insertions(+), 13 deletions(-) diff --git a/config/dbstructure.config.php b/config/dbstructure.config.php index f03132add..b2f451b04 100644 --- a/config/dbstructure.config.php +++ b/config/dbstructure.config.php @@ -34,7 +34,7 @@ use Friendica\Database\DBA; if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1304); + define('DB_UPDATE_VERSION', 1305); } return [ @@ -529,6 +529,20 @@ return [ "hook_file_function" => ["UNIQUE", "hook", "file", "function"], ] ], + "inbox-status" => [ + "comment" => "Status of ActivityPub inboxes", + "fields" => [ + "url" => ["type" => "varbinary(255)", "not null" => "1", "primary" => "1", "comment" => "URL of the inbox"], + "created" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Creation date of this entry"], + "success" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Date of the last successful delivery"], + "failure" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Date of the last failed delivery"], + "previous" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Previous delivery date"], + "archive" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "Is the inbox archived?"] + ], + "indexes" => [ + "PRIMARY" => ["url"] + ] + ], "intro" => [ "comment" => "", "fields" => [ diff --git a/database.sql b/database.sql index 10a428fff..5b415b65d 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ --- Friendica 2019.03-dev (The Tazmans Flax-lily) --- DB_UPDATE_VERSION 1300 +-- Friendica 2019.03 (Dalmatian Bellflower) +-- DB_UPDATE_VERSION 1305 -- ------------------------------------------ @@ -470,6 +470,19 @@ CREATE TABLE IF NOT EXISTS `hook` ( UNIQUE INDEX `hook_file_function` (`hook`,`file`,`function`) ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='addon hook registry'; +-- +-- TABLE inbox-status +-- +CREATE TABLE IF NOT EXISTS `inbox-status` ( + `url` varbinary(255) NOT NULL COMMENT 'URL of the inbox', + `created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Creation date of this entry', + `success` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of the last successful delivery', + `failure` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of the last failed delivery', + `previous` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Previous delivery date', + `archive` boolean NOT NULL DEFAULT '0' COMMENT 'Is the inbox archived?', + PRIMARY KEY(`url`) +) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Status of ActivityPub inboxes'; + -- -- TABLE intro -- @@ -879,7 +892,7 @@ CREATE TABLE IF NOT EXISTS `photo` ( `deny_gid` mediumtext COMMENT 'Access Control - list of denied groups', `backend-class` tinytext COMMENT 'Storage backend class', `backend-ref` text COMMENT 'Storage backend data reference', - `updated` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'edited timestamp', + `updated` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '', PRIMARY KEY(`id`), INDEX `contactid` (`contact-id`), INDEX `uid_contactid` (`uid`,`contact-id`), @@ -1270,13 +1283,12 @@ CREATE TABLE IF NOT EXISTS `workerqueue` ( `retrial` tinyint NOT NULL DEFAULT 0 COMMENT 'Retrial counter', `done` boolean NOT NULL DEFAULT '0' COMMENT 'Marked 1 when the task was done - will be deleted later', PRIMARY KEY(`id`), - INDEX `pid` (`pid`), - INDEX `parameter` (`parameter`(64)), - INDEX `priority_created_next_try` (`priority`,`created`,`next_try`), - INDEX `done_priority_executed_next_try` (`done`,`priority`,`executed`,`next_try`), - INDEX `done_executed_next_try` (`done`,`executed`,`next_try`), + INDEX `done_parameter` (`done`,`parameter`(64)), + INDEX `done_executed` (`done`,`executed`), + INDEX `done_priority_created` (`done`,`priority`,`created`), INDEX `done_priority_next_try` (`done`,`priority`,`next_try`), - INDEX `done_next_try` (`done`,`next_try`) + INDEX `done_pid_next_try` (`done`,`pid`,`next_try`), + INDEX `done_pid_priority_created` (`done`,`pid`,`priority`,`created`) ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Background tasks queue entries'; -- diff --git a/src/Protocol/ActivityPub/Transmitter.php b/src/Protocol/ActivityPub/Transmitter.php index ce23e6db2..eb1da09b3 100644 --- a/src/Protocol/ActivityPub/Transmitter.php +++ b/src/Protocol/ActivityPub/Transmitter.php @@ -465,6 +465,18 @@ class Transmitter return $receivers; } + /** + * Check if an inbox is archived + * + * @param string $url Inbox url + * + * @return boolean "true" if inbox is archived + */ + private static function archivedInbox($url) + { + return DBA::exists('inbox-status', ['url' => $url, 'archive' => true]); + } + /** * Fetches a list of inboxes of followers of a given user * @@ -506,7 +518,9 @@ class Transmitter } else { $target = $profile['sharedinbox']; } - $inboxes[$target] = $target; + if (!self::archivedInbox($target)) { + $inboxes[$target] = $target; + } } } DBA::close($contacts); @@ -563,7 +577,9 @@ class Transmitter } else { $target = $profile['sharedinbox']; } - $inboxes[$target] = $target; + if (!self::archivedInbox($target)) { + $inboxes[$target] = $target; + } } } } diff --git a/src/Util/HTTPSignature.php b/src/Util/HTTPSignature.php index d5e1732c0..e002d5981 100644 --- a/src/Util/HTTPSignature.php +++ b/src/Util/HTTPSignature.php @@ -5,6 +5,7 @@ */ namespace Friendica\Util; +use Friendica\Database\DBA; use Friendica\Core\Config; use Friendica\Core\Logger; use Friendica\Model\User; @@ -314,7 +315,66 @@ class HTTPSignature Logger::log('Transmit to ' . $target . ' returned ' . $return_code, Logger::DEBUG); - return ($return_code >= 200) && ($return_code <= 299); + $success = ($return_code >= 200) && ($return_code <= 299); + + self::setInboxStatus($target, $success); + + return $success; + } + + /** + * @brief Set the delivery status for a given inbox + * + * @param string $url The URL of the inbox + * @param boolean $success Transmission status + */ + static private function setInboxStatus($url, $success) + { + $now = DateTimeFormat::utcNow(); + + $status = DBA::selectFirst('inbox-status', [], ['url' => $url]); + if (!DBA::isResult($status)) { + DBA::insert('inbox-status', ['url' => $url, 'created' => $now]); + $status = DBA::selectFirst('inbox-status', [], ['url' => $url]); + } + + if ($success) { + $fields = ['success' => $now]; + } else { + $fields = ['failure' => $now]; + } + + if ($status['failure'] > DBA::NULL_DATETIME) { + $new_previous_stamp = strtotime($status['failure']); + $old_previous_stamp = strtotime($status['previous']); + + // Only set "previous" with at least one day difference. + // We use this to assure to not accidentally archive too soon. + if (($new_previous_stamp - $old_previous_stamp) >= 86400) { + $fields['previous'] = $status['failure']; + } + } + + if (!$success) { + if ($status['success'] <= DBA::NULL_DATETIME) { + $stamp1 = strtotime($status['created']); + } else { + $stamp1 = strtotime($status['success']); + } + + $stamp2 = strtotime($now); + $previous_stamp = strtotime($status['previous']); + + // Archive the inbox when there had been failures for five days. + // Additionally ensure that at least one previous attempt has to be in between. + if ((($stamp2 - $stamp1) >= 86400 * 5) && ($previous_stamp > $stamp1)) { + $fields['archive'] = true; + } + } else { + $fields['archive'] = false; + } + + DBA::update('inbox-status', $fields, ['url' => $url]); } /** From 18f3ff7b8ef0ea52f9b74563c3b7885214dab220 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 25 Mar 2019 22:43:00 +0000 Subject: [PATCH 2/4] Unarchive inboxes on profile update --- src/Model/APContact.php | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/Model/APContact.php b/src/Model/APContact.php index bf36306b4..4ae6dbd1d 100644 --- a/src/Model/APContact.php +++ b/src/Model/APContact.php @@ -123,11 +123,14 @@ class APContact extends BaseObject $apcontact['following'] = JsonLD::fetchElement($compacted, 'as:following', '@id'); $apcontact['followers'] = JsonLD::fetchElement($compacted, 'as:followers', '@id'); $apcontact['inbox'] = JsonLD::fetchElement($compacted, 'ldp:inbox', '@id'); + self::unarchiveInbox($apcontact['inbox']); + $apcontact['outbox'] = JsonLD::fetchElement($compacted, 'as:outbox', '@id'); $apcontact['sharedinbox'] = ''; if (!empty($compacted['as:endpoints'])) { $apcontact['sharedinbox'] = JsonLD::fetchElement($compacted['as:endpoints'], 'as:sharedInbox', '@id'); + self::unarchiveInbox($apcontact['sharedinbox']); } $apcontact['nick'] = JsonLD::fetchElement($compacted, 'as:preferredUsername'); @@ -231,4 +234,27 @@ class APContact extends BaseObject return $apcontact; } + + /** + * Unarchive inboxes + * + * @param string $url inbox url + */ + private static function unarchiveInbox($url) + { + if (empty($url)) { + return; + } + + $now = DateTimeFormat::utcNow(); + + if (!DBA::exists('inbox-status', ['url' => $url])) { + $fields = ['archive' => false, 'success' => $now, + 'url' => $url, 'created' => $now]; + DBA::insert('inbox-status', $fields); + } else { + $fields = ['archive' => false, 'success' => $now]; + DBA::update('inbox-status', $fields, ['url' => $url]); + } + } } From fe0c516c3fa36e240439f5603c541e5796b939b5 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 26 Mar 2019 05:14:47 +0000 Subject: [PATCH 3/4] Added field for "shared" inbox --- config/dbstructure.config.php | 3 ++- src/Model/APContact.php | 12 ++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/config/dbstructure.config.php b/config/dbstructure.config.php index b2f451b04..398f671b2 100644 --- a/config/dbstructure.config.php +++ b/config/dbstructure.config.php @@ -537,7 +537,8 @@ return [ "success" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Date of the last successful delivery"], "failure" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Date of the last failed delivery"], "previous" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Previous delivery date"], - "archive" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "Is the inbox archived?"] + "archive" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "Is the inbox archived?"], + "shared" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "Is it a shared inbox?"] ], "indexes" => [ "PRIMARY" => ["url"] diff --git a/src/Model/APContact.php b/src/Model/APContact.php index 4ae6dbd1d..97178f697 100644 --- a/src/Model/APContact.php +++ b/src/Model/APContact.php @@ -123,14 +123,14 @@ class APContact extends BaseObject $apcontact['following'] = JsonLD::fetchElement($compacted, 'as:following', '@id'); $apcontact['followers'] = JsonLD::fetchElement($compacted, 'as:followers', '@id'); $apcontact['inbox'] = JsonLD::fetchElement($compacted, 'ldp:inbox', '@id'); - self::unarchiveInbox($apcontact['inbox']); + self::unarchiveInbox($apcontact['inbox'], false); $apcontact['outbox'] = JsonLD::fetchElement($compacted, 'as:outbox', '@id'); $apcontact['sharedinbox'] = ''; if (!empty($compacted['as:endpoints'])) { $apcontact['sharedinbox'] = JsonLD::fetchElement($compacted['as:endpoints'], 'as:sharedInbox', '@id'); - self::unarchiveInbox($apcontact['sharedinbox']); + self::unarchiveInbox($apcontact['sharedinbox'], true); } $apcontact['nick'] = JsonLD::fetchElement($compacted, 'as:preferredUsername'); @@ -240,7 +240,7 @@ class APContact extends BaseObject * * @param string $url inbox url */ - private static function unarchiveInbox($url) + private static function unarchiveInbox($url, $shared) { if (empty($url)) { return; @@ -248,12 +248,12 @@ class APContact extends BaseObject $now = DateTimeFormat::utcNow(); + $fields = ['archive' => false, 'success' => $now, 'shared' => $shared]; + if (!DBA::exists('inbox-status', ['url' => $url])) { - $fields = ['archive' => false, 'success' => $now, - 'url' => $url, 'created' => $now]; + $fields = array_merge($fields, ['url' => $url, 'created' => $now]); DBA::insert('inbox-status', $fields); } else { - $fields = ['archive' => false, 'success' => $now]; DBA::update('inbox-status', $fields, ['url' => $url]); } } From 87cdbfcee4e251478731ee6ba2521a8e43863fd1 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 26 Mar 2019 21:38:15 +0000 Subject: [PATCH 4/4] Updated database.sql --- database.sql | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/database.sql b/database.sql index 272d02de6..5ba700bd8 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ --- Friendica 2019.03 (Dalmatian Bellflower) --- DB_UPDATE_VERSION 1305 +-- Friendica 2019.06-dev (Dalmatian Bellflower) +-- DB_UPDATE_VERSION 1308 -- ------------------------------------------ @@ -480,6 +480,7 @@ CREATE TABLE IF NOT EXISTS `inbox-status` ( `failure` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of the last failed delivery', `previous` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Previous delivery date', `archive` boolean NOT NULL DEFAULT '0' COMMENT 'Is the inbox archived?', + `shared` boolean NOT NULL DEFAULT '0' COMMENT 'Is it a shared inbox?', PRIMARY KEY(`url`) ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Status of ActivityPub inboxes'; @@ -1112,7 +1113,7 @@ CREATE TABLE IF NOT EXISTS `term` ( `global` boolean NOT NULL DEFAULT '0' COMMENT '', `uid` mediumint unsigned NOT NULL DEFAULT 0 COMMENT 'User id', PRIMARY KEY(`tid`), - INDEX `term_type` (`term`(64), `type`), + INDEX `term_type` (`term`(64),`type`), INDEX `oid_otype_type_term` (`oid`,`otype`,`type`,`term`(32)), INDEX `uid_otype_type_term_global_created` (`uid`,`otype`,`type`,`term`(32),`global`,`created`), INDEX `uid_otype_type_url` (`uid`,`otype`,`type`,`url`(64)),