keith/friendica
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Improved message handling / new activity relay handling

Browse source
This commit is contained in:
Michael 2022-07-27 17:39:00 +00:00
parent 7834359957
commit 86105635ca
26 changed files with 280 additions and 428 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
database.sql
View file

@ -1,6 +1,6 @@
-- ------------------------------------------
-- Friendica 2022.09-dev (Giant Rhubarb)
-- DB_UPDATE_VERSION 1474
-- DB_UPDATE_VERSION 1475
-- ------------------------------------------
@ -338,6 +338,7 @@ CREATE TABLE IF NOT EXISTS `apcontact` (
`featured-tags` varchar(255) COMMENT 'Address for the collection of featured tags',
`manually-approve` boolean COMMENT '',
`discoverable` boolean COMMENT 'Mastodon extension: true if profile is published in their directory',
`suspended` boolean COMMENT 'Mastodon extension: true if profile is suspended',
`nick` varchar(255) NOT NULL DEFAULT '' COMMENT '',
`name` varchar(255) COMMENT '',
`about` text COMMENT '',
@ -503,23 +504,6 @@ CREATE TABLE IF NOT EXISTS `conv` (
FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='private messages';
--
-- TABLE conversation
--
CREATE TABLE IF NOT EXISTS `conversation` (
`item-uri` varbinary(255) NOT NULL COMMENT 'Original URI of the item - unrelated to the table with the same name',
`reply-to-uri` varbinary(255) NOT NULL DEFAULT '' COMMENT 'URI to which this item is a reply',
`conversation-uri` varbinary(255) NOT NULL DEFAULT '' COMMENT 'GNU Social conversation URI',
`conversation-href` varbinary(255) NOT NULL DEFAULT '' COMMENT 'GNU Social conversation link',
`protocol` tinyint unsigned NOT NULL DEFAULT 255 COMMENT 'The protocol of the item',
`direction` tinyint unsigned NOT NULL DEFAULT 0 COMMENT 'How the message arrived here: 1=push, 2=pull',
`source` mediumtext COMMENT 'Original source',
`received` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Receiving date',
PRIMARY KEY(`item-uri`),
INDEX `conversation-uri` (`conversation-uri`),
INDEX `received` (`received`)
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Raw data and structure information for messages';
--
-- TABLE workerqueue
--
@ -1117,6 +1101,17 @@ CREATE TABLE IF NOT EXISTS `post` (
FOREIGN KEY (`vid`) REFERENCES `verb` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Structure for all posts';
--
-- TABLE post-activity
--
CREATE TABLE IF NOT EXISTS `post-activity` (
`uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri',
`activity` mediumtext COMMENT 'Original activity',
`received` datetime COMMENT '',
PRIMARY KEY(`uri-id`),
FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Original remote activity';
--
-- TABLE post-category
--

2
doc/database.md
View file

@ -19,7 +19,6 @@ Database Tables
| [contact](help/database/db_contact) | contact table |
| [contact-relation](help/database/db_contact-relation) | Contact relations |
| [conv](help/database/db_conv) | private messages |
| [conversation](help/database/db_conversation) | Raw data and structure information for messages |
| [delayed-post](help/database/db_delayed-post) | Posts that are about to be distributed at a later time |
| [diaspora-interaction](help/database/db_diaspora-interaction) | Signed Diaspora Interaction |
| [endpoint](help/database/db_endpoint) | ActivityPub endpoints - used in the ActivityPub implementation |
@ -50,6 +49,7 @@ Database Tables
| [permissionset](help/database/db_permissionset) | |
| [photo](help/database/db_photo) | photo storage |
| [post](help/database/db_post) | Structure for all posts |
| [post-activity](help/database/db_post-activity) | Original remote activity |
| [post-category](help/database/db_post-category) | post relation to categories |
| [post-collection](help/database/db_post-collection) | Collection of posts |
| [post-content](help/database/db_post-content) | Content for all posts |

1
doc/database/db_apcontact.md
View file

@ -21,6 +21,7 @@ Fields
| featured-tags | Address for the collection of featured tags | varchar(255) | YES | | NULL | |
| manually-approve | | boolean | YES | | NULL | |
| discoverable | Mastodon extension: true if profile is published in their directory | boolean | YES | | NULL | |
| suspended | Mastodon extension: true if profile is suspended | boolean | YES | | NULL | |
| nick | | varchar(255) | NO | | | |
| name | | varchar(255) | YES | | NULL | |
| about | | text | YES | | NULL | |

29
doc/database/db_post-activity.md Normal file
View file

@ -0,0 +1,29 @@
Table post-activity
===========
Original remote activity
Fields
------
| Field | Description | Type | Null | Key | Default | Extra |
| -------- | --------------------------------------------------------- | ------------ | ---- | --- | ------- | ----- |
| uri-id | Id of the item-uri table entry that contains the item uri | int unsigned | NO | PRI | NULL | |
| activity | Original activity | mediumtext | YES | | NULL | |
| received | | datetime | YES | | NULL | |
Indexes
------------
| Name | Fields |
| ------- | ------ |
| PRIMARY | uri-id |
Foreign Keys
------------
| Field | Target Table | Target Field |
|-------|--------------|--------------|
| uri-id | [item-uri](help/database/db_item-uri) | id |
Return to [database documentation](help/database)

10
mod/item.php
View file

@ -595,16 +595,6 @@ function item_post(App $a) {
$datarray['protocol'] = Conversation::PARCEL_DIRECT;
$datarray['direction'] = Conversation::PUSH;
$conversation = DBA::selectFirst('conversation', ['conversation-uri', 'conversation-href'], ['item-uri' => $datarray['thr-parent']]);
if (DBA::isResult($conversation)) {
if ($conversation['conversation-uri'] != '') {
$datarray['conversation-uri'] = $conversation['conversation-uri'];
}
if ($conversation['conversation-href'] != '') {
$datarray['conversation-href'] = $conversation['conversation-href'];
}
}
if ($orig_post) {
$datarray['edit'] = true;
} else {

2
src/Database/DBStructure.php
View file

@ -74,7 +74,7 @@ class DBStructure
$old_tables = ['fserver', 'gcign', 'gcontact', 'gcontact-relation', 'gfollower' ,'glink', 'item-delivery-data',
'item-activity', 'item-content', 'item_id', 'participation', 'poll', 'poll_result', 'queue', 'retriever_rule',
'deliverq', 'dsprphotoq', 'ffinder', 'sign', 'spam', 'term', 'user-item', 'thread', 'item', 'challenge',
'auth_codes', 'tokens', 'clients', 'profile_check', 'host'];
'auth_codes', 'tokens', 'clients', 'profile_check', 'host', 'conversation'];
$tables = DBA::selectToArray('INFORMATION_SCHEMA.TABLES', ['TABLE_NAME'],
['TABLE_SCHEMA' => DBA::databaseName(), 'TABLE_TYPE' => 'BASE TABLE']);

7
src/Model/APContact.php
View file

@ -38,6 +38,7 @@ use Friendica\Util\DateTimeFormat;
use Friendica\Util\HTTPSignature;
use Friendica\Util\JsonLD;
use Friendica\Util\Network;
use GuzzleHttp\Psr7\Uri;
class APContact
{
@ -310,6 +311,8 @@ class APContact
$apcontact['manually-approve'] = (int)JsonLD::fetchElement($compacted, 'as:manuallyApprovesFollowers');
$apcontact['suspended'] = (int)JsonLD::fetchElement($compacted, 'toot:suspended');
if (!empty($compacted['as:generator'])) {
$apcontact['baseurl'] = JsonLD::fetchElement($compacted['as:generator'], 'as:url', '@id');
$apcontact['generator'] = JsonLD::fetchElement($compacted['as:generator'], 'as:name', '@value');
@ -380,11 +383,11 @@ class APContact
if (strlen($apcontact['photo']) > 255) {
$parts = parse_url($apcontact['photo']);
unset($parts['fragment']);
$apcontact['photo'] = Network::unparseURL($parts);
$apcontact['photo'] = Uri::fromParts($parts);
if (strlen($apcontact['photo']) > 255) {
unset($parts['query']);
$apcontact['photo'] = Network::unparseURL($parts);
$apcontact['photo'] = Uri::fromParts($parts);
}
if (strlen($apcontact['photo']) > 255) {

62
src/Model/Conversation.php
View file

@ -21,11 +21,6 @@
namespace Friendica\Model;
use Friendica\Core\Protocol;
use Friendica\Database\Database;
use Friendica\Database\DBA;
use Friendica\Util\DateTimeFormat;
class Conversation
{
/*
@ -62,61 +57,4 @@ class Conversation
*/
const RELAY = 3;
public static function getByItemUri(string $item_uri)
{
return DBA::selectFirst('conversation', [], ['item-uri' => $item_uri]);
}
/**
* Store the conversation data
*
* @param array $arr Item array with conversation data
* @return array Item array with removed conversation data
* @throws \Exception
*/
public static function insert(array $arr): array
{
if (in_array(($arr['network'] ?? '') ?: Protocol::PHANTOM,
[Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS, Protocol::TWITTER]) && !empty($arr['uri'])) {
$conversation = ['item-uri' => $arr['uri'], 'received' => DateTimeFormat::utcNow()];
if (isset($arr['parent-uri']) && ($arr['parent-uri'] != $arr['uri'])) {
$conversation['reply-to-uri'] = $arr['parent-uri'];
}
if (isset($arr['thr-parent']) && ($arr['thr-parent'] != $arr['uri'])) {
$conversation['reply-to-uri'] = $arr['thr-parent'];
}
if (isset($arr['conversation-uri'])) {
$conversation['conversation-uri'] = $arr['conversation-uri'];
}
if (isset($arr['conversation-href'])) {
$conversation['conversation-href'] = $arr['conversation-href'];
}
if (isset($arr['protocol'])) {
$conversation['protocol'] = $arr['protocol'];
}
if (isset($arr['direction'])) {
$conversation['direction'] = $arr['direction'];
}
if (isset($arr['source'])) {
$conversation['source'] = $arr['source'];
}
if (!DBA::exists('conversation', ['item-uri' => $conversation['item-uri']])) {
DBA::insert('conversation', $conversation, Database::INSERT_IGNORE);
}
}
unset($arr['conversation-uri']);
unset($arr['conversation-href']);
unset($arr['source']);
return $arr;
}
}

7
src/Model/GServer.php
View file

@ -563,8 +563,13 @@ class GServer
$serverdata['registered-users'] = $serverdata['registered-users'] ?? 0;
// Numbers above a reasonable value (10 millions) are ignored
if ($serverdata['registered-users'] > 10000000) {
$serverdata['registered-users'] = 0;
}
// On an active server there has to be at least a single user
if (!in_array($serverdata['network'], [Protocol::PHANTOM, Protocol::FEED]) && ($serverdata['registered-users'] == 0)) {
if (!in_array($serverdata['network'], [Protocol::PHANTOM, Protocol::FEED]) && ($serverdata['registered-users'] <= 0)) {
$serverdata['registered-users'] = 1;
} elseif (in_array($serverdata['network'], [Protocol::PHANTOM, Protocol::FEED])) {
$serverdata['registered-users'] = 0;

8
src/Model/Item.php
View file

@ -801,7 +801,10 @@ class Item
$item['parent-uri-id'] = ItemURI::getIdByURI($item['parent-uri']);
// Store conversation data
$item = Conversation::insert($item);
$source = $item['source'] ?? '';
unset($item['conversation-uri']);
unset($item['conversation-href']);
unset($item['source']);
/*
* Do we already have this item?
@ -1259,6 +1262,9 @@ class Item
}
if ($transmit) {
if (!empty($source)) {
Post\Activity::insert($item['uri-id'], $source);
}
Worker::add(['priority' => $priority, 'dont_fork' => true], 'Notifier', $notify_type, (int)$posted_item['uri-id'], (int)$posted_item['uid']);
}

74
src/Model/Post/Activity.php Normal file
View file

@ -0,0 +1,74 @@
<?php
/**
* @copyright Copyright (C) 2010-2022, the Friendica project
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
namespace Friendica\Model\Post;
use Friendica\Database\Database;
use Friendica\Database\DBA;
use Friendica\Util\DateTimeFormat;
class Activity
{
/**
* Insert a new post-activity entry
*
* @param integer $uri_id
* @param array $fields
*
* @return bool success
*/
public static function insert(int $uri_id, string $source): bool
{
// Additionally assign the key fields
$fields = [
'uri-id' => $uri_id,
'activity' => $source,
'received' => DateTimeFormat::utcNow()
];
return DBA::insert('post-activity', $fields, Database::INSERT_IGNORE);
}
/**
* Retrieves activity of the given uri-id
*
* @param int $uriId
*
* @return array
*/
public static function getByURIId(int $uriId): array
{
$activity = DBA::selectFirst('post-activity', [], ['uri-id' => $uriId]);
return json_decode($activity['activity'] ?? '', true) ?? [];
}
/**
* Checks if the given uridid has a stored activity
*
* @param integer $uriId
*
* @return boolean
*/
public static function exists(int $uriId): bool
{
return DBA::exists('post-activity', ['uri-id' => $uriId]);
}
}

2
src/Model/Tag.php
View file

@ -545,7 +545,7 @@ class Tag
break;
default:
Logger:warning('Unknown tag type found', $tag);
Logger::warning('Unknown tag type found', $tag);
}
}
DBA::close($taglist);

6
src/Module/Admin/Item/Source.php
View file

@ -35,7 +35,6 @@ class Source extends BaseAdmin
$guid = basename($_REQUEST['guid'] ?? $this->parameters['guid'] ?? '');
$source = '';
$item_uri = '';
$item_id = '';
$terms = [];
@ -43,11 +42,8 @@ class Source extends BaseAdmin
$item = Model\Post::selectFirst(['id', 'uri-id', 'guid', 'uri'], ['guid' => $guid]);
if ($item) {
$conversation = Model\Conversation::getByItemUri($item['uri']);
$item_id = $item['id'];
$item_uri = $item['uri'];
$source = $conversation['source'];
$terms = Model\Tag::getByURIId($item['uri-id'], [Model\Tag::HASHTAG, Model\Tag::MENTION, Model\Tag::IMPLICIT_MENTION]);
}
}
@ -56,7 +52,6 @@ class Source extends BaseAdmin
$o = Renderer::replaceMacros($tpl, [
'$title' => DI::l10n()->t('Item Source'),
'$guid' => ['guid', DI::l10n()->t('Item Guid'), $guid, ''],
'$source' => $source,
'$item_uri' => $item_uri,
'$item_id' => $item_id,
'$terms' => $terms,
@ -70,7 +65,6 @@ class Source extends BaseAdmin
'$urllbl' => DI::l10n()->t('URL'),
'$mentionlbl' => DI::l10n()->t('Mention'),
'$implicitlbl' => DI::l10n()->t('Implicit Mention'),
'$sourcelbl' => DI::l10n()->t('Source'),
]);
return $o;

2
src/Module/DFRN/Notify.php
View file

@ -59,7 +59,7 @@ class Notify extends BaseModule
}
}
private static function dispatchPublic(array $postdata)
private static function dispatchPublic(string $postdata)
{
$msg = Diaspora::decodeRaw($postdata, '', true);
if (!is_array($msg)) {

5
src/Navigation/Notifications/Factory/Notification.php
View file

@ -163,9 +163,10 @@ class Notification extends BaseFactory implements ICanCreateFromTableRow
}
if (($Notification->verb == Activity::POST) || ($Notification->type === Post\UserNotification::TYPE_SHARED)) {
$item = Post::selectFirst([], ['uri-id' => $item['thr-parent-id'], 'uid' => [0, $Notification->uid]], ['order' => ['uid' => true]]);
$thrparentid = $item['thr-parent-id'];
$item = Post::selectFirst([], ['uri-id' => $thrparentid, 'uid' => [0, $Notification->uid]], ['order' => ['uid' => true]]);
if (empty($item)) {
$this->logger->info('Thread parent post not found', ['uri-id' => $item['thr-parent-id']]);
$this->logger->info('Thread parent post not found', ['uri-id' => $thrparentid]);
return $message;
}
}

5
src/Protocol/ActivityPub/Delivery.php
View file

@ -110,7 +110,10 @@ class Delivery
} elseif ($cmd == WorkerDelivery::PROFILEUPDATE) {
$success = ActivityPub\Transmitter::sendProfileUpdate($uid, $inbox);
} else {
$data = ActivityPub\Transmitter::createCachedActivityFromItem($item_id);
$data = Post\Activity::getByURIId($uri_id);
if (empty($data)) {
$data = ActivityPub\Transmitter::createCachedActivityFromItem($item_id);
}
if (!empty($data)) {
$timestamp = microtime(true);
$response = HTTPSignature::post($data, $inbox, $uid);

16
src/Protocol/ActivityPub/Processor.php
View file

@ -361,8 +361,6 @@ class Processor
if (!empty($activity['raw'])) {
$item['source'] = $activity['raw'];
$item['protocol'] = Conversation::PARCEL_ACTIVITYPUB;
$item['conversation-href'] = $activity['context'] ?? '';
$item['conversation-uri'] = $activity['conversation'] ?? '';
if (isset($activity['push'])) {
$item['direction'] = $activity['push'] ? Conversation::PUSH : Conversation::PULL;
@ -475,7 +473,19 @@ class Processor
}
// @todo To ensure that the remote system is working correctly, we can check if the "Content-Type" contains JSON
return in_array($curlResult->getReturnCode(), [404]);
if (in_array($curlResult->getReturnCode(), [404])) {
return true;
}
$object = json_decode($curlResult->getBody(), true);
if (!empty($object)) {
$activity = JsonLD::compact($object);
if (JsonLD::fetchElement($activity, '@type') == 'as:Tombstone') {
return true;
}
}
return false;
}
/**
* Delete items

56
src/Protocol/ActivityPub/Queue.php
View file

@ -25,7 +25,9 @@ use Friendica\Core\Logger;
use Friendica\Database\Database;
use Friendica\Database\DBA;
use Friendica\DI;
use Friendica\Protocol\ActivityPub;
use Friendica\Util\DateTimeFormat;
use Friendica\Util\JsonLD;
/**
* This class handles the processing of incoming posts
@ -214,6 +216,7 @@ class Queue
Logger::debug('Process leftover entry', $entry);
self::process($entry['id']);
}
DBA::close($entries);
}
/**
@ -247,5 +250,58 @@ class Queue
while ($entry = DBA::fetch($entries)) {
self::process($entry['id']);
}
DBA::close($entries);
}
/**
* Prepare the queue entry.
* This is a test function that is used solely for development.
*
* @param integer $id
* @return array
*/
public static function reprepareActivityById(int $id): array
{
$entry = DBA::selectFirst('inbox-entry', [], ['id' => $id]);
if (empty($entry)) {
return [];
}
$receiver = DBA::selectFirst('inbox-entry-receiver', ['uid'], ['queue-id' => $id]);
if (!empty($receiver)) {
$uid = $receiver['uid'];
} else {
$uid = 0;
}
$trust_source = $entry['trust'];
$data = json_decode($entry['activity'], true);
$activity = json_decode($data['raw'], true);
$ldactivity = JsonLD::compact($activity);
return [
'data' => Receiver::prepareObjectData($ldactivity, $uid, $entry['push'], $trust_source),
'trust' => $trust_source
];
}
/**
* Set the trust for all untrusted entries.
* This is a test function that is used solely for development.
*
* @return void
*/
public static function reprepareAll()
{
$entries = DBA::select('inbox-entry', ['id'], ["NOT `trust` AND `wid` IS NULL"], ['order' => ['id' => true]]);
while ($entry = DBA::fetch($entries)) {
$data = self::reprepareActivityById($entry['id'], false);
if ($data['trust']) {
DBA::update('inbox-entry', ['trust' => true], ['id' => $entry['id']]);
}
}
DBA::close($entries);
}
}

49
src/Protocol/ActivityPub/Receiver.php
View file

@ -272,26 +272,39 @@ class Receiver
*/
public static function prepareObjectData(array $activity, int $uid, bool $push, bool &$trust_source): array
{
$id = JsonLD::fetchElement($activity, '@id');
$id = JsonLD::fetchElement($activity, '@id');
$type = JsonLD::fetchElement($activity, '@type');
$object_id = JsonLD::fetchElement($activity, 'as:object', '@id');
if (!empty($object_id) && in_array($type, ['as:Create', 'as:Update'])) {
$fetch_id = $object_id;
} else {
$fetch_id = $id;
}
if (!empty($activity['as:object'])) {
$object_type = JsonLD::fetchElement($activity['as:object'], '@type');
}
if (!empty($id) && !$trust_source) {
$fetch_uid = $uid ?: self::getBestUserForActivity($activity);
$fetched_activity = ActivityPub::fetchContent($id, $fetch_uid);
$fetched_activity = ActivityPub::fetchContent($fetch_id, $fetch_uid);
if (!empty($fetched_activity)) {
$object = JsonLD::compact($fetched_activity);
$fetched_id = JsonLD::fetchElement($object, '@id');
if ($fetched_id == $id) {
$fetched_id = JsonLD::fetchElement($object, '@id');
$fetched_type = JsonLD::fetchElement($object, '@type');
if (($fetched_id == $id) && !empty($fetched_type) && ($fetched_type == $type)) {
Logger::info('Activity had been fetched successfully', ['id' => $id]);
$trust_source = true;
if ($id != $object_id) {
$activity = $object;
} else {
Logger::info('Fetched data is the object instead of the activity', ['id' => $id]);
unset($object['@context']);
$activity['as:object'] = $object;
}
$activity = $object;
} elseif (($fetched_id == $object_id) && !empty($fetched_type) && ($fetched_type == $object_type)) {
Logger::info('Fetched data is the object instead of the activity', ['id' => $id]);
$trust_source = true;
unset($object['@context']);
$activity['as:object'] = $object;
} else {
Logger::info('Activity id is not equal', ['id' => $id, 'fetched' => $fetched_id]);
}
@ -371,9 +384,9 @@ class Receiver
$object_data['object_object'] = JsonLD::fetchElement($activity['as:object'], 'as:object');
$object_data['object_type'] = JsonLD::fetchElement($activity['as:object'], '@type');
$object_data['push'] = $push;
if ($type == 'as:Delete') {
if (!$trust_source && ($type == 'as:Delete')) {
$apcontact = APContact::getByURL($object_data['object_id'], true);
$trust_source = ($apcontact['type'] == 'Tombstone');
$trust_source = empty($apcontact) || ($apcontact['type'] == 'Tombstone') || $apcontact['suspended'];
}
} elseif (in_array($type, ['as:Create', 'as:Update', 'as:Announce', 'as:Invite']) || strpos($type, '#emojiReaction')) {
// Fetch the content only on activities where this matters
@ -430,8 +443,11 @@ class Receiver
$object_data['object_object_type'] = self::fetchObjectType([], $object_data['object_object'], $fetch_uid);
}
if (($type == 'as:Delete') && in_array($object_data['object_type'], array_merge(['as:Tombstone'], self::CONTENT_TYPES))) {
if (!$trust_source && ($type == 'as:Delete') && in_array($object_data['object_type'], array_merge(['as:Tombstone', ''], self::CONTENT_TYPES))) {
$trust_source = Processor::isActivityGone($object_data['object_id']);
if (!$trust_source) {
$trust_source = !empty(APContact::getByURL($object_data['object_id'], false));
}
}
}
@ -668,6 +684,9 @@ class Receiver
if (!empty($object_data['raw'])) {
$announce_object_data['raw'] = $object_data['raw'];
}
if (!empty($object_data['raw-object'])) {
$announce_object_data['raw-object'] = $object_data['raw-object'];
}
ActivityPub\Processor::createActivity($announce_object_data, Activity::ANNOUNCE);
}
} else {
@ -1305,7 +1324,7 @@ class Receiver
$object_data = self::processObject($object);
if (!empty($data)) {
$object_data['raw'] = json_encode($data);
$object_data['raw-object'] = json_encode($data);
}
return $object_data;
}

6
src/Protocol/ActivityPub/Transmitter.php
View file

@ -1232,10 +1232,8 @@ class Transmitter
}
if (!$item['deleted']) {
$condition = ['item-uri' => $item['uri'], 'protocol' => Conversation::PARCEL_ACTIVITYPUB];
$conversation = DBA::selectFirst('conversation', ['source'], $condition);
if (!$item['origin'] && DBA::isResult($conversation)) {
$data = json_decode($conversation['source'], true);
$data = Post\Activity::getByURIId($item['uri-id']);
if (!$item['origin'] && !empty($data)) {
if (!empty($data['type'])) {
if (in_array($data['type'], ['Create', 'Update'])) {
if ($object_mode) {

27
src/Protocol/DFRN.php
View file

@ -810,27 +810,12 @@ class DFRN
}
// Add conversation data. This is used for OStatus
$conversation_href = DI::baseUrl()."/display/".$item["parent-guid"];
$conversation_uri = $conversation_href;
if (isset($parent_item)) {
$conversation = DBA::selectFirst('conversation', ['conversation-uri', 'conversation-href'], ['item-uri' => $item['thr-parent']]);
if (DBA::isResult($conversation)) {
if ($conversation['conversation-uri'] != '') {
$conversation_uri = $conversation['conversation-uri'];
}
if ($conversation['conversation-href'] != '') {
$conversation_href = $conversation['conversation-href'];
}
}
}
$attributes = [
'href' => $conversation_href,
'ref' => $conversation_uri,
'href' => $item['conversation'],
'ref' => $item['conversation'],
];
XML::addElement($doc, $entry, 'ostatus:conversation', $conversation_uri, $attributes);
XML::addElement($doc, $entry, 'ostatus:conversation', $item['conversation'], $attributes);
XML::addElement($doc, $entry, 'id', $item['uri']);
XML::addElement($doc, $entry, 'title', $item['title']);
@ -1994,16 +1979,16 @@ class DFRN
self::parseLinks($links, $item);
}
$item['conversation-uri'] = XML::getFirstNodeValue($xpath, 'ostatus:conversation/text()', $entry);
$item['conversation'] = XML::getFirstNodeValue($xpath, 'ostatus:conversation/text()', $entry);
$conv = $xpath->query('ostatus:conversation', $entry);
if (is_object($conv->item(0))) {
foreach ($conv->item(0)->attributes as $attributes) {
if ($attributes->name == 'ref') {
$item['conversation-uri'] = $attributes->textContent;
$item['conversation'] = $attributes->textContent;
}
if ($attributes->name == 'href') {
$item['conversation-href'] = $attributes->textContent;
$item['conversation'] = $attributes->textContent;
}
}
}

253
src/Protocol/OStatus.php
View file

@ -616,16 +616,16 @@ class OStatus
$item['created'] = XML::getFirstNodeValue($xpath, 'atom:published/text()', $entry);
$item['edited'] = XML::getFirstNodeValue($xpath, 'atom:updated/text()', $entry);
$item['conversation-uri'] = XML::getFirstNodeValue($xpath, 'ostatus:conversation/text()', $entry);
$item['conversation'] = XML::getFirstNodeValue($xpath, 'ostatus:conversation/text()', $entry);
$conv = $xpath->query('ostatus:conversation', $entry);
if (is_object($conv->item(0))) {
foreach ($conv->item(0)->attributes as $attributes) {
if ($attributes->name == 'ref') {
$item['conversation-uri'] = $attributes->textContent;
$item['conversation'] = $attributes->textContent;
}
if ($attributes->name == 'href') {
$item['conversation-href'] = $attributes->textContent;
$item['conversation'] = $attributes->textContent;
}
}
}
@ -704,14 +704,6 @@ class OStatus
}
}
if (($self != '') && empty($item['protocol'])) {
self::fetchSelf($self, $item);
}
if (!empty($item['conversation-href'])) {
self::fetchConversation($item['conversation-href'], $item['conversation-uri']);
}
if (isset($item['thr-parent'])) {
if (!Post::exists(['uid' => $importer['uid'], 'uri' => $item['thr-parent']])) {
if ($related != '') {
@ -725,194 +717,9 @@ class OStatus
$item['gravity'] = GRAVITY_PARENT;
}
if (($item['author-link'] != '') && !empty($item['protocol'])) {
$item = Conversation::insert($item);
}
self::$itemlist[] = $item;
}
/**
* Fetch the conversation for posts
*
* @param string $conversation The link to the conversation
* @param string $conversation_uri The conversation in "uri" format
* @return void
* @throws \Friendica\Network\HTTPException\InternalServerErrorException
*/
private static function fetchConversation(string $conversation, string $conversation_uri)
{
// Ensure that we only store a conversation once in a process
if (isset(self::$conv_list[$conversation])) {
return;
}
self::$conv_list[$conversation] = true;
$curlResult = DI::httpClient()->get($conversation, HttpClientAccept::ATOM_XML);
if (!$curlResult->isSuccess() || empty($curlResult->getBody())) {
return;
}
$xml = '';
if ($curlResult->inHeader('Content-Type') &&
in_array('application/atom+xml', $curlResult->getHeader('Content-Type'))) {
$xml = $curlResult->getBody();
}
if ($xml == '') {
$doc = new DOMDocument();
if (!@$doc->loadHTML($curlResult->getBody())) {
return;
}
$xpath = new DOMXPath($doc);
$links = $xpath->query('//link');
if ($links) {
$file = '';
foreach ($links as $link) {
$attribute = self::readAttributes($link);
if (($attribute['rel'] == 'alternate') && ($attribute['type'] == 'application/atom+xml')) {
$file = $attribute['href'];
}
}
if ($file != '') {
$conversation_atom = DI::httpClient()->get($attribute['href'], HttpClientAccept::ATOM_XML);
if ($conversation_atom->isSuccess()) {
$xml = $conversation_atom->getBody();
}
}
}
}
if ($xml == '') {
return;
}
self::storeConversation($xml, $conversation, $conversation_uri);
}
/**
* Store a feed in several conversation entries
*
* @param string $xml The feed
* @param string $conversation conversation
* @param string $conversation_uri conversation uri
* @return void
* @throws \Exception
*/
private static function storeConversation(string $xml, string $conversation = '', string $conversation_uri = '')
{
$doc = new DOMDocument();
@$doc->loadXML($xml);
$xpath = new DOMXPath($doc);
$xpath->registerNamespace('atom', ActivityNamespace::ATOM1);
$xpath->registerNamespace('thr', ActivityNamespace::THREAD);
$xpath->registerNamespace('ostatus', ActivityNamespace::OSTATUS);
$entries = $xpath->query('/atom:feed/atom:entry');
// Now store the entries
foreach ($entries as $entry) {
$doc2 = new DOMDocument();
$doc2->preserveWhiteSpace = false;
$doc2->formatOutput = true;
$conv_data = [];
$conv_data['protocol'] = Conversation::PARCEL_SPLIT_CONVERSATION;
$conv_data['direction'] = Conversation::PULL;
$conv_data['network'] = Protocol::OSTATUS;
$conv_data['uri'] = XML::getFirstNodeValue($xpath, 'atom:id/text()', $entry);
$inreplyto = $xpath->query('thr:in-reply-to', $entry);
if (is_object($inreplyto->item(0))) {
foreach ($inreplyto->item(0)->attributes as $attributes) {
if ($attributes->name == 'ref') {
$conv_data['reply-to-uri'] = $attributes->textContent;
}
}
}
$conv_data['conversation-uri'] = XML::getFirstNodeValue($xpath, 'ostatus:conversation/text()', $entry);
$conv = $xpath->query('ostatus:conversation', $entry);
if (is_object($conv->item(0))) {
foreach ($conv->item(0)->attributes as $attributes) {
if ($attributes->name == 'ref') {
$conv_data['conversation-uri'] = $attributes->textContent;
}
if ($attributes->name == 'href') {
$conv_data['conversation-href'] = $attributes->textContent;
}
}
}
if ($conversation != '') {
$conv_data['conversation-uri'] = $conversation;
}
if ($conversation_uri != '') {
$conv_data['conversation-uri'] = $conversation_uri;
}
$entry = $doc2->importNode($entry, true);
$doc2->appendChild($entry);
$conv_data['source'] = $doc2->saveXML();
Logger::info('Store conversation data for uri '.$conv_data['uri']);
Conversation::insert($conv_data);
}
}
/**
* Fetch the own post so that it can be stored later
*
* We want to store the original data for later processing.
* This function is meant for cases where we process a feed with multiple entries.
* In that case we need to fetch the single posts here.
*
* @param string $self The link to the self item
* @param array $item The item array
* @return void
* @throws \Friendica\Network\HTTPException\InternalServerErrorException
*/
private static function fetchSelf(string $self, array &$item)
{
$condition = ['item-uri' => $self, 'protocol' => [Conversation::PARCEL_DFRN,
Conversation::PARCEL_DIASPORA_DFRN, Conversation::PARCEL_LOCAL_DFRN,
Conversation::PARCEL_DIRECT, Conversation::PARCEL_SALMON]];
if (DBA::exists('conversation', $condition)) {
Logger::info('Conversation '.$item['uri'].' is already stored.');
return;
}
$curlResult = DI::httpClient()->get($self, HttpClientAccept::ATOM_XML);
if (!$curlResult->isSuccess()) {
return;
}
// We reformat the XML to make it better readable
$doc = new DOMDocument();
$doc->loadXML($curlResult->getBody());
$doc->preserveWhiteSpace = false;
$doc->formatOutput = true;
$xml = $doc->saveXML();
$item['protocol'] = Conversation::PARCEL_SALMON;
$item['source'] = $xml;
$item['direction'] = Conversation::PULL;
Logger::info('Conversation '.$item['uri'].' is now fetched.');
}
/**
* Fetch related posts and processes them
*
@ -925,30 +732,6 @@ class OStatus
*/
private static function fetchRelated(string $related, string $related_uri, array $importer)
{
$condition = [
'item-uri' => $related_uri,
'protocol' => [
Conversation::PARCEL_DFRN,
Conversation::PARCEL_DIASPORA_DFRN,
Conversation::PARCEL_LOCAL_DFRN,
Conversation::PARCEL_DIRECT,
Conversation::PARCEL_SALMON,
],
];
$conversation = DBA::selectFirst('conversation', ['source', 'protocol'], $condition);
if (DBA::isResult($conversation)) {
$stored = true;
$xml = $conversation['source'];
if (self::process($xml, $importer, $contact, $hub, $stored, false, Conversation::PULL)) {
Logger::info('Got valid cached XML for URI '.$related_uri);
return;
}
if ($conversation['protocol'] == Conversation::PARCEL_SALMON) {
Logger::info('Delete invalid cached XML for URI '.$related_uri);
DBA::delete('conversation', ['item-uri' => $related_uri]);
}
}
$stored = false;
$curlResult = DI::httpClient()->get($related, HttpClientAccept::ATOM_XML);
@ -1013,17 +796,6 @@ class OStatus
}
}
// Finally we take the data that we fetched from "ostatus:conversation"
if ($xml == '') {
$condition = ['item-uri' => $related_uri, 'protocol' => Conversation::PARCEL_SPLIT_CONVERSATION];
$conversation = DBA::selectFirst('conversation', ['source'], $condition);
if (DBA::isResult($conversation)) {
$stored = true;
Logger::info('Got cached XML from conversation for URI ' . $related_uri);
$xml = $conversation['source'];
}
}
if ($xml != '') {
self::process($xml, $importer, $contact, $hub, $stored, false, Conversation::PULL);
} else {
@ -1130,10 +902,7 @@ class OStatus
case 'ostatus:conversation':
$link_data['conversation'] = $attribute['href'];
$item['conversation-href'] = $link_data['conversation'];
if (!isset($item['conversation-uri'])) {
$item['conversation-uri'] = $item['conversation-href'];
}
$item['conversation'] = $link_data['conversation'];
break;
case 'enclosure':
@ -1904,19 +1673,7 @@ class OStatus
}
if (intval($item['parent']) > 0) {
$conversation_href = $conversation_uri = str_replace('/objects/', '/context/', $item['thr-parent']);
if (isset($parent_item)) {
$conversation = DBA::selectFirst('conversation', ['conversation-uri', 'conversation-href'], ['item-uri' => $parent_item]);
if (DBA::isResult($conversation)) {
if ($conversation['conversation-uri'] != '') {
$conversation_uri = $conversation['conversation-uri'];
}
if ($conversation['conversation-href'] != '') {
$conversation_href = $conversation['conversation-href'];
}
}
}
$conversation_href = $conversation_uri = $item['conversation'];
XML::addElement($doc, $entry, 'link', '', ['rel' => 'ostatus:conversation', 'href' => $conversation_href]);

2
src/Worker/Cron.php
View file

@ -115,7 +115,7 @@ class Cron
Worker::add(PRIORITY_LOW, 'ExpirePosts');
Worker::add(PRIORITY_LOW, 'ExpireConversations');
Worker::add(PRIORITY_LOW, 'ExpireActivities');
Worker::add(PRIORITY_LOW, 'RemoveUnusedTags');

12
src/Worker/ExpireConversations.php → src/Worker/ExpireActivities.php
View file

@ -22,21 +22,15 @@
namespace Friendica\Worker;
use Friendica\Database\DBA;
use Friendica\DI;
use Friendica\Util\DateTimeFormat;
class ExpireConversations
class ExpireActivities
{
/**
* Delete old conversation entries
* Delete old post-activity entries
*/
public static function execute()
{
$days = intval(DI::config()->get('system', 'dbclean_expire_conversation', 90));
if (empty($days)) {
return;
}
DBA::delete('conversation', ["`received` < ?", DateTimeFormat::utc('now - ' . $days . ' days')]);
DBA::delete('post-activity', ["`received` < ?", DateTimeFormat::utc('now - 7 days')]);
}
}

2
src/Worker/Notifier.php
View file

@ -746,7 +746,7 @@ class Notifier
}
Logger::info('Origin item ' . $target_item['id'] . ' with URL ' . $target_item['uri'] . ' will be distributed.');
} elseif (!DBA::exists('conversation', ['item-uri' => $target_item['uri'], 'protocol' => Conversation::PARCEL_ACTIVITYPUB])) {
} elseif (!Post\Activity::exists($target_item['uri-id'])) {
Logger::info('Remote item ' . $target_item['id'] . ' with URL ' . $target_item['uri'] . ' is no AP post. It will not be distributed.');
return ['count' => 0, 'contacts' => []];
} elseif ($parent['origin']) {

32
static/dbstructure.config.php
View file

@ -55,7 +55,7 @@
use Friendica\Database\DBA;
if (!defined('DB_UPDATE_VERSION')) {
define('DB_UPDATE_VERSION', 1474);
define('DB_UPDATE_VERSION', 1475);
}
return [
@ -400,6 +400,7 @@ return [
"featured-tags" => ["type" => "varchar(255)", "comment" => "Address for the collection of featured tags"],
"manually-approve" => ["type" => "boolean", "comment" => ""],
"discoverable" => ["type" => "boolean", "comment" => "Mastodon extension: true if profile is published in their directory"],
"suspended" => ["type" => "boolean", "comment" => "Mastodon extension: true if profile is suspended"],
"nick" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
"name" => ["type" => "varchar(255)", "comment" => ""],
"about" => ["type" => "text", "comment" => ""],
@ -565,24 +566,6 @@ return [
"uid" => ["uid"],
]
],
"conversation" => [
"comment" => "Raw data and structure information for messages",
"fields" => [
"item-uri" => ["type" => "varbinary(255)", "not null" => "1", "primary" => "1", "comment" => "Original URI of the item - unrelated to the table with the same name"],
"reply-to-uri" => ["type" => "varbinary(255)", "not null" => "1", "default" => "", "comment" => "URI to which this item is a reply"],
"conversation-uri" => ["type" => "varbinary(255)", "not null" => "1", "default" => "", "comment" => "GNU Social conversation URI"],
"conversation-href" => ["type" => "varbinary(255)", "not null" => "1", "default" => "", "comment" => "GNU Social conversation link"],
"protocol" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "255", "comment" => "The protocol of the item"],
"direction" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => "How the message arrived here: 1=push, 2=pull"],
"source" => ["type" => "mediumtext", "comment" => "Original source"],
"received" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Receiving date"],
],
"indexes" => [
"PRIMARY" => ["item-uri"],
"conversation-uri" => ["conversation-uri"],
"received" => ["received"],
]
],
"workerqueue" => [
"comment" => "Background tasks queue entries",
"fields" => [
@ -1156,6 +1139,17 @@ return [
"vid" => ["vid"],
]
],
"post-activity" => [
"comment" => "Original remote activity",
"fields" => [
"uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
"activity" => ["type" => "mediumtext", "comment" => "Original activity"],
"received" => ["type" => "datetime", "comment" => ""],
],
"indexes" => [
"PRIMARY" => ["uri-id"],
]
],
"post-category" => [
"comment" => "post relation to categories",
"fields" => [