The post-reason / protocol is now filled in most cases

database.sql

@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 2022.09-dev (Giant Rhubarb)
--- DB_UPDATE_VERSION 1475
+-- DB_UPDATE_VERSION 1476
 -- ------------------------------------------
 
 
@@ -1774,6 +1774,7 @@ CREATE VIEW `post-user-view` AS SELECT
 	`post-user`.`global` AS `global`,
 	EXISTS(SELECT `type` FROM `post-collection` WHERE `type` = 0 AND `uri-id` = `post-user`.`uri-id`) AS `featured`,
 	`post-user`.`network` AS `network`,
+	`post-user`.`protocol` AS `protocol`,
 	`post-user`.`vid` AS `vid`,
 	`post-user`.`psid` AS `psid`,
 	IF (`post-user`.`vid` IS NULL, '', `verb`.`name`) AS `verb`,

src/Model/Conversation.php

@@ -32,11 +32,15 @@ class Conversation
 	const PARCEL_DIASPORA           = 2;
 	const PARCEL_SALMON             = 3;
 	const PARCEL_FEED               = 4; // Deprecated
-	const PARCEL_SPLIT_CONVERSATION = 6;
+	const PARCEL_SPLIT_CONVERSATION = 6; // @deprecated since version 2022.09
 	const PARCEL_LEGACY_DFRN        = 7; // @deprecated since version 2021.09
 	const PARCEL_DIASPORA_DFRN      = 8;
 	const PARCEL_LOCAL_DFRN         = 9;
 	const PARCEL_DIRECT             = 10;
+	const PARCEL_IMAP               = 11;
+	const PARCEL_RDF                = 12;
+	const PARCEL_RSS                = 13;
+	const PARCEL_ATOM               = 14;
 	const PARCEL_TWITTER            = 67;
 	const PARCEL_UNKNOWN            = 255;
 

src/Model/Item.php

@@ -124,7 +124,7 @@ class Item
 	// All fields in the item table
 	const ITEM_FIELDLIST = ['id', 'uid', 'parent', 'uri', 'parent-uri', 'thr-parent',
 			'guid', 'uri-id', 'parent-uri-id', 'thr-parent-id', 'conversation', 'vid',
-			'contact-id', 'wall', 'gravity', 'extid', 'psid',
+			'contact-id', 'wall', 'gravity', 'extid', 'psid', 
 			'created', 'edited', 'commented', 'received', 'changed', 'verb',
 			'postopts', 'plink', 'resource-id', 'event-id', 'inform',
 			'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid', 'post-type', 'post-reason',
@@ -813,6 +813,10 @@ class Item
 		unset($item['conversation-href']);
 		unset($item['source']);
 
+		if (in_array($item['network'], Protocol::FEDERATED) && (!isset($item['protocol']) || is_null($item['protocol']))) {
+			Logger::notice('Blubb', ['guid' => $item['guid'], 'uri-id' => $item['uri-id'], 'uri' => $item['uri'], 'callstack' => System::callstack(20)]);
+		}
+
 		/*
 		 * Do we already have this item?
 		 * We have to check several networks since Friendica posts could be repeated
@@ -1391,7 +1395,7 @@ class Item
 		$condition = ['id' => $itemid, 'uid' => 0,
 			'network' => array_merge(Protocol::FEDERATED ,['']),
 			'visible' => true, 'deleted' => false, 'private' => [self::PUBLIC, self::UNLISTED]];
-		$item = Post::selectFirst(self::ITEM_FIELDLIST, $condition);
+		$item = Post::selectFirst(array_merge(self::ITEM_FIELDLIST, ['protocol']), $condition);
 		if (!DBA::isResult($item)) {
 			Logger::warning('Item not found', ['condition' => $condition]);
 			return;
@@ -1480,7 +1484,7 @@ class Item
 			return 0;
 		}
 
-		$item = Post::selectFirst(self::ITEM_FIELDLIST, ['uri-id' => $uri_id, 'uid' => $source_uid]);
+		$item = Post::selectFirst(array_merge(self::ITEM_FIELDLIST, ['protocol']), ['uri-id' => $uri_id, 'uid' => $source_uid]);
 		if (!DBA::isResult($item)) {
 			Logger::warning('Item could not be fetched', ['uri-id' => $uri_id, 'uid' => $source_uid]);
 			return 0;
@@ -1697,7 +1701,7 @@ class Item
 			return;
 		}
 
-		$item = Post::selectFirst(self::ITEM_FIELDLIST, ['id' => $itemid]);
+		$item = Post::selectFirst(array_merge(self::ITEM_FIELDLIST, ['protocol']), ['id' => $itemid]);
 
 		if (DBA::isResult($item)) {
 			// Preparing public shadow (removing user specific data)
@@ -1733,7 +1737,7 @@ class Item
 	 */
 	private static function addShadowPost(int $itemid)
 	{
-		$item = Post::selectFirst(self::ITEM_FIELDLIST, ['id' => $itemid]);
+		$item = Post::selectFirst(array_merge(self::ITEM_FIELDLIST, ['protocol']), ['id' => $itemid]);
 		if (!DBA::isResult($item)) {
 			return;
 		}

src/Module/Api/Mastodon/PushSubscription.php

@@ -43,7 +43,9 @@ class PushSubscription extends BaseApi
 			'subscription' => [],
 			'data'         => [],
 		], $request);
-
+/*
+2022-07-31T08:08:11Z index [ERROR]: DB Error {"code":1366,"error":"Incorrect integer value: 'true' for column `piratica`.`subscription`.`follow` at row 1","callstack":"DBA::replace, Subscription::replace, PushSubscription::post, BaseModule::run, BaseApi::run, App::runFrontend","params":"REPLACE `subscription` (`application-id`, `uid`, `endpoint`, `pubkey`, `secret`, `follow`, `favourite`, `reblog`, `mention`, `poll`, `follow_request`, `status`) VALUES (213, 130, 'https://ntfy.sh/upNpejYKlqt5du?up=1', 'BEiGDNV6jsPwdtP186ZLbpqewcWBJHzM0qboxp8fVGoVxVUy6xiir_2RO4gM2FnE9sVg58sQdNuyDrr1jOmMj9Y', 'En7GzwQO8xuvXka5bIF3Sg', 'true', 'true', 'true', 'true', 'true', 0, 0)"} - {"file":"Database.php","line":801,"function":"e","uid":"c35eee","process_id":1404415}
+*/
 		$subscription = [
 			'application-id'                => $application['id'],
 			'uid'                           => $uid,

src/Protocol/DFRN.php

@@ -1830,8 +1830,6 @@ class DFRN
 
 		$item = $header;
 
-		$item['protocol'] = $protocol;
-
 		$item['source'] = $xml;
 
 		// Get the uri
@@ -2260,10 +2258,12 @@ class DFRN
 		$header = [];
 		$header['uid'] = $importer['importer_uid'];
 		$header['network'] = Protocol::DFRN;
+		$header['protocol'] = $protocol;
 		$header['wall'] = 0;
 		$header['origin'] = 0;
 		$header['contact-id'] = $importer['id'];
-		$header['direction'] = $direction;
+
+		$header = Diaspora::setDirection($header, $direction);
 
 		if ($direction === Conversation::RELAY) {
 			$header['post-reason'] = Item::PR_RELAY;

src/Protocol/Diaspora.php

@@ -1535,13 +1535,7 @@ class Diaspora
 		$datarray['owner-id'] = Contact::getIdForURL($contact['url'], 0);
 
 		// Will be overwritten for sharing accounts in Item::insert
-		if (in_array($direction, [self::FETCHED, self::FORCED_FETCH])) {
+		$datarray = self::setDirection($datarray, $direction);
-			$datarray['post-reason'] = Item::PR_FETCHED;
-		} elseif ($datarray['uid'] == 0) {
-			$datarray['post-reason'] = Item::PR_GLOBAL;
-		} else {
-			$datarray['post-reason'] = Item::PR_COMMENT;
-		}
 
 		$datarray['guid'] = $guid;
 		$datarray['uri'] = self::getUriFromGuid($author, $guid);
@@ -1557,7 +1551,8 @@ class Diaspora
 
 		$datarray['protocol'] = Conversation::PARCEL_DIASPORA;
 		$datarray['source'] = $xml;
-		$datarray['direction'] = in_array($direction, [self::FETCHED, self::FORCED_FETCH]) ? Conversation::PULL : Conversation::PUSH;
+
+		$datarray = self::setDirection($datarray, $direction);
 
 		$datarray['changed'] = $datarray['created'] = $datarray['edited'] = $created_at;
 
@@ -1786,12 +1781,13 @@ class Diaspora
 		$datarray = [];
 
 		$datarray['protocol'] = Conversation::PARCEL_DIASPORA;
-		$datarray['direction'] = in_array($direction, [self::FETCHED, self::FORCED_FETCH]) ? Conversation::PULL : Conversation::PUSH;
 
 		$datarray['uid'] = $importer['uid'];
 		$datarray['contact-id'] = $author_contact['cid'];
 		$datarray['network']  = $author_contact['network'];
 
+		$datarray = self::setDirection($datarray, $direction);
+
 		$datarray['owner-link'] = $datarray['author-link'] = $person['url'];
 		$datarray['owner-id'] = $datarray['author-id'] = Contact::getIdForURL($person['url'], 0);
 
@@ -1965,12 +1961,13 @@ class Diaspora
 		$datarray = [];
 
 		$datarray['protocol'] = Conversation::PARCEL_DIASPORA;
-		$datarray['direction'] = in_array($direction, [self::FETCHED, self::FORCED_FETCH]) ? Conversation::PULL : Conversation::PUSH;
 
 		$datarray['uid'] = $importer['uid'];
 		$datarray['contact-id'] = $author_contact['cid'];
 		$datarray['network']  = $author_contact['network'];
 
+		$datarray = self::setDirection($datarray, $direction);
+
 		$datarray['owner-link'] = $datarray['author-link'] = $person['url'];
 		$datarray['owner-id'] = $datarray['author-id'] = Contact::getIdForURL($person['url'], 0);
 
@@ -2382,6 +2379,7 @@ class Diaspora
 		$datarray['protocol'] = $item['protocol'];
 		$datarray['source'] = $item['source'];
 		$datarray['direction'] = $item['direction'];
+		$datarray['post-reason'] = $item['post-reason'];
 
 		$datarray['plink'] = self::plink($author, $datarray['guid']);
 		$datarray['private'] = $item['private'];
@@ -2468,7 +2466,8 @@ class Diaspora
 
 		$datarray['protocol'] = Conversation::PARCEL_DIASPORA;
 		$datarray['source'] = $xml;
-		$datarray['direction'] = in_array($direction, [self::FETCHED, self::FORCED_FETCH]) ? Conversation::PULL : Conversation::PUSH;
+
+		$datarray = self::setDirection($datarray, $direction);
 
 		/// @todo Copy tag data from original post
 
@@ -2690,6 +2689,29 @@ class Diaspora
 		Post\Media::insert($data);
 	}
 
+	/**
+	 * Set direction and post reason
+	 *
+	 * @param array $datarray
+	 * @param integer $direction
+	 *
+	 * @return array
+	 */
+	public static function setDirection(array $datarray, int $direction): array
+	{
+		$datarray['direction'] = in_array($direction, [self::FETCHED, self::FORCED_FETCH]) ? Conversation::PULL : Conversation::PUSH;
+
+		if (in_array($direction, [self::FETCHED, self::FORCED_FETCH])) {
+			$datarray['post-reason'] = Item::PR_FETCHED;
+		} elseif ($datarray['uid'] == 0) {
+			$datarray['post-reason'] = Item::PR_GLOBAL;
+		} else {
+			$datarray['post-reason'] = Item::PR_PUSHED;
+		}
+
+		return $datarray;
+	}
+
 	/**
 	 * Receives status messages
 	 *
@@ -2780,13 +2802,8 @@ class Diaspora
 
 		$datarray['protocol'] = Conversation::PARCEL_DIASPORA;
 		$datarray['source'] = $xml;
-		$datarray['direction'] = in_array($direction, [self::FETCHED, self::FORCED_FETCH]) ? Conversation::PULL : Conversation::PUSH;
 
-		if (in_array($direction, [self::FETCHED, self::FORCED_FETCH])) {
+		$datarray = self::setDirection($datarray, $direction);
-			$datarray['post-reason'] = Item::PR_FETCHED;
-		} elseif ($datarray['uid'] == 0) {
-			$datarray['post-reason'] = Item::PR_GLOBAL;
-		}
 
 		$datarray['body'] = self::replacePeopleGuid($body, $contact['url']);
 		$datarray['raw-body'] = self::replacePeopleGuid($raw_body, $contact['url']);

src/Protocol/Feed.php

@@ -33,6 +33,7 @@ use Friendica\Core\Protocol;
 use Friendica\Database\DBA;
 use Friendica\DI;
 use Friendica\Model\Contact;
+use Friendica\Model\Conversation;
 use Friendica\Model\Item;
 use Friendica\Model\Post;
 use Friendica\Model\Tag;
@@ -98,9 +99,11 @@ class Feed
 
 		$author = [];
 		$entries = null;
+		$protocol = Conversation::PARCEL_UNKNOWN;
 
 		// Is it RDF?
 		if ($xpath->query('/rdf:RDF/rss:channel')->length > 0) {
+			$protocol = Conversation::PARCEL_RDF;
 			$author['author-link'] = XML::getFirstNodeValue($xpath, '/rdf:RDF/rss:channel/rss:link/text()');
 			$author['author-name'] = XML::getFirstNodeValue($xpath, '/rdf:RDF/rss:channel/rss:title/text()');
 
@@ -112,6 +115,7 @@ class Feed
 
 		// Is it Atom?
 		if ($xpath->query('/atom:feed')->length > 0) {
+			$protocol = Conversation::PARCEL_ATOM;
 			$alternate = XML::getFirstAttributes($xpath, "atom:link[@rel='alternate']");
 			if (is_object($alternate)) {
 				foreach ($alternate as $attribute) {
@@ -195,6 +199,7 @@ class Feed
 
 		// Is it RSS?
 		if ($xpath->query('/rss/channel')->length > 0) {
+			$protocol = Conversation::PARCEL_RSS;
 			$author['author-link'] = XML::getFirstNodeValue($xpath, '/rss/channel/link/text()');
 
 			$author['author-name'] = XML::getFirstNodeValue($xpath, '/rss/channel/title/text()');
@@ -250,6 +255,8 @@ class Feed
 		$header = [];
 		$header['uid'] = $importer['uid'] ?? 0;
 		$header['network'] = Protocol::FEED;
+		$datarray['protocol'] = $protocol;
+		$datarray['direction'] = Conversation::PULL;
 		$header['wall'] = 0;
 		$header['origin'] = 0;
 		$header['gravity'] = GRAVITY_PARENT;

src/Worker/OnePoll.php

@@ -27,6 +27,7 @@ use Friendica\Core\System;
 use Friendica\Database\DBA;
 use Friendica\DI;
 use Friendica\Model\Contact;
+use Friendica\Model\Conversation;
 use Friendica\Model\Item;
 use Friendica\Model\Post;
 use Friendica\Model\User;
@@ -247,6 +248,9 @@ class OnePoll
 					$datarray['verb'] = Activity::POST;
 					$datarray['object-type'] = Activity\ObjectType::NOTE;
 					$datarray['network'] = Protocol::MAIL;
+					$datarray['protocol'] = Conversation::PARCEL_IMAP;
+					$datarray['direction'] = Conversation::PULL;
+				
 					// $meta = Email::messageMeta($mbox, $msg_uid);
 
 					$datarray['thr-parent'] = $datarray['uri'] = Email::msgid2iri(trim($meta->message_id, '<>'));

static/dbstructure.config.php

@@ -55,7 +55,7 @@
 use Friendica\Database\DBA;
 
 if (!defined('DB_UPDATE_VERSION')) {
-	define('DB_UPDATE_VERSION', 1475);
+	define('DB_UPDATE_VERSION', 1476);
 }
 
 return [

static/dbview.config.php

@@ -96,6 +96,7 @@
 			"global" => ["post-user", "global"],
 			"featured" => "EXISTS(SELECT `type` FROM `post-collection` WHERE `type` = 0 AND `uri-id` = `post-user`.`uri-id`)",
 			"network" => ["post-user", "network"],
+			"protocol" => ["post-user", "protocol"],
 			"vid" => ["post-user", "vid"],
 			"psid" => ["post-user", "psid"],
 			"verb" => "IF (`post-user`.`vid` IS NULL, '', `verb`.`name`)",