New table "post-content"

include/enotify.php

@@ -20,6 +20,7 @@
  */
 
 use Friendica\Content\Text\BBCode;
+use Friendica\Content\Text\Plaintext;
 use Friendica\Core\Hook;
 use Friendica\Core\Logger;
 use Friendica\Core\Renderer;
@@ -28,7 +29,6 @@ use Friendica\Database\DBA;
 use Friendica\DI;
 use Friendica\Model\Contact;
 use Friendica\Model\Item;
-use Friendica\Model\ItemContent;
 use Friendica\Model\Notification;
 use Friendica\Model\Post;
 use Friendica\Model\User;
@@ -176,7 +176,7 @@ function notification($params)
 
 		$item_post_type = Item::postType($item);
 
-		$content = ItemContent::getPlaintextPost($item, 70);
+		$content = Plaintext::getPost($item, 70);
 		if (!empty($content['text'])) {
 			$title = '"' . trim(str_replace("\n", " ", $content['text'])) . '"';
 		} else {

src/Content/Text/Plaintext.php

@@ -21,6 +21,9 @@
 
 namespace Friendica\Content\Text;
 
+use Friendica\Core\Protocol;
+use Friendica\DI;
+
 class Plaintext
 {
 	/**
@@ -88,4 +91,162 @@ class Plaintext
 
 		return $res;
 	}
+
+	/**
+	 * Convert a message into plaintext for connectors to other networks
+	 *
+	 * @param array  $item           The message array that is about to be posted
+	 * @param int    $limit          The maximum number of characters when posting to that network
+	 * @param bool   $includedlinks  Has an attached link to be included into the message?
+	 * @param int    $htmlmode       This controls the behavior of the BBCode conversion
+	 * @param string $target_network Name of the network where the post should go to.
+	 *
+	 * @return array Same array structure than \Friendica\Content\Text\BBCode::getAttachedData
+	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
+	 * @see   \Friendica\Content\Text\BBCode::getAttachedData
+	 *
+	 */
+	public static function getPost($item, $limit = 0, $includedlinks = false, $htmlmode = BBCode::API, $target_network = '')
+	{
+		// Remove hashtags
+		$URLSearchString = '^\[\]';
+		$body = preg_replace("/([#@])\[url\=([$URLSearchString]*)\](.*?)\[\/url\]/ism", '$1$3', $item['body']);
+
+		// Add an URL element if the text contains a raw link
+		$body = preg_replace('/([^\]\=\'"]|^)(https?\:\/\/[a-zA-Z0-9\:\/\-\?\&\;\.\=\_\~\#\%\$\!\+\,]+)/ism',
+			'$1[url]$2[/url]', $body);
+
+		// Remove the abstract
+		$body = BBCode::stripAbstract($body);
+
+		// At first look at data that is attached via "type-..." stuff
+		// This will hopefully replaced with a dedicated bbcode later
+		//$post = self::getAttachedData($b['body']);
+		$post = BBCode::getAttachedData($body, $item);
+
+		if (($item['title'] != '') && ($post['text'] != '')) {
+			$post['text'] = trim($item['title'] . "\n\n" . $post['text']);
+		} elseif ($item['title'] != '') {
+			$post['text'] = trim($item['title']);
+		}
+
+		$abstract = '';
+
+		// Fetch the abstract from the given target network
+		if ($target_network != '') {
+			$default_abstract = BBCode::getAbstract($item['body']);
+			$abstract = BBCode::getAbstract($item['body'], $target_network);
+
+			// If we post to a network with no limit we only fetch
+			// an abstract exactly for this network
+			if (($limit == 0) && ($abstract == $default_abstract)) {
+				$abstract = '';
+			}
+		} else {// Try to guess the correct target network
+			switch ($htmlmode) {
+				case BBCode::TWITTER:
+					$abstract = BBCode::getAbstract($item['body'], Protocol::TWITTER);
+					break;
+
+				case BBCode::OSTATUS:
+					$abstract = BBCode::getAbstract($item['body'], Protocol::STATUSNET);
+					break;
+
+				default: // We don't know the exact target.
+					// We fetch an abstract since there is a posting limit.
+					if ($limit > 0) {
+						$abstract = BBCode::getAbstract($item['body']);
+					}
+			}
+		}
+
+		if ($abstract != '') {
+			$post['text'] = $abstract;
+
+			if ($post['type'] == 'text') {
+				$post['type'] = 'link';
+				$post['url'] = $item['plink'];
+			}
+		}
+
+		$html = BBCode::convert($post['text'] . ($post['after'] ?? ''), false, $htmlmode);
+		$msg = HTML::toPlaintext($html, 0, true);
+		$msg = trim(html_entity_decode($msg, ENT_QUOTES, 'UTF-8'));
+
+		$link = '';
+		if ($includedlinks) {
+			if ($post['type'] == 'link') {
+				$link = $post['url'];
+			} elseif ($post['type'] == 'text') {
+				$link = $post['url'] ?? '';
+			} elseif ($post['type'] == 'video') {
+				$link = $post['url'];
+			} elseif ($post['type'] == 'photo') {
+				$link = $post['image'];
+			}
+
+			if (($msg == '') && isset($post['title'])) {
+				$msg = trim($post['title']);
+			}
+
+			if (($msg == '') && isset($post['description'])) {
+				$msg = trim($post['description']);
+			}
+
+			// If the link is already contained in the post, then it neeedn't to be added again
+			// But: if the link is beyond the limit, then it has to be added.
+			if (($link != '') && strstr($msg, $link)) {
+				$pos = strpos($msg, $link);
+
+				// Will the text be shortened in the link?
+				// Or is the link the last item in the post?
+				if (($limit > 0) && ($pos < $limit) && (($pos + 23 > $limit) || ($pos + strlen($link) == strlen($msg)))) {
+					$msg = trim(str_replace($link, '', $msg));
+				} elseif (($limit == 0) || ($pos < $limit)) {
+					// The limit has to be increased since it will be shortened - but not now
+					// Only do it with Twitter
+					if (($limit > 0) && (strlen($link) > 23) && ($htmlmode == BBCode::TWITTER)) {
+						$limit = $limit - 23 + strlen($link);
+					}
+
+					$link = '';
+
+					if ($post['type'] == 'text') {
+						unset($post['url']);
+					}
+				}
+			}
+		}
+
+		if ($limit > 0) {
+			// Reduce multiple spaces
+			// When posted to a network with limited space, we try to gain space where possible
+			while (strpos($msg, '  ') !== false) {
+				$msg = str_replace('  ', ' ', $msg);
+			}
+
+			// Twitter is using its own limiter, so we always assume that shortened links will have this length
+			if (iconv_strlen($link, 'UTF-8') > 0) {
+				$limit = $limit - 23;
+			}
+
+			if (iconv_strlen($msg, 'UTF-8') > $limit) {
+				if (($post['type'] == 'text') && isset($post['url'])) {
+					$post['url'] = $item['plink'];
+				} elseif (!isset($post['url'])) {
+					$limit = $limit - 23;
+					$post['url'] = $item['plink'];
+				} elseif (strpos($item['body'], '[share') !== false) {
+					$post['url'] = $item['plink'];
+				} elseif (DI::pConfig()->get($item['uid'], 'system', 'no_intelligent_shortening')) {
+					$post['url'] = $item['plink'];
+				}
+				$msg = self::shorten($msg, $limit);
+			}
+		}
+
+		$post['text'] = trim($msg);
+
+		return $post;
+	}
 }

src/Core/Update.php

@@ -72,8 +72,8 @@ class Update
 			}
 		}
 
-		// The postupdate has to completed version 1281 for the new post views to take over
+		// The postupdate has to completed version 1288 for the new post views to take over
-		$postupdate = DI::config()->get("system", "post_update_version");
+		$postupdate = DI::config()->get("system", "post_update_version", NEW_TABLE_STRUCTURE_VERSION);
 		if ($postupdate < NEW_TABLE_STRUCTURE_VERSION) {
 			$error = DI::l10n()->t('Updates from postupdate version %s are not supported. Please update at least to version 2021.01 and wait until the postupdate finished version 1383.', $postupdate);
 			if (DI::mode()->getExecutor() == Mode::INDEX) {

src/Database/DBStructure.php

@@ -72,8 +72,14 @@ class DBStructure
 	 */
 	public static function dropTables(bool $execute)
 	{
+		$postupdate = DI::config()->get("system", "post_update_version", PostUpdate::VERSION);
+		if ($postupdate < PostUpdate::VERSION) {
+			echo DI::l10n()->t('The post update is at version %d, it has to be at %d to safely drop the tables.', $postupdate, PostUpdate::VERSION);
+			return;
+		}
+
 		$old_tables = ['fserver', 'gcign', 'gcontact', 'gcontact-relation', 'gfollower' ,'glink', 'item-delivery-data',
-		'item_id', 'poll', 'poll_result', 'queue', 'retriever_rule', 'sign', 'spam', 'term'];
+			'item-activity', 'item-content', 'item_id', 'poll', 'poll_result', 'queue', 'retriever_rule', 'sign', 'spam', 'term'];
 
 		$tables = DBA::selectToArray(['INFORMATION_SCHEMA' => 'TABLES'], ['TABLE_NAME'],
 			['TABLE_SCHEMA' => DBA::databaseName(), 'TABLE_TYPE' => 'BASE TABLE']);

src/Database/PostUpdate.php

@@ -230,6 +230,11 @@ class PostUpdate
 			return true;
 		}
 
+		if (!DBStructure::existsTable('item-content')) {
+			DI::config()->set('system', 'post_update_version', 1342);
+			return true;
+		}
+
 		$id = DI::config()->get('system', 'post_update_version_1341_id', 0);
 
 		Logger::info('Start', ['item' => $id]);
@@ -283,7 +288,7 @@ class PostUpdate
 			return true;
 		}
 
-		if (!DBStructure::existsTable('term')) {
+		if (!DBStructure::existsTable('term') || !DBStructure::existsTable('item-content')) {
 			DI::config()->set('system', 'post_update_version', 1342);
 			return true;
 		}
@@ -510,6 +515,11 @@ class PostUpdate
 			return true;
 		}
 
+		if (!DBStructure::existsTable('item-activity')) {
+			DI::config()->set('system', 'post_update_version', 1347);
+			return true;
+		}
+
 		$id = DI::config()->get("system", "post_update_version_1347_id", 0);
 
 		Logger::info('Start', ['item' => $id]);

src/Model/Item.php

@@ -99,12 +99,12 @@ class Item
 			'author-id', 'author-link', 'owner-link', 'contact-uid',
 			'signed_text', 'network'];
 
-	// Field list for "item-content" table that is mixed with the item table
+	// Field list for "post-content" table that is mixed with the item table
 	const MIXED_CONTENT_FIELDLIST = ['title', 'content-warning', 'body', 'location',
 			'coord', 'app', 'rendered-hash', 'rendered-html', 'verb',
 			'object-type', 'object', 'target-type', 'target', 'plink'];
 
-	// Field list for "item-content" table that is not present in the "item" table
+	// Field list for "post-content" table that is not present in the "item" table
 	const CONTENT_FIELDLIST = ['language', 'raw-body'];
 
 	// All fields in the item table
@@ -133,7 +133,7 @@ class Item
 	const PRIVATE = 1;
 	const UNLISTED = 2;
 
-	const TABLES = ['item', 'user-item', 'item-content', 'post-delivery-data', 'diaspora-interaction'];
+	const TABLES = ['item', 'user-item', 'post-content', 'post-delivery-data', 'diaspora-interaction'];
 
 	private static function getItemFields()
 	{
@@ -224,7 +224,6 @@ class Item
 				// Remove all media attachments from the body and store them in the post-media table
 				$content_fields['raw-body'] = Post\Media::insertFromBody($item['uri-id'], $content_fields['raw-body']);
 				$content_fields['raw-body'] = self::setHashtags($content_fields['raw-body']);
-				self::updateContent($content_fields, ['uri-id' => $item['uri-id']]);
 			}
 
 			if (!empty($fields['file'])) {
@@ -1054,9 +1053,9 @@ class Item
 			$notify_type = Delivery::POST;
 		}
 
-		if (!in_array($item['verb'], self::ACTIVITIES) && !self::insertContent($item)) {
+		if (!in_array($item['verb'], self::ACTIVITIES) && !Post\Content::insert($item['uri-id'], $item)) {
 			// This shouldn't happen
-			Logger::warning('No content stored, quitting', ['guid' => $item['guid'], 'uri-id' => $item['uri-id'], 'causer-id' => ($item['causer-id'] ?? 0), 'post-type' => $item['post-type'], 'network' => $item['network']]);
+			Logger::warning('No post-content stored, quitting', ['guid' => $item['guid'], 'uri-id' => $item['uri-id'], 'causer-id' => ($item['causer-id'] ?? 0), 'post-type' => $item['post-type'], 'network' => $item['network']]);
 			return 0;
 		}
 
@@ -1300,67 +1299,6 @@ class Item
 		}
 	}
 
-	/**
-	 * Insert a new item content entry
-	 *
-	 * @param array $item The item fields that are to be inserted
-	 * @return bool "true" if content was inserted or already existed
-	 * @throws \Exception
-	 */
-	private static function insertContent(array $item)
-	{
-		$fields = ['uri-plink-hash' => (string)$item['uri-id'], 'uri-id' => $item['uri-id']];
-
-		foreach (array_merge(self::CONTENT_FIELDLIST, self::MIXED_CONTENT_FIELDLIST) as $field) {
-			if (isset($item[$field])) {
-				$fields[$field] = $item[$field];
-			}
-		}
-
-		$found = DBA::exists('item-content', ['uri-id' => $item['uri-id']]);
-		if ($found) {
-			Logger::info('Existing content found', ['uri-id' => $item['uri-id'], 'uri' => $item['uri']]);
-			return true;
-		}
-
-		DBA::insert('item-content', $fields, Database::INSERT_IGNORE);
-
-		$found = DBA::exists('item-content', ['uri-id' => $item['uri-id']]);
-		if ($found) {
-			Logger::notice('Content inserted', ['uri-id' => $item['uri-id'], 'uri' => $item['uri']]);
-			return true;
-		}
-
-		// This shouldn't happen.
-		Logger::error("Content wasn't inserted", $item);
-		return false;
-	}
-
-	/**
-	 * Update existing item content entries
-	 *
-	 * @param array $item      The item fields that are to be changed
-	 * @param array $condition The condition for finding the item content entries
-	 * @throws \Exception
-	 */
-	private static function updateContent($item, $condition)
-	{
-		// We have to select only the fields from the "item-content" table
-		$fields = [];
-		foreach (array_merge(self::CONTENT_FIELDLIST, self::MIXED_CONTENT_FIELDLIST) as $field) {
-			if (isset($item[$field])) {
-				$fields[$field] = $item[$field];
-			}
-		}
-
-		if (empty($fields)) {
-			return;
-		}
-
-		DBA::update('item-content', $fields, $condition, true);
-		Logger::info('Updated content', ['condition' => $condition]);
-	}
-
 	/**
 	 * Distributes public items to the receivers
 	 *

src/Model/ItemContent.php

@@ -1,236 +0,0 @@
-<?php
-/**
- * @copyright Copyright (C) 2020, Friendica
- *
- * @license GNU AGPL version 3 or any later version
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- *
- */
-
-namespace Friendica\Model;
-
-use Friendica\Content\Text;
-use Friendica\Content\Text\BBCode;
-use Friendica\Core\Protocol;
-use Friendica\Database\DBA;
-use Friendica\DI;
-
-class ItemContent
-{
-	/**
-	 * Search posts for given content
-	 *
-	 * @param string $search
-	 * @param integer $uid
-	 * @param integer $start
-	 * @param integer $limit
-	 * @param integer $last_uriid
-	 * @return array
-	 */
-	public static function getURIIdListBySearch(string $search, int $uid = 0, int $start = 0, int $limit = 100, int $last_uriid = 0)
-	{
-		$condition = ["`uri-id` IN (SELECT `uri-id` FROM `item-content` WHERE MATCH (`title`, `content-warning`, `body`) AGAINST (? IN BOOLEAN MODE))
-			AND (NOT `private` OR (`private` AND `uid` = ?))
-			AND `uri-id` IN (SELECT `uri-id` FROM `post-view` WHERE `network` IN (?, ?, ?, ?))",
-			$search, $uid, Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS];
-
-		if (!empty($last_uriid)) {
-			$condition = DBA::mergeConditions($condition, ["`uri-id` < ?", $last_uriid]);
-		}
-
-		$params = [
-			'order' => ['uri-id' => true],
-			'group_by' => ['uri-id'],
-			'limit' => [$start, $limit]
-		];
-
-		$tags = Post::select(['uri-id'], $condition, $params);
-
-		$uriids = [];
-		while ($tag = DBA::fetch($tags)) {
-			$uriids[] = $tag['uri-id'];
-		}
-		DBA::close($tags);
-
-		return $uriids;
-	}
-
-	public static function countBySearch(string $search, int $uid = 0)
-	{
-		$condition = ["`uri-id` IN (SELECT `uri-id` FROM `item-content` WHERE MATCH (`title`, `content-warning`, `body`) AGAINST (? IN BOOLEAN MODE))
-			AND (NOT `private` OR (`private` AND `uid` = ?))
-			AND `uri-id` IN (SELECT `uri-id` FROM `post-view` WHERE `network` IN (?, ?, ?, ?))",
-			$search, $uid, Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS];
-		return Post::count($condition);
-	}
-
-	/**
-	 * Convert a message into plaintext for connectors to other networks
-	 *
-	 * @param array  $item           The message array that is about to be posted
-	 * @param int    $limit          The maximum number of characters when posting to that network
-	 * @param bool   $includedlinks  Has an attached link to be included into the message?
-	 * @param int    $htmlmode       This controls the behavior of the BBCode conversion
-	 * @param string $target_network Name of the network where the post should go to.
-	 *
-	 * @return array Same array structure than \Friendica\Content\Text\BBCode::getAttachedData
-	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
-	 * @see   \Friendica\Content\Text\BBCode::getAttachedData
-	 *
-	 */
-	public static function getPlaintextPost($item, $limit = 0, $includedlinks = false, $htmlmode = BBCode::API, $target_network = '')
-	{
-		// Remove hashtags
-		$URLSearchString = '^\[\]';
-		$body = preg_replace("/([#@])\[url\=([$URLSearchString]*)\](.*?)\[\/url\]/ism", '$1$3', $item['body']);
-
-		// Add an URL element if the text contains a raw link
-		$body = preg_replace('/([^\]\=\'"]|^)(https?\:\/\/[a-zA-Z0-9\:\/\-\?\&\;\.\=\_\~\#\%\$\!\+\,]+)/ism',
-			'$1[url]$2[/url]', $body);
-
-		// Remove the abstract
-		$body = Text\BBCode::stripAbstract($body);
-
-		// At first look at data that is attached via "type-..." stuff
-		// This will hopefully replaced with a dedicated bbcode later
-		//$post = self::getAttachedData($b['body']);
-		$post = Text\BBCode::getAttachedData($body, $item);
-
-		if (($item['title'] != '') && ($post['text'] != '')) {
-			$post['text'] = trim($item['title'] . "\n\n" . $post['text']);
-		} elseif ($item['title'] != '') {
-			$post['text'] = trim($item['title']);
-		}
-
-		$abstract = '';
-
-		// Fetch the abstract from the given target network
-		if ($target_network != '') {
-			$default_abstract = Text\BBCode::getAbstract($item['body']);
-			$abstract = Text\BBCode::getAbstract($item['body'], $target_network);
-
-			// If we post to a network with no limit we only fetch
-			// an abstract exactly for this network
-			if (($limit == 0) && ($abstract == $default_abstract)) {
-				$abstract = '';
-			}
-		} else {// Try to guess the correct target network
-			switch ($htmlmode) {
-				case BBCode::TWITTER:
-					$abstract = Text\BBCode::getAbstract($item['body'], Protocol::TWITTER);
-					break;
-
-				case BBCode::OSTATUS:
-					$abstract = Text\BBCode::getAbstract($item['body'], Protocol::STATUSNET);
-					break;
-
-				default: // We don't know the exact target.
-					// We fetch an abstract since there is a posting limit.
-					if ($limit > 0) {
-						$abstract = Text\BBCode::getAbstract($item['body']);
-					}
-			}
-		}
-
-		if ($abstract != '') {
-			$post['text'] = $abstract;
-
-			if ($post['type'] == 'text') {
-				$post['type'] = 'link';
-				$post['url'] = $item['plink'];
-			}
-		}
-
-		$html = Text\BBCode::convert($post['text'] . ($post['after'] ?? ''), false, $htmlmode);
-		$msg = Text\HTML::toPlaintext($html, 0, true);
-		$msg = trim(html_entity_decode($msg, ENT_QUOTES, 'UTF-8'));
-
-		$link = '';
-		if ($includedlinks) {
-			if ($post['type'] == 'link') {
-				$link = $post['url'];
-			} elseif ($post['type'] == 'text') {
-				$link = $post['url'] ?? '';
-			} elseif ($post['type'] == 'video') {
-				$link = $post['url'];
-			} elseif ($post['type'] == 'photo') {
-				$link = $post['image'];
-			}
-
-			if (($msg == '') && isset($post['title'])) {
-				$msg = trim($post['title']);
-			}
-
-			if (($msg == '') && isset($post['description'])) {
-				$msg = trim($post['description']);
-			}
-
-			// If the link is already contained in the post, then it neeedn't to be added again
-			// But: if the link is beyond the limit, then it has to be added.
-			if (($link != '') && strstr($msg, $link)) {
-				$pos = strpos($msg, $link);
-
-				// Will the text be shortened in the link?
-				// Or is the link the last item in the post?
-				if (($limit > 0) && ($pos < $limit) && (($pos + 23 > $limit) || ($pos + strlen($link) == strlen($msg)))) {
-					$msg = trim(str_replace($link, '', $msg));
-				} elseif (($limit == 0) || ($pos < $limit)) {
-					// The limit has to be increased since it will be shortened - but not now
-					// Only do it with Twitter
-					if (($limit > 0) && (strlen($link) > 23) && ($htmlmode == BBCode::TWITTER)) {
-						$limit = $limit - 23 + strlen($link);
-					}
-
-					$link = '';
-
-					if ($post['type'] == 'text') {
-						unset($post['url']);
-					}
-				}
-			}
-		}
-
-		if ($limit > 0) {
-			// Reduce multiple spaces
-			// When posted to a network with limited space, we try to gain space where possible
-			while (strpos($msg, '  ') !== false) {
-				$msg = str_replace('  ', ' ', $msg);
-			}
-
-			// Twitter is using its own limiter, so we always assume that shortened links will have this length
-			if (iconv_strlen($link, 'UTF-8') > 0) {
-				$limit = $limit - 23;
-			}
-
-			if (iconv_strlen($msg, 'UTF-8') > $limit) {
-				if (($post['type'] == 'text') && isset($post['url'])) {
-					$post['url'] = $item['plink'];
-				} elseif (!isset($post['url'])) {
-					$limit = $limit - 23;
-					$post['url'] = $item['plink'];
-				} elseif (strpos($item['body'], '[share') !== false) {
-					$post['url'] = $item['plink'];
-				} elseif (DI::pConfig()->get($item['uid'], 'system', 'no_intelligent_shortening')) {
-					$post['url'] = $item['plink'];
-				}
-				$msg = Text\Plaintext::shorten($msg, $limit);
-			}
-		}
-
-		$post['text'] = trim($msg);
-
-		return $post;
-	}
-}

src/Model/Post.php

@@ -446,13 +446,13 @@ class Post
 			$affected = DBA::affectedRows();			
 		}
 
-		$update_fields = DBStructure::getFieldsForTable('item-content', $fields);
+		$update_fields = DBStructure::getFieldsForTable('post-content', $fields);
 		if (!empty($update_fields)) {
 			$rows = DBA::selectToArray('post-view', ['uri-id'], $condition, ['group_by' => ['uri-id']]);
 			$uriids = array_column($rows, 'uri-id');
-			if (!DBA::update('item-content', $update_fields, ['uri-id' => $uriids])) {
+			if (!DBA::update('post-content', $update_fields, ['uri-id' => $uriids])) {
 				DBA::rollback();
-				Logger::notice('Updating item-content failed', ['fields' => $update_fields, 'condition' => $condition]);
+				Logger::notice('Updating post-content failed', ['fields' => $update_fields, 'condition' => $condition]);
 				return false;
 			}
 			$affected = max($affected, DBA::affectedRows());

src/Model/Post/Content.php

@@ -0,0 +1,147 @@
+<?php
+/**
+ * @copyright Copyright (C) 2020, Friendica
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace Friendica\Model\Post;
+
+use \BadMethodCallException;
+use Friendica\Core\Protocol;
+use Friendica\Database\Database;
+use Friendica\Database\DBA;
+use Friendica\Database\DBStructure;
+use Friendica\Model\Post;
+
+class Content
+{
+	/**
+	 * Insert a new post-content entry
+	 *
+	 * @param integer $uri_id
+	 * @param array   $fields
+	 * @return bool   success
+	 * @throws \Exception
+	 */
+	public static function insert(int $uri_id, array $data = [])
+	{
+		if (empty($uri_id)) {
+			throw new BadMethodCallException('Empty URI_id');
+		}
+
+		if (DBA::exists('post-content', ['uri-id' => $uri_id])) {
+			return false;
+		}
+
+		$fields = DBStructure::getFieldsForTable('post-content', $data);
+
+		// Additionally assign the key fields
+		$fields['uri-id'] = $uri_id;
+
+		return DBA::insert('post-content', $fields, Database::INSERT_IGNORE);
+	}
+
+	/**
+	 * Update a post content entry
+	 *
+	 * @param integer $uri_id
+	 * @param array   $data
+	 * @param bool    $insert_if_missing
+	 * @return bool
+	 * @throws \Exception
+	 */
+	public static function update(int $uri_id, array $data = [], bool $insert_if_missing = false)
+	{
+		if (empty($uri_id)) {
+			throw new BadMethodCallException('Empty URI_id');
+		}
+
+		$fields = DBStructure::getFieldsForTable('post-content', $data);
+
+		// Remove the key fields
+		unset($fields['uri-id']);
+
+		if (empty($fields)) {
+			return true;
+		}
+
+		return DBA::update('post-content', $fields, ['uri-id' => $uri_id], $insert_if_missing ? true : []);
+	}
+
+	/**
+	 * Delete a row from the post-content table
+	 *
+	 * @param array        $conditions Field condition(s)
+	 * @param array        $options
+	 *                           - cascade: If true we delete records in other tables that depend on the one we're deleting through
+	 *                           relations (default: true)
+	 *
+	 * @return boolean was the delete successful?
+	 * @throws \Exception
+	 */
+	public static function delete(array $conditions, array $options = [])
+	{
+		return DBA::delete('post-content', $conditions, $options);
+	}
+
+
+	/**
+	 * Search posts for given content
+	 *
+	 * @param string $search
+	 * @param integer $uid
+	 * @param integer $start
+	 * @param integer $limit
+	 * @param integer $last_uriid
+	 * @return array
+	 */
+	public static function getURIIdListBySearch(string $search, int $uid = 0, int $start = 0, int $limit = 100, int $last_uriid = 0)
+	{
+		$condition = ["`uri-id` IN (SELECT `uri-id` FROM `post-content` WHERE MATCH (`title`, `content-warning`, `body`) AGAINST (? IN BOOLEAN MODE))
+			AND (NOT `private` OR (`private` AND `uid` = ?)) AND (`network` IN (?, ?, ?, ?) OR (`uid` = ? AND `uid` != ?))",
+			$search, $uid, Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS, $uid, 0];
+
+		if (!empty($last_uriid)) {
+			$condition = DBA::mergeConditions($condition, ["`uri-id` < ?", $last_uriid]);
+		}
+
+		$params = [
+			'order' => ['uri-id' => true],
+			'group_by' => ['uri-id'],
+			'limit' => [$start, $limit]
+		];
+
+		$tags = Post::select(['uri-id'], $condition, $params);
+
+		$uriids = [];
+		while ($tag = DBA::fetch($tags)) {
+			$uriids[] = $tag['uri-id'];
+		}
+		DBA::close($tags);
+
+		return $uriids;
+	}
+
+	public static function countBySearch(string $search, int $uid = 0)
+	{
+		$condition = ["`uri-id` IN (SELECT `uri-id` FROM `post-content` WHERE MATCH (`title`, `content-warning`, `body`) AGAINST (? IN BOOLEAN MODE))
+			AND (NOT `private` OR (`private` AND `uid` = ?)) AND (`network` IN (?, ?, ?, ?) OR (`uid` = ? AND `uid` != ?))",
+			$search, $uid, Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS, $uid, 0];
+		return Post::count($condition);
+	}
+}

src/Module/Search/Index.php

@@ -166,8 +166,8 @@ class Index extends BaseSearch
 			$count = Tag::countByTag($search, local_user());
 		} else {
 			Logger::info('Start fulltext search.', ['q' => $search]);
-			$uriids = ItemContent::getURIIdListBySearch($search, local_user(), $pager->getStart(), $pager->getItemsPerPage(), $last_uriid);
+			$uriids = Post\Content::getURIIdListBySearch($search, local_user(), $pager->getStart(), $pager->getItemsPerPage(), $last_uriid);
-			$count = ItemContent::countBySearch($search, local_user());
+			$count = Post\Content::countBySearch($search, local_user());
 		}
 
 		if (!empty($uriids)) {

src/Worker/Expire.php

@@ -55,8 +55,8 @@ class Expire
 			// Normally we shouldn't have orphaned data at all.
 			// If we do have some, then we have to check why.
 			Logger::log('Deleting orphaned item content - start', Logger::DEBUG);
-			$condition = ["NOT EXISTS (SELECT `uri-id` FROM `item` WHERE `item`.`uri-id` = `item-content`.`uri-id`)"];
+			$condition = ["NOT EXISTS (SELECT `uri-id` FROM `item` WHERE `item`.`uri-id` = `post-content`.`uri-id`)"];
-			DBA::delete('item-content', $condition);
+			DBA::delete('post-content', $condition);
 			Logger::log('Orphaned item content deleted: ' . DBA::affectedRows(), Logger::DEBUG);
 
 			// make this optional as it could have a performance impact on large sites

static/dbstructure.config.php

@@ -55,7 +55,7 @@
 use Friendica\Database\DBA;
 
 if (!defined('DB_UPDATE_VERSION')) {
-	define('DB_UPDATE_VERSION', 1395);
+	define('DB_UPDATE_VERSION', 1396);
 }
 
 return [
@@ -858,55 +858,6 @@ return [
 			"causer-id" => ["causer-id"],
 		]
 	],
-	"item-activity" => [
-		"comment" => "Activities for items",
-		"fields" => [
-			"id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1"],
-			"uri" => ["type" => "varchar(255)", "comment" => ""],
-			"uri-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
-			"uri-hash" => ["type" => "varchar(80)", "not null" => "1", "default" => "", "comment" => "RIPEMD-128 hash from uri"],
-			"activity" => ["type" => "smallint unsigned", "not null" => "1", "default" => "0", "comment" => ""]
-		],
-		"indexes" => [
-			"PRIMARY" => ["id"],
-			"uri-hash" => ["UNIQUE", "uri-hash"],
-			"uri" => ["uri(191)"],
-			"uri-id" => ["uri-id"]
-		]
-	],
-	"item-content" => [
-		"comment" => "Content for all posts",
-		"fields" => [
-			"id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1"],
-			"uri" => ["type" => "varchar(255)", "comment" => ""],
-			"uri-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
-			"uri-plink-hash" => ["type" => "varchar(80)", "not null" => "1", "default" => "", "comment" => "RIPEMD-128 hash from uri"],
-			"title" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "item title"],
-			"content-warning" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
-			"body" => ["type" => "mediumtext", "comment" => "item body content"],
-			"raw-body" => ["type" => "mediumtext", "comment" => "Body without embedded media links"],
-			"location" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "text location where this item originated"],
-			"coord" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "longitude/latitude pair representing location where this item originated"],
-			"language" => ["type" => "text", "comment" => "Language information about this post"],
-			"app" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "application which generated this item"],
-			"rendered-hash" => ["type" => "varchar(32)", "not null" => "1", "default" => "", "comment" => ""],
-			"rendered-html" => ["type" => "mediumtext", "comment" => "item.body converted to html"],
-			"object-type" => ["type" => "varchar(100)", "not null" => "1", "default" => "", "comment" => "ActivityStreams object type"],
-			"object" => ["type" => "text", "comment" => "JSON encoded object structure unless it is an implied object (normal post)"],
-			"target-type" => ["type" => "varchar(100)", "not null" => "1", "default" => "", "comment" => "ActivityStreams target type if applicable (URI)"],
-			"target" => ["type" => "text", "comment" => "JSON encoded target structure if used"],
-			"plink" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "permalink or URL to a displayable copy of the message at its source"],
-			"verb" => ["type" => "varchar(100)", "not null" => "1", "default" => "", "comment" => "ActivityStreams verb"]
-		],
-		"indexes" => [
-			"PRIMARY" => ["id"],
-			"uri-plink-hash" => ["UNIQUE", "uri-plink-hash"],
-			"title-content-warning-body" => ["FULLTEXT", "title", "content-warning", "body"],
-			"uri" => ["uri(191)"],
-			"plink" => ["plink(191)"],
-			"uri-id" => ["uri-id"]
-		]
-	],
 	"locks" => [
 		"comment" => "",
 		"fields" => [
@@ -1161,6 +1112,33 @@ return [
 			"uid" => ["uid"],
 		]
 	],
+	"post-content" => [
+		"comment" => "Content for all posts",
+		"fields" => [
+			"uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
+			"title" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "item title"],
+			"content-warning" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
+			"body" => ["type" => "mediumtext", "comment" => "item body content"],
+			"raw-body" => ["type" => "mediumtext", "comment" => "Body without embedded media links"],
+			"location" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "text location where this item originated"],
+			"coord" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "longitude/latitude pair representing location where this item originated"],
+			"language" => ["type" => "text", "comment" => "Language information about this post"],
+			"app" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "application which generated this item"],
+			"rendered-hash" => ["type" => "varchar(32)", "not null" => "1", "default" => "", "comment" => ""],
+			"rendered-html" => ["type" => "mediumtext", "comment" => "item.body converted to html"],
+			"object-type" => ["type" => "varchar(100)", "not null" => "1", "default" => "", "comment" => "ActivityStreams object type"],
+			"object" => ["type" => "text", "comment" => "JSON encoded object structure unless it is an implied object (normal post)"],
+			"target-type" => ["type" => "varchar(100)", "not null" => "1", "default" => "", "comment" => "ActivityStreams target type if applicable (URI)"],
+			"target" => ["type" => "text", "comment" => "JSON encoded target structure if used"],
+			"resource-id" => ["type" => "varchar(32)", "not null" => "1", "default" => "", "comment" => "Used to link other tables to items, it identifies the linked resource (e.g. photo) and if set must also set resource_type"],
+			"plink" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "permalink or URL to a displayable copy of the message at its source"]
+		],
+		"indexes" => [
+			"PRIMARY" => ["uri-id"],
+			"plink" => ["plink(191)"],
+			"title-content-warning-body" => ["FULLTEXT", "title", "content-warning", "body"],
+		]
+	],
 	"post-delivery-data" => [
 		"comment" => "Delivery data for items",
 		"fields" => [

static/dbview.config.php

@@ -60,7 +60,6 @@
 			"commented" => ["item", "commented"],
 			"received" => ["item", "received"],
 			"changed" => ["item", "changed"],
-			"resource-id" => ["item", "resource-id"],
 			"post-type" => ["item", "post-type"],
 			"private" => ["item", "private"],
 			"pubmail" => ["item", "pubmail"],
@@ -78,21 +77,22 @@
 			"vid" => ["item", "vid"],
 			"psid" => ["item", "psid"],
 			"verb" => "IF (`item`.`vid` IS NULL, '', `verb`.`name`)",
-			"title" => ["item-content", "title"],
+			"title" => ["post-content", "title"],
-			"content-warning" => ["item-content", "content-warning"],
+			"content-warning" => ["post-content", "content-warning"],
-			"raw-body" => ["item-content", "raw-body"],
+			"raw-body" => ["post-content", "raw-body"],
-			"body" => ["item-content", "body"],
+			"body" => ["post-content", "body"],
-			"rendered-hash" => ["item-content", "rendered-hash"],
+			"rendered-hash" => ["post-content", "rendered-hash"],
-			"rendered-html" => ["item-content", "rendered-html"],
+			"rendered-html" => ["post-content", "rendered-html"],
-			"language" => ["item-content", "language"],
+			"language" => ["post-content", "language"],
-			"plink" => ["item-content", "plink"],
+			"plink" => ["post-content", "plink"],
-			"location" => ["item-content", "location"],
+			"location" => ["post-content", "location"],
-			"coord" => ["item-content", "coord"],
+			"coord" => ["post-content", "coord"],
-			"app" => ["item-content", "app"],
+			"app" => ["post-content", "app"],
-			"object-type" => ["item-content", "object-type"],
+			"object-type" => ["post-content", "object-type"],
-			"object" => ["item-content", "object"],
+			"object" => ["post-content", "object"],
-			"target-type" => ["item-content", "target-type"],
+			"target-type" => ["post-content", "target-type"],
-			"target" => ["item-content", "target"],
+			"target" => ["post-content", "target"],
+			"resource-id" => ["post-content", "resource-id"],
 			"contact-id" => ["item", "contact-id"],
 			"contact-link" => ["contact", "url"],
 			"contact-addr" => ["contact", "addr"],
@@ -184,7 +184,7 @@
 			LEFT JOIN `verb` ON `verb`.`id` = `item`.`vid`
 			LEFT JOIN `event` ON `event`.`id` = `item`.`event-id`
 			LEFT JOIN `diaspora-interaction` ON `diaspora-interaction`.`uri-id` = `item`.`uri-id`
-			LEFT JOIN `item-content` ON `item-content`.`uri-id` = `item`.`uri-id`
+			LEFT JOIN `post-content` ON `post-content`.`uri-id` = `item`.`uri-id`
 			LEFT JOIN `post-delivery-data` ON `post-delivery-data`.`uri-id` = `item`.`uri-id` AND `item`.`origin`
 			LEFT JOIN `permissionset` ON `permissionset`.`id` = `item`.`psid`
 			STRAIGHT_JOIN `item` AS `parent-item` ON `parent-item`.`uri-id` = `item`.`parent-uri-id` AND `parent-item`.`uid` = `item`.`uid`
@@ -213,7 +213,6 @@
 			"commented" => ["thread", "commented"],
 			"received" => ["thread", "received"],
 			"changed" => ["thread", "changed"],
-			"resource-id" => ["item", "resource-id"],
 			"post-type" => ["thread", "post-type"],
 			"private" => ["thread", "private"],
 			"pubmail" => ["thread", "pubmail"],
@@ -232,21 +231,22 @@
 			"vid" => ["item", "vid"],
 			"psid" => ["item", "psid"],
 			"verb" => "IF (`item`.`vid` IS NULL, '', `verb`.`name`)",
-			"title" => ["item-content", "title"],
+			"title" => ["post-content", "title"],
-			"content-warning" => ["item-content", "content-warning"],
+			"content-warning" => ["post-content", "content-warning"],
-			"raw-body" => ["item-content", "raw-body"],
+			"raw-body" => ["post-content", "raw-body"],
-			"body" => ["item-content", "body"],
+			"body" => ["post-content", "body"],
-			"rendered-hash" => ["item-content", "rendered-hash"],
+			"rendered-hash" => ["post-content", "rendered-hash"],
-			"rendered-html" => ["item-content", "rendered-html"],
+			"rendered-html" => ["post-content", "rendered-html"],
-			"language" => ["item-content", "language"],
+			"language" => ["post-content", "language"],
-			"plink" => ["item-content", "plink"],
+			"plink" => ["post-content", "plink"],
-			"location" => ["item-content", "location"],
+			"location" => ["post-content", "location"],
-			"coord" => ["item-content", "coord"],
+			"coord" => ["post-content", "coord"],
-			"app" => ["item-content", "app"],
+			"app" => ["post-content", "app"],
-			"object-type" => ["item-content", "object-type"],
+			"object-type" => ["post-content", "object-type"],
-			"object" => ["item-content", "object"],
+			"object" => ["post-content", "object"],
-			"target-type" => ["item-content", "target-type"],
+			"target-type" => ["post-content", "target-type"],
-			"target" => ["item-content", "target"],
+			"target" => ["post-content", "target"],
+			"resource-id" => ["post-content", "resource-id"],
 			"contact-id" => ["thread", "contact-id"],
 			"contact-link" => ["contact", "url"],
 			"contact-addr" => ["contact", "addr"],
@@ -338,7 +338,7 @@
 			LEFT JOIN `verb` ON `verb`.`id` = `item`.`vid`
 			LEFT JOIN `event` ON `event`.`id` = `item`.`event-id`
 			LEFT JOIN `diaspora-interaction` ON `diaspora-interaction`.`uri-id` = `thread`.`uri-id`
-			LEFT JOIN `item-content` ON `item-content`.`uri-id` = `thread`.`uri-id`
+			LEFT JOIN `post-content` ON `post-content`.`uri-id` = `thread`.`uri-id`
 			LEFT JOIN `post-delivery-data` ON `post-delivery-data`.`uri-id` = `thread`.`uri-id` AND `thread`.`origin`
 			LEFT JOIN `permissionset` ON `permissionset`.`id` = `item`.`psid`
 			STRAIGHT_JOIN `item` AS `parent-item` ON `parent-item`.`id` = `item`.`parent`

tests/datasets/api.fixture.php

@@ -179,48 +179,36 @@ return [
 			'guid' => '6',
 		],
 	],
-	'item-content' => [
+	'post-content' => [
 		[
-			'id'          => 1,
+			'uri-id' => 1,
-			'uri-id'      => 1,
+			'body'   => 'Parent status',
-			'uri-plink-hash' => '1',
+			'plink'  => 'http://localhost/display/1',
-			'body'        => 'Parent status',
-			'plink'       => 'http://localhost/display/1',
 		],
 		[
-			'id'          => 2,
+			'uri-id' => 2,
-			'uri-id'      => 2,
+			'body'   => 'Reply',
-			'uri-plink-hash' => '2',
+			'plink'  => 'http://localhost/display/2',
-			'body'        => 'Reply',
-			'plink'       => 'http://localhost/display/2',
 		],
 		[
-			'id'          => 3,
+			'uri-id' => 3,
-			'uri-id'      => 3,
+			'body'   => 'Other user status',
-			'uri-plink-hash' => '3',
+			'plink'  => 'http://localhost/display/3',
-			'body'        => 'Other user status',
-			'plink'       => 'http://localhost/display/3',
 		],
 		[
-			'id'          => 4,
+			'uri-id' => 4,
-			'uri-id'      => 4,
+			'body'   => 'Friend user reply',
-			'uri-plink-hash' => '4',
+			'plink'  => 'http://localhost/display/4',
-			'body'        => 'Friend user reply',
-			'plink'       => 'http://localhost/display/4',
 		],
 		[
-			'id'          => 5,
+			'uri-id' => 5,
-			'uri-id'      => 5,
+			'body'   => '[share]Shared status[/share]',
-			'uri-plink-hash' => '5',
+			'plink'  => 'http://localhost/display/5',
-			'body'        => '[share]Shared status[/share]',
-			'plink'       => 'http://localhost/display/5',
 		],
 		[
-			'id'          => 6,
+			'uri-id' => 6,
-			'uri-id'      => 6,
+			'body'   => 'Friend user status',
-			'uri-plink-hash' => '6',
+			'plink'  => 'http://localhost/display/6',
-			'body'        => 'Friend user status',
-			'plink'       => 'http://localhost/display/6',
 		],
 	],
 	'post-user' => [

update.php

@@ -266,6 +266,10 @@ function update_1348()
 
 function update_1349()
 {
+	if (!DBStructure::existsTable('item-activity')) {
+		return Update::SUCCESS;
+	}
+
 	$correct = true;
 	foreach (Item::ACTIVITIES as $index => $activity) {
 		if (!DBA::exists('verb', ['id' => $index + 1, 'name' => $activity])) {
@@ -690,3 +694,23 @@ function update_1395()
 	}
 	return Update::SUCCESS;
 }
+
+function update_1396()
+{
+	if (!DBStructure::existsTable('item-content')) {
+		return Update::SUCCESS;
+	}
+
+	if (!DBA::e("INSERT IGNORE INTO `post-content`(`uri-id`, `title`, `content-warning`, `body`, `raw-body`,
+		`location`, `coord`, `language`, `app`, `rendered-hash`, `rendered-html`,
+		`object-type`, `object`, `target-type`, `target`, `resource-id`, `plink`)
+		SELECT `item-content`.`uri-id`, `item-content`.`title`, `item-content`.`content-warning`,
+			`item-content`.`body`, `item-content`.`raw-body`, `item-content`.`location`, `item-content`.`coord`,
+			`item-content`.`language`, `item-content`.`app`, `item-content`.`rendered-hash`,
+			`item-content`.`rendered-html`, `item-content`.`object-type`, `item-content`.`object`,
+			`item-content`.`target-type`, `item-content`.`target`, `item`.`resource-id`, `item-content`.`plink`
+			FROM `item-content` INNER JOIN `item` ON `item`.`uri-id` = `item-content`.`uri-id`")) {
+		return Update::FAILED;
+	}
+	return Update::SUCCESS;
+}