File and category aren't using "term" anymore

mod/network.php

@@ -37,8 +37,8 @@ use Friendica\DI;
 use Friendica\Model\Contact;
 use Friendica\Model\Group;
 use Friendica\Model\Item;
+use Friendica\Model\Post\Category;
 use Friendica\Model\Profile;
-use Friendica\Model\Term;
 use Friendica\Module\Security\Login;
 use Friendica\Util\DateTimeFormat;
 use Friendica\Util\Proxy as ProxyUtils;
@@ -379,25 +379,25 @@ function networkFlatView(App $a, $update = 0)
 
 	networkPager($a, $pager, $update);
 
-	$item_params = ['order' => ['id' => true]];
 
 	if (strlen($file)) {
-		$term_condition = ["`term` = ? AND `otype` = ? AND `type` = ? AND `uid` = ?",
+		$item_params = ['order' => ['uri-id' => true]];
-			$file, Term::OBJECT_TYPE_POST, Term::FILE, local_user()];
+		$term_condition = ['name' => $file, 'type' => Category::FILE, 'uid' => local_user()];
 		$term_params = ['order' => ['tid' => true], 'limit' => [$pager->getStart(), $pager->getItemsPerPage()]];
-		$result = DBA::select('term', ['oid'], $term_condition, $term_params);
+		$result = DBA::select('category-view', ['uri-id'], $term_condition, $term_params);
 
 		$posts = [];
 		while ($term = DBA::fetch($result)) {
-			$posts[] = $term['oid'];
+			$posts[] = $term['uri-id'];
 		}
 		DBA::close($result);
 
 		if (count($posts) == 0) {
 			return '';
 		}
-		$item_condition = ['uid' => local_user(), 'id' => $posts];
+		$item_condition = ['uid' => local_user(), 'uri-id' => $posts];
 	} else {
+		$item_params = ['order' => ['id' => true]];
 		$item_condition = ['uid' => local_user()];
 		$item_params['limit'] = [$pager->getStart(), $pager->getItemsPerPage()];
 

src/Database/PostUpdate.php

@@ -28,7 +28,9 @@ use Friendica\Model\Contact;
 use Friendica\Model\Item;
 use Friendica\Model\ItemURI;
 use Friendica\Model\PermissionSet;
+use Friendica\Model\Post\Category;
 use Friendica\Model\Tag;
+use Friendica\Model\Term;
 use Friendica\Model\UserItem;
 use Friendica\Util\Strings;
 
@@ -72,6 +74,9 @@ class PostUpdate
 		if (!self::update1342()) {
 			return false;
 		}
+		if (!self::update1346()) {
+			return false;
+		}
 
 		return true;
 	}
@@ -667,4 +672,66 @@ class PostUpdate
 
 		return false;
 	}
+
+	/**
+	 * Fill the "tag" table with tags and mentions from the "term" table
+	 *
+	 * @return bool "true" when the job is done
+	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
+	 */
+	private static function update1346()
+	{
+		// Was the script completed?
+		if (DI::config()->get('system', 'post_update_version') >= 1346) {
+			return true;
+		}
+
+		$id = DI::config()->get('system', 'post_update_version_1346_id', 0);
+
+		Logger::info('Start', ['item' => $id]);
+
+		$rows = 0;
+
+		$terms = DBA::select('term', ['oid'],
+			["`type` IN (?, ?) AND `oid` >= ?", Category::CATEGORY, Category::FILE, $id],
+			['order' => ['oid'], 'limit' => 1000, 'group_by' => ['oid']]);
+
+		if (DBA::errorNo() != 0) {
+			Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
+			return false;
+		}
+
+		while ($term = DBA::fetch($terms)) {
+			$item = Item::selectFirst(['uri-id', 'uid'], ['id' => $term['oid']]);
+			if (!DBA::isResult($item)) {
+				continue;
+			}
+
+			$file = Term::fileTextFromItemId($term['oid']);
+			if (!empty($file)) {
+				Category::storeTextByURIId($item['uri-id'], $item['uid'], $file);
+			}
+
+			$id = $term['oid'];
+			++$rows;
+			if ($rows % 100 == 0) {
+				DI::config()->set('system', 'post_update_version_1346_id', $id);
+			}
+		}
+		DBA::close($terms);
+
+		DI::config()->set('system', 'post_update_version_1346_id', $id);
+
+		Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
+
+		// When there are less than 10 items processed this means that we reached the end
+		// The other entries will then be processed with the regular functionality
+		if ($rows < 10) {
+			DI::config()->set('system', 'post_update_version', 1346);
+			Logger::info('Done');
+			return true;
+		}
+
+		return false;
+	}
 }

src/Model/FileTag.php

@@ -23,7 +23,7 @@ namespace Friendica\Model;
 
 use Friendica\Database\DBA;
 use Friendica\DI;
-use Friendica\Model\Term;
+use Friendica\Model\Post\Category;
 
 /**
  * This class handles FileTag related functions
@@ -196,11 +196,11 @@ class FileTag
 			if ($type == 'file') {
 				$lbracket = '[';
 				$rbracket = ']';
-				$termtype = Term::FILE;
+				$termtype = Category::FILE;
 			} else {
 				$lbracket = '<';
 				$rbracket = '>';
-				$termtype = Term::CATEGORY;
+				$termtype = Category::CATEGORY;
 			}
 
 			$filetags_updated = $saved;
@@ -224,13 +224,7 @@ class FileTag
 			}
 
 			foreach ($deleted_tags as $key => $tag) {
-				$r = q("SELECT `oid` FROM `term` WHERE `term` = '%s' AND `otype` = %d AND `type` = %d AND `uid` = %d",
+				if (DBA::exists('category-view', ['name' => $tag, 'type' => $termtype, 'uid' => $uid])) {
-					DBA::escape($tag),
-					intval(Term::OBJECT_TYPE_POST),
-					intval($termtype),
-					intval($uid));
-
-				if (DBA::isResult($r)) {
 					unset($deleted_tags[$key]);
 				} else {
 					$filetags_updated = str_replace($lbracket . self::encode($tag) . $rbracket, '', $filetags_updated);
@@ -303,10 +297,10 @@ class FileTag
 
 		if ($cat == true) {
 			$pattern = '<' . self::encode($file) . '>';
-			$termtype = Term::CATEGORY;
+			$termtype = Category::CATEGORY;
 		} else {
 			$pattern = '[' . self::encode($file) . ']';
-			$termtype = Term::FILE;
+			$termtype = Category::FILE;
 		}
 
 		$item = Item::selectFirst(['file'], ['id' => $item_id, 'uid' => $uid]);
@@ -319,14 +313,7 @@ class FileTag
 
 		Item::update($fields, ['id' => $item_id]);
 
-		$r = q("SELECT `oid` FROM `term` WHERE `term` = '%s' AND `otype` = %d AND `type` = %d AND `uid` = %d",
+		if (!DBA::exists('category-view', ['name' => $file, 'type' => $termtype, 'uid' => $uid])) {
-			DBA::escape($file),
-			intval(Term::OBJECT_TYPE_POST),
-			intval($termtype),
-			intval($uid)
-		);
-
-		if (!DBA::isResult($r)) {
 			$saved = DI::pConfig()->get($uid, 'system', 'filetags');
 			DI::pConfig()->set($uid, 'system', 'filetags', str_replace($pattern, '', $saved));
 		}

src/Model/Item.php

@@ -32,6 +32,7 @@ use Friendica\Core\System;
 use Friendica\Core\Worker;
 use Friendica\Database\DBA;
 use Friendica\DI;
+use Friendica\Model\Post\Category;
 use Friendica\Protocol\Activity;
 use Friendica\Protocol\ActivityPub;
 use Friendica\Protocol\Diaspora;
@@ -319,7 +320,7 @@ class Item
 		if (!array_key_exists('verb', $row) || in_array($row['verb'], ['', Activity::POST, Activity::SHARE])) {
 			// Build the file string out of the term entries
 			if (array_key_exists('file', $row) && empty($row['file'])) {
-				$row['file'] = Term::fileTextFromItemId($row['internal-iid']);
+				$row['file'] = Category::getTextByURIId($row['internal-uri-id'], $row['internal-uid']);
 			}
 		}
 
@@ -345,7 +346,8 @@ class Item
 		// Remove internal fields
 		unset($row['internal-activity']);
 		unset($row['internal-network']);
-		unset($row['internal-iid']);
+		unset($row['internal-uri-id']);
+		unset($row['internal-uid']);
 		unset($row['internal-psid']);
 		unset($row['internal-iaid']);
 		unset($row['internal-user-ignored']);
@@ -671,7 +673,8 @@ class Item
 			'resource-id', 'event-id', 'attach', 'post-type', 'file',
 			'private', 'pubmail', 'moderated', 'visible', 'starred', 'bookmark',
 			'unseen', 'deleted', 'origin', 'forum_mode', 'mention', 'global',
-			'id' => 'item_id', 'network', 'icid', 'iaid', 'id' => 'internal-iid',
+			'id' => 'item_id', 'network', 'icid', 'iaid',
+			'uri-id' => 'internal-uri-id', 'uid' => 'internal-uid',
 			'network' => 'internal-network', 'iaid' => 'internal-iaid', 'psid' => 'internal-psid'];
 
 		if ($usermode) {
@@ -834,7 +837,7 @@ class Item
 	private static function constructSelectFields(array $fields, array $selected)
 	{
 		if (!empty($selected)) {
-			$selected = array_merge($selected, ['internal-iid', 'internal-psid', 'internal-iaid', 'internal-network']);
+			$selected = array_merge($selected, ['internal-uri-id', 'internal-uid', 'internal-psid', 'internal-iaid', 'internal-network']);
 		}
 
 		if (in_array('verb', $selected)) {
@@ -917,7 +920,7 @@ class Item
 		// We cannot simply expand the condition to check for origin entries
 		// The condition needn't to be a simple array but could be a complex condition.
 		// And we have to execute this query before the update to ensure to fetch the same data.
-		$items = DBA::select('item', ['id', 'origin', 'uri', 'uri-id', 'iaid', 'icid', 'file'], $condition);
+		$items = DBA::select('item', ['id', 'origin', 'uri', 'uri-id', 'iaid', 'icid', 'uid', 'file'], $condition);
 
 		$content_fields = [];
 		foreach (array_merge(self::CONTENT_FIELDLIST, self::MIXED_CONTENT_FIELDLIST) as $field) {
@@ -1013,7 +1016,7 @@ class Item
 			}
 
 			if (!is_null($files)) {
-				Term::insertFromFileFieldByItemId($item['id'], $files);
+				Category::storeTextByURIId($item['uri-id'], $item['uid'], $files);
 				if (!empty($item['file'])) {
 					DBA::update('item', ['file' => ''], ['id' => $item['id']]);
 				}
@@ -1099,7 +1102,7 @@ class Item
 	{
 		Logger::info('Mark item for deletion by id', ['id' => $item_id, 'callstack' => System::callstack()]);
 		// locate item to be deleted
-		$fields = ['id', 'uri', 'uid', 'parent', 'parent-uri', 'origin',
+		$fields = ['id', 'uri', 'uri-id', 'uid', 'parent', 'parent-uri', 'origin',
 			'deleted', 'file', 'resource-id', 'event-id', 'attach',
 			'verb', 'object-type', 'object', 'target', 'contact-id',
 			'icid', 'iaid', 'psid'];
@@ -1172,7 +1175,7 @@ class Item
 		$item_fields = ['deleted' => true, 'edited' => DateTimeFormat::utcNow(), 'changed' => DateTimeFormat::utcNow()];
 		DBA::update('item', $item_fields, ['id' => $item['id']]);
 
-		Term::insertFromFileFieldByItemId($item['id'], '');
+		Category::storeTextByURIId($item['uri-id'], $item['uid'], '');
 		self::deleteThread($item['id'], $item['parent-uri']);
 
 		if (!self::exists(["`uri` = ? AND `uid` != 0 AND NOT `deleted`", $item['uri']])) {
@@ -1949,7 +1952,7 @@ class Item
 		 * This is not perfect - but a workable solution until we found the reason for the problem.
 		 */
 		if (!empty($files)) {
-			Term::insertFromFileFieldByItemId($current_post, $files);
+			Category::storeTextByURIId($item['uri-id'], $item['uid'], $files);
 		}
 
 		// In that function we check if this is a forum post. Additionally we delete the item under certain circumstances

src/Model/Post/Category.php

@@ -0,0 +1,119 @@
+<?php
+/**
+ * @copyright Copyright (C) 2020, Friendica
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace Friendica\Model\Post;
+
+use Friendica\Database\DBA;
+use Friendica\Model\Item;
+use Friendica\Model\Tag;
+
+/**
+ * Class Category
+ *
+ * This Model class handles category table interactions.
+ * This tables stores user-applied categories related to posts.
+ */
+class Category
+{
+    const UNKNOWN           = 0;
+    const CATEGORY          = 3;
+    const FILE              = 5;
+
+	/**
+	 * Generates the legacy item.file field string from an item ID.
+	 * Includes only file and category terms.
+	 *
+	 * @param int $item_id
+	 * @return string
+	 * @throws \Exception
+	 */
+	public static function getTextByURIId(int $uri_id, int $uid)
+	{
+		$file_text = '';
+
+		$tags = DBA::selectToArray('post-category', ['type', 'name'], ['uri-id' => $uri_id, 'uid' => $uid]);
+		foreach ($tags as $tag) {
+			if ($tag['type'] == self::CATEGORY) {
+				$file_text .= '<' . $tag['term'] . '>';
+			} else {
+				$file_text .= '[' . $tag['term'] . ']';
+			}
+		}
+
+		return $file_text;
+	}
+
+	/**
+	 * Inserts new terms for the provided item ID based on the legacy item.file field BBCode content.
+	 * Deletes all previous file terms for the same item ID.
+	 *
+	 * @param integer $item_id item id
+	 * @param         $files
+	 * @return void
+	 * @throws \Exception
+	 */
+	public static function storeTextByURIId(int $uri_id, int $uid, string $files)
+	{
+		$message = Item::selectFirst(['deleted'], ['uri-id' => $uri_id, 'uid' => $uid]);
+		if (!DBA::isResult($message)) {
+			return;
+		}
+
+		// Clean up all tags
+		DBA::delete('post-category', ['uri-id' => $uri_id, 'uid' => $uid]);
+
+		if ($message['deleted']) {
+			return;
+		}
+
+		if (preg_match_all("/\[(.*?)\]/ism", $files, $result)) {
+			foreach ($result[1] as $file) {
+				$tagid = Tag::getID($file);
+				if (empty($tagid)) {
+					continue;
+				}
+
+				DBA::insert('post-category', [
+					'uri-id' => $uri_id,
+					'uid' => $uid,
+					'type' => self::FILE,
+					'tid' => $tagid
+				]);
+			}
+		}
+
+		if (preg_match_all("/\<(.*?)\>/ism", $files, $result)) {
+			foreach ($result[1] as $file) {
+				$tagid = Tag::getID($file);
+				if (empty($tagid)) {
+					continue;
+				}
+
+				DBA::insert('post-category', [
+					'uri-id' => $uri_id,
+					'uid' => $uid,
+					'type' => self::CATEGORY,
+					'tid' => $tagid
+				]);
+			}
+		}
+	}
+}

src/Model/Tag.php

@@ -124,22 +124,14 @@ class Tag
 		}
 
 		if (empty($cid)) {
-			$fields = ['name' => substr($name, 0, 96), 'url' => ''];
-
 			if (($type != self::HASHTAG) && !empty($url) && ($url != $name)) {
-				$fields['url'] = strtolower($url);
+				$url = strtolower($url);
-			}
-
-			$tag = DBA::selectFirst('tag', ['id'], $fields);
-			if (!DBA::isResult($tag)) {
-				DBA::insert('tag', $fields, true);
-				$tagid = DBA::lastInsertId();
 			} else {
-				$tagid = $tag['id'];
+				$url = '';
 			}
 
+			$tagid = self::getID($name, $url);
 			if (empty($tagid)) {
-				Logger::error('No tag id created', $fields);
 				return;
 			}
 		}
@@ -160,6 +152,32 @@ class Tag
 		Logger::info('Stored tag/mention', ['uri-id' => $uriid, 'tag-id' => $tagid, 'contact-id' => $cid, 'name' => $name, 'type' => $type, 'callstack' => System::callstack(8)]);
 	}
 
+	/**
+	 * Get a tag id for a given tag name and url
+	 *
+	 * @param string $name
+	 * @param string $url
+	 * @return void
+	 */
+	public static function getID(string $name, string $url = '')
+	{
+		$fields = ['name' => substr($name, 0, 96), 'url' => $url];
+
+		$tag = DBA::selectFirst('tag', ['id'], $fields);
+		if (DBA::isResult($tag)) {
+			return $tag['id'];
+		}
+
+		DBA::insert('tag', $fields, true);
+		$tid = DBA::lastInsertId();
+		if (!empty($tid)) {
+			return $tid;
+		}
+
+		Logger::error('No tag id created', $fields);
+		return 0;
+	}
+
 	/**
 	 * Store tag/mention elements
 	 *

src/Module/Profile/Status.php

@@ -29,9 +29,9 @@ use Friendica\Core\Session;
 use Friendica\Database\DBA;
 use Friendica\DI;
 use Friendica\Model\Item;
+use Friendica\Model\Post\Category;
 use Friendica\Model\Profile as ProfileModel;
 use Friendica\Model\User;
-use Friendica\Model\Term;
 use Friendica\Module\BaseProfile;
 use Friendica\Module\Security\Login;
 use Friendica\Util\DateTimeFormat;
@@ -142,8 +142,8 @@ class Status extends BaseProfile
 		$sql_post_table = "";
 
 		if (!empty($category)) {
-			$sql_post_table = sprintf("INNER JOIN (SELECT `oid` FROM `term` WHERE `term` = '%s' AND `otype` = %d AND `type` = %d AND `uid` = %d ORDER BY `tid` DESC) AS `term` ON `item`.`id` = `term`.`oid` ",
+			$sql_post_table = sprintf("INNER JOIN (SELECT `uri-id` FROM `category-view` WHERE `name` = '%s' AND `type` = %d AND `uid` = %d ORDER BY `uri-id` DESC) AS `category` ON `item`.`uri-id` = `category`.`uri-id` ",
-				DBA::escape(Strings::protectSprintf($category)), intval(Term::OBJECT_TYPE_POST), intval(Term::CATEGORY), intval($a->profile['uid']));
+				DBA::escape(Strings::protectSprintf($category)), intval(Category::CATEGORY), intval($a->profile['uid']));
 		}
 
 		if (!empty($hashtags)) {

src/Protocol/DFRN.php

@@ -39,9 +39,9 @@ use Friendica\Model\ItemURI;
 use Friendica\Model\Mail;
 use Friendica\Model\Notify\Type;
 use Friendica\Model\PermissionSet;
+use Friendica\Model\Post\Category;
 use Friendica\Model\Profile;
 use Friendica\Model\Tag;
-use Friendica\Model\Term;
 use Friendica\Model\User;
 use Friendica\Network\Probe;
 use Friendica\Util\Crypto;
@@ -241,13 +241,8 @@ class DFRN
 		}
 
 		if (isset($category)) {
-			$sql_post_table = sprintf(
+			$sql_post_table = sprintf("INNER JOIN (SELECT `uri-id` FROM `category-view` WHERE `name` = '%s' AND `type` = %d AND `uid` = %d ORDER BY `uri-id` DESC) AS `category` ON `item`.`uri-id` = `category`.`uri-id` ",
-				"INNER JOIN (SELECT `oid` FROM `term` WHERE `term` = '%s' AND `otype` = %d AND `type` = %d AND `uid` = %d ORDER BY `tid` DESC) AS `term` ON `item`.`id` = `term`.`oid` ",
+				DBA::escape(Strings::protectSprintf($category)), intval(Category::CATEGORY), intval($owner_id));
-				DBA::escape(Strings::protectSprintf($category)),
-				intval(Term::OBJECT_TYPE_POST),
-				intval(Term::CATEGORY),
-				intval($owner_id)
-			);
 		}
 
 		if ($public_feed && ! $converse) {

static/dbstructure.config.php

@@ -51,7 +51,7 @@
 use Friendica\Database\DBA;
 
 if (!defined('DB_UPDATE_VERSION')) {
-	define('DB_UPDATE_VERSION', 1343);
+	define('DB_UPDATE_VERSION', 1345);
 }
 
 return [
@@ -1306,6 +1306,19 @@ return [
 			"url" => ["url"]
 		]
 	],
+	"post-category" => [
+		"comment" => "post relation to categories",
+		"fields" => [
+			"uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "relation" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
+			"uid" => ["type" => "mediumint unsigned", "not null" => "1", "default" => "0", "relation" => ["user" => "uid"], "comment" => "User id"],
+			"type" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "primary" => "1", "comment" => ""],
+			"tid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "primary" => "1", "relation" => ["tag" => "id"], "comment" => ""],
+		],
+		"indexes" => [
+			"PRIMARY" => ["uri-id", "uid", "type", "tid"],
+			"uri-id" => ["tid"]
+		]
+	],
 	"post-tag" => [
 		"comment" => "post relation to tags",
 		"fields" => [

static/dbview.config.php

@@ -37,6 +37,21 @@
  */
 
 return [
+	"category-view" => [
+		"fields" => [
+			"uri-id" => ["post-category", "uri-id"],
+			"uid" => ["post-category", "uid"],
+			"uri" => ["item-uri", "uri"],
+			"guid" => ["item-uri", "guid"],
+			"type" => ["post-category", "type"],
+			"tid" => ["post-category", "tid"],
+			"name" => ["tag", "name"],
+			"url" => ["tag", "url"],
+		],
+		"query" => "FROM `post-category`
+			INNER JOIN `item-uri` ON `item-uri`.id = `post-category`.`uri-id`
+			LEFT JOIN `tag` ON `post-category`.`tid` = `tag`.`id`"
+	],
 	"tag-view" => [
 		"fields" => [
 			"uri-id" => ["post-tag", "uri-id"],