From 875d6b36e952781f47df2f0f31467fa476938240 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 3 May 2020 15:13:40 +0000 Subject: [PATCH 1/8] File and category aren't using "term" anymore --- mod/network.php | 14 ++-- src/Database/PostUpdate.php | 67 +++++++++++++++++++ src/Model/FileTag.php | 27 ++------ src/Model/Item.php | 21 +++--- src/Model/Post/Category.php | 119 ++++++++++++++++++++++++++++++++++ src/Model/Tag.php | 40 ++++++++---- src/Module/Profile/Status.php | 6 +- src/Protocol/DFRN.php | 11 +--- static/dbstructure.config.php | 15 ++++- static/dbview.config.php | 15 +++++ 10 files changed, 276 insertions(+), 59 deletions(-) create mode 100644 src/Model/Post/Category.php diff --git a/mod/network.php b/mod/network.php index 9ec8c95b9..b6afda608 100644 --- a/mod/network.php +++ b/mod/network.php @@ -37,8 +37,8 @@ use Friendica\DI; use Friendica\Model\Contact; use Friendica\Model\Group; use Friendica\Model\Item; +use Friendica\Model\Post\Category; use Friendica\Model\Profile; -use Friendica\Model\Term; use Friendica\Module\Security\Login; use Friendica\Util\DateTimeFormat; use Friendica\Util\Proxy as ProxyUtils; @@ -379,25 +379,25 @@ function networkFlatView(App $a, $update = 0) networkPager($a, $pager, $update); - $item_params = ['order' => ['id' => true]]; if (strlen($file)) { - $term_condition = ["`term` = ? AND `otype` = ? AND `type` = ? AND `uid` = ?", - $file, Term::OBJECT_TYPE_POST, Term::FILE, local_user()]; + $item_params = ['order' => ['uri-id' => true]]; + $term_condition = ['name' => $file, 'type' => Category::FILE, 'uid' => local_user()]; $term_params = ['order' => ['tid' => true], 'limit' => [$pager->getStart(), $pager->getItemsPerPage()]]; - $result = DBA::select('term', ['oid'], $term_condition, $term_params); + $result = DBA::select('category-view', ['uri-id'], $term_condition, $term_params); $posts = []; while ($term = DBA::fetch($result)) { - $posts[] = $term['oid']; + $posts[] = $term['uri-id']; } DBA::close($result); if (count($posts) == 0) { return ''; } - $item_condition = ['uid' => local_user(), 'id' => $posts]; + $item_condition = ['uid' => local_user(), 'uri-id' => $posts]; } else { + $item_params = ['order' => ['id' => true]]; $item_condition = ['uid' => local_user()]; $item_params['limit'] = [$pager->getStart(), $pager->getItemsPerPage()]; diff --git a/src/Database/PostUpdate.php b/src/Database/PostUpdate.php index 9ae7691d5..9bce1f1f3 100644 --- a/src/Database/PostUpdate.php +++ b/src/Database/PostUpdate.php @@ -28,7 +28,9 @@ use Friendica\Model\Contact; use Friendica\Model\Item; use Friendica\Model\ItemURI; use Friendica\Model\PermissionSet; +use Friendica\Model\Post\Category; use Friendica\Model\Tag; +use Friendica\Model\Term; use Friendica\Model\UserItem; use Friendica\Util\Strings; @@ -72,6 +74,9 @@ class PostUpdate if (!self::update1342()) { return false; } + if (!self::update1346()) { + return false; + } return true; } @@ -667,4 +672,66 @@ class PostUpdate return false; } + + /** + * Fill the "tag" table with tags and mentions from the "term" table + * + * @return bool "true" when the job is done + * @throws \Friendica\Network\HTTPException\InternalServerErrorException + */ + private static function update1346() + { + // Was the script completed? + if (DI::config()->get('system', 'post_update_version') >= 1346) { + return true; + } + + $id = DI::config()->get('system', 'post_update_version_1346_id', 0); + + Logger::info('Start', ['item' => $id]); + + $rows = 0; + + $terms = DBA::select('term', ['oid'], + ["`type` IN (?, ?) AND `oid` >= ?", Category::CATEGORY, Category::FILE, $id], + ['order' => ['oid'], 'limit' => 1000, 'group_by' => ['oid']]); + + if (DBA::errorNo() != 0) { + Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]); + return false; + } + + while ($term = DBA::fetch($terms)) { + $item = Item::selectFirst(['uri-id', 'uid'], ['id' => $term['oid']]); + if (!DBA::isResult($item)) { + continue; + } + + $file = Term::fileTextFromItemId($term['oid']); + if (!empty($file)) { + Category::storeTextByURIId($item['uri-id'], $item['uid'], $file); + } + + $id = $term['oid']; + ++$rows; + if ($rows % 100 == 0) { + DI::config()->set('system', 'post_update_version_1346_id', $id); + } + } + DBA::close($terms); + + DI::config()->set('system', 'post_update_version_1346_id', $id); + + Logger::info('Processed', ['rows' => $rows, 'last' => $id]); + + // When there are less than 10 items processed this means that we reached the end + // The other entries will then be processed with the regular functionality + if ($rows < 10) { + DI::config()->set('system', 'post_update_version', 1346); + Logger::info('Done'); + return true; + } + + return false; + } } diff --git a/src/Model/FileTag.php b/src/Model/FileTag.php index 3539f2b96..0b728e33d 100644 --- a/src/Model/FileTag.php +++ b/src/Model/FileTag.php @@ -23,7 +23,7 @@ namespace Friendica\Model; use Friendica\Database\DBA; use Friendica\DI; -use Friendica\Model\Term; +use Friendica\Model\Post\Category; /** * This class handles FileTag related functions @@ -196,11 +196,11 @@ class FileTag if ($type == 'file') { $lbracket = '['; $rbracket = ']'; - $termtype = Term::FILE; + $termtype = Category::FILE; } else { $lbracket = '<'; $rbracket = '>'; - $termtype = Term::CATEGORY; + $termtype = Category::CATEGORY; } $filetags_updated = $saved; @@ -224,13 +224,7 @@ class FileTag } foreach ($deleted_tags as $key => $tag) { - $r = q("SELECT `oid` FROM `term` WHERE `term` = '%s' AND `otype` = %d AND `type` = %d AND `uid` = %d", - DBA::escape($tag), - intval(Term::OBJECT_TYPE_POST), - intval($termtype), - intval($uid)); - - if (DBA::isResult($r)) { + if (DBA::exists('category-view', ['name' => $tag, 'type' => $termtype, 'uid' => $uid])) { unset($deleted_tags[$key]); } else { $filetags_updated = str_replace($lbracket . self::encode($tag) . $rbracket, '', $filetags_updated); @@ -303,10 +297,10 @@ class FileTag if ($cat == true) { $pattern = '<' . self::encode($file) . '>'; - $termtype = Term::CATEGORY; + $termtype = Category::CATEGORY; } else { $pattern = '[' . self::encode($file) . ']'; - $termtype = Term::FILE; + $termtype = Category::FILE; } $item = Item::selectFirst(['file'], ['id' => $item_id, 'uid' => $uid]); @@ -319,14 +313,7 @@ class FileTag Item::update($fields, ['id' => $item_id]); - $r = q("SELECT `oid` FROM `term` WHERE `term` = '%s' AND `otype` = %d AND `type` = %d AND `uid` = %d", - DBA::escape($file), - intval(Term::OBJECT_TYPE_POST), - intval($termtype), - intval($uid) - ); - - if (!DBA::isResult($r)) { + if (!DBA::exists('category-view', ['name' => $file, 'type' => $termtype, 'uid' => $uid])) { $saved = DI::pConfig()->get($uid, 'system', 'filetags'); DI::pConfig()->set($uid, 'system', 'filetags', str_replace($pattern, '', $saved)); } diff --git a/src/Model/Item.php b/src/Model/Item.php index 1ef481057..8c66e05f4 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -32,6 +32,7 @@ use Friendica\Core\System; use Friendica\Core\Worker; use Friendica\Database\DBA; use Friendica\DI; +use Friendica\Model\Post\Category; use Friendica\Protocol\Activity; use Friendica\Protocol\ActivityPub; use Friendica\Protocol\Diaspora; @@ -319,7 +320,7 @@ class Item if (!array_key_exists('verb', $row) || in_array($row['verb'], ['', Activity::POST, Activity::SHARE])) { // Build the file string out of the term entries if (array_key_exists('file', $row) && empty($row['file'])) { - $row['file'] = Term::fileTextFromItemId($row['internal-iid']); + $row['file'] = Category::getTextByURIId($row['internal-uri-id'], $row['internal-uid']); } } @@ -345,7 +346,8 @@ class Item // Remove internal fields unset($row['internal-activity']); unset($row['internal-network']); - unset($row['internal-iid']); + unset($row['internal-uri-id']); + unset($row['internal-uid']); unset($row['internal-psid']); unset($row['internal-iaid']); unset($row['internal-user-ignored']); @@ -671,7 +673,8 @@ class Item 'resource-id', 'event-id', 'attach', 'post-type', 'file', 'private', 'pubmail', 'moderated', 'visible', 'starred', 'bookmark', 'unseen', 'deleted', 'origin', 'forum_mode', 'mention', 'global', - 'id' => 'item_id', 'network', 'icid', 'iaid', 'id' => 'internal-iid', + 'id' => 'item_id', 'network', 'icid', 'iaid', + 'uri-id' => 'internal-uri-id', 'uid' => 'internal-uid', 'network' => 'internal-network', 'iaid' => 'internal-iaid', 'psid' => 'internal-psid']; if ($usermode) { @@ -834,7 +837,7 @@ class Item private static function constructSelectFields(array $fields, array $selected) { if (!empty($selected)) { - $selected = array_merge($selected, ['internal-iid', 'internal-psid', 'internal-iaid', 'internal-network']); + $selected = array_merge($selected, ['internal-uri-id', 'internal-uid', 'internal-psid', 'internal-iaid', 'internal-network']); } if (in_array('verb', $selected)) { @@ -917,7 +920,7 @@ class Item // We cannot simply expand the condition to check for origin entries // The condition needn't to be a simple array but could be a complex condition. // And we have to execute this query before the update to ensure to fetch the same data. - $items = DBA::select('item', ['id', 'origin', 'uri', 'uri-id', 'iaid', 'icid', 'file'], $condition); + $items = DBA::select('item', ['id', 'origin', 'uri', 'uri-id', 'iaid', 'icid', 'uid', 'file'], $condition); $content_fields = []; foreach (array_merge(self::CONTENT_FIELDLIST, self::MIXED_CONTENT_FIELDLIST) as $field) { @@ -1013,7 +1016,7 @@ class Item } if (!is_null($files)) { - Term::insertFromFileFieldByItemId($item['id'], $files); + Category::storeTextByURIId($item['uri-id'], $item['uid'], $files); if (!empty($item['file'])) { DBA::update('item', ['file' => ''], ['id' => $item['id']]); } @@ -1099,7 +1102,7 @@ class Item { Logger::info('Mark item for deletion by id', ['id' => $item_id, 'callstack' => System::callstack()]); // locate item to be deleted - $fields = ['id', 'uri', 'uid', 'parent', 'parent-uri', 'origin', + $fields = ['id', 'uri', 'uri-id', 'uid', 'parent', 'parent-uri', 'origin', 'deleted', 'file', 'resource-id', 'event-id', 'attach', 'verb', 'object-type', 'object', 'target', 'contact-id', 'icid', 'iaid', 'psid']; @@ -1172,7 +1175,7 @@ class Item $item_fields = ['deleted' => true, 'edited' => DateTimeFormat::utcNow(), 'changed' => DateTimeFormat::utcNow()]; DBA::update('item', $item_fields, ['id' => $item['id']]); - Term::insertFromFileFieldByItemId($item['id'], ''); + Category::storeTextByURIId($item['uri-id'], $item['uid'], ''); self::deleteThread($item['id'], $item['parent-uri']); if (!self::exists(["`uri` = ? AND `uid` != 0 AND NOT `deleted`", $item['uri']])) { @@ -1949,7 +1952,7 @@ class Item * This is not perfect - but a workable solution until we found the reason for the problem. */ if (!empty($files)) { - Term::insertFromFileFieldByItemId($current_post, $files); + Category::storeTextByURIId($item['uri-id'], $item['uid'], $files); } // In that function we check if this is a forum post. Additionally we delete the item under certain circumstances diff --git a/src/Model/Post/Category.php b/src/Model/Post/Category.php new file mode 100644 index 000000000..680e32e3a --- /dev/null +++ b/src/Model/Post/Category.php @@ -0,0 +1,119 @@ +. + * + */ + +namespace Friendica\Model\Post; + +use Friendica\Database\DBA; +use Friendica\Model\Item; +use Friendica\Model\Tag; + +/** + * Class Category + * + * This Model class handles category table interactions. + * This tables stores user-applied categories related to posts. + */ +class Category +{ + const UNKNOWN = 0; + const CATEGORY = 3; + const FILE = 5; + + /** + * Generates the legacy item.file field string from an item ID. + * Includes only file and category terms. + * + * @param int $item_id + * @return string + * @throws \Exception + */ + public static function getTextByURIId(int $uri_id, int $uid) + { + $file_text = ''; + + $tags = DBA::selectToArray('post-category', ['type', 'name'], ['uri-id' => $uri_id, 'uid' => $uid]); + foreach ($tags as $tag) { + if ($tag['type'] == self::CATEGORY) { + $file_text .= '<' . $tag['term'] . '>'; + } else { + $file_text .= '[' . $tag['term'] . ']'; + } + } + + return $file_text; + } + + /** + * Inserts new terms for the provided item ID based on the legacy item.file field BBCode content. + * Deletes all previous file terms for the same item ID. + * + * @param integer $item_id item id + * @param $files + * @return void + * @throws \Exception + */ + public static function storeTextByURIId(int $uri_id, int $uid, string $files) + { + $message = Item::selectFirst(['deleted'], ['uri-id' => $uri_id, 'uid' => $uid]); + if (!DBA::isResult($message)) { + return; + } + + // Clean up all tags + DBA::delete('post-category', ['uri-id' => $uri_id, 'uid' => $uid]); + + if ($message['deleted']) { + return; + } + + if (preg_match_all("/\[(.*?)\]/ism", $files, $result)) { + foreach ($result[1] as $file) { + $tagid = Tag::getID($file); + if (empty($tagid)) { + continue; + } + + DBA::insert('post-category', [ + 'uri-id' => $uri_id, + 'uid' => $uid, + 'type' => self::FILE, + 'tid' => $tagid + ]); + } + } + + if (preg_match_all("/\<(.*?)\>/ism", $files, $result)) { + foreach ($result[1] as $file) { + $tagid = Tag::getID($file); + if (empty($tagid)) { + continue; + } + + DBA::insert('post-category', [ + 'uri-id' => $uri_id, + 'uid' => $uid, + 'type' => self::CATEGORY, + 'tid' => $tagid + ]); + } + } + } +} diff --git a/src/Model/Tag.php b/src/Model/Tag.php index 0a06dcc25..9b4b1eb41 100644 --- a/src/Model/Tag.php +++ b/src/Model/Tag.php @@ -124,22 +124,14 @@ class Tag } if (empty($cid)) { - $fields = ['name' => substr($name, 0, 96), 'url' => '']; - if (($type != self::HASHTAG) && !empty($url) && ($url != $name)) { - $fields['url'] = strtolower($url); - } - - $tag = DBA::selectFirst('tag', ['id'], $fields); - if (!DBA::isResult($tag)) { - DBA::insert('tag', $fields, true); - $tagid = DBA::lastInsertId(); + $url = strtolower($url); } else { - $tagid = $tag['id']; + $url = ''; } + $tagid = self::getID($name, $url); if (empty($tagid)) { - Logger::error('No tag id created', $fields); return; } } @@ -160,6 +152,32 @@ class Tag Logger::info('Stored tag/mention', ['uri-id' => $uriid, 'tag-id' => $tagid, 'contact-id' => $cid, 'name' => $name, 'type' => $type, 'callstack' => System::callstack(8)]); } + /** + * Get a tag id for a given tag name and url + * + * @param string $name + * @param string $url + * @return void + */ + public static function getID(string $name, string $url = '') + { + $fields = ['name' => substr($name, 0, 96), 'url' => $url]; + + $tag = DBA::selectFirst('tag', ['id'], $fields); + if (DBA::isResult($tag)) { + return $tag['id']; + } + + DBA::insert('tag', $fields, true); + $tid = DBA::lastInsertId(); + if (!empty($tid)) { + return $tid; + } + + Logger::error('No tag id created', $fields); + return 0; + } + /** * Store tag/mention elements * diff --git a/src/Module/Profile/Status.php b/src/Module/Profile/Status.php index 685ab4e07..75ba60cb2 100644 --- a/src/Module/Profile/Status.php +++ b/src/Module/Profile/Status.php @@ -29,9 +29,9 @@ use Friendica\Core\Session; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\Item; +use Friendica\Model\Post\Category; use Friendica\Model\Profile as ProfileModel; use Friendica\Model\User; -use Friendica\Model\Term; use Friendica\Module\BaseProfile; use Friendica\Module\Security\Login; use Friendica\Util\DateTimeFormat; @@ -142,8 +142,8 @@ class Status extends BaseProfile $sql_post_table = ""; if (!empty($category)) { - $sql_post_table = sprintf("INNER JOIN (SELECT `oid` FROM `term` WHERE `term` = '%s' AND `otype` = %d AND `type` = %d AND `uid` = %d ORDER BY `tid` DESC) AS `term` ON `item`.`id` = `term`.`oid` ", - DBA::escape(Strings::protectSprintf($category)), intval(Term::OBJECT_TYPE_POST), intval(Term::CATEGORY), intval($a->profile['uid'])); + $sql_post_table = sprintf("INNER JOIN (SELECT `uri-id` FROM `category-view` WHERE `name` = '%s' AND `type` = %d AND `uid` = %d ORDER BY `uri-id` DESC) AS `category` ON `item`.`uri-id` = `category`.`uri-id` ", + DBA::escape(Strings::protectSprintf($category)), intval(Category::CATEGORY), intval($a->profile['uid'])); } if (!empty($hashtags)) { diff --git a/src/Protocol/DFRN.php b/src/Protocol/DFRN.php index 5060a7d56..47272bf6c 100644 --- a/src/Protocol/DFRN.php +++ b/src/Protocol/DFRN.php @@ -39,9 +39,9 @@ use Friendica\Model\ItemURI; use Friendica\Model\Mail; use Friendica\Model\Notify\Type; use Friendica\Model\PermissionSet; +use Friendica\Model\Post\Category; use Friendica\Model\Profile; use Friendica\Model\Tag; -use Friendica\Model\Term; use Friendica\Model\User; use Friendica\Network\Probe; use Friendica\Util\Crypto; @@ -241,13 +241,8 @@ class DFRN } if (isset($category)) { - $sql_post_table = sprintf( - "INNER JOIN (SELECT `oid` FROM `term` WHERE `term` = '%s' AND `otype` = %d AND `type` = %d AND `uid` = %d ORDER BY `tid` DESC) AS `term` ON `item`.`id` = `term`.`oid` ", - DBA::escape(Strings::protectSprintf($category)), - intval(Term::OBJECT_TYPE_POST), - intval(Term::CATEGORY), - intval($owner_id) - ); + $sql_post_table = sprintf("INNER JOIN (SELECT `uri-id` FROM `category-view` WHERE `name` = '%s' AND `type` = %d AND `uid` = %d ORDER BY `uri-id` DESC) AS `category` ON `item`.`uri-id` = `category`.`uri-id` ", + DBA::escape(Strings::protectSprintf($category)), intval(Category::CATEGORY), intval($owner_id)); } if ($public_feed && ! $converse) { diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 2c3a9fb5f..788ff7d13 100755 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -51,7 +51,7 @@ use Friendica\Database\DBA; if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1343); + define('DB_UPDATE_VERSION', 1345); } return [ @@ -1306,6 +1306,19 @@ return [ "url" => ["url"] ] ], + "post-category" => [ + "comment" => "post relation to categories", + "fields" => [ + "uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "relation" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"], + "uid" => ["type" => "mediumint unsigned", "not null" => "1", "default" => "0", "relation" => ["user" => "uid"], "comment" => "User id"], + "type" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "primary" => "1", "comment" => ""], + "tid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "primary" => "1", "relation" => ["tag" => "id"], "comment" => ""], + ], + "indexes" => [ + "PRIMARY" => ["uri-id", "uid", "type", "tid"], + "uri-id" => ["tid"] + ] + ], "post-tag" => [ "comment" => "post relation to tags", "fields" => [ diff --git a/static/dbview.config.php b/static/dbview.config.php index 0f0d69108..fa302b866 100755 --- a/static/dbview.config.php +++ b/static/dbview.config.php @@ -37,6 +37,21 @@ */ return [ + "category-view" => [ + "fields" => [ + "uri-id" => ["post-category", "uri-id"], + "uid" => ["post-category", "uid"], + "uri" => ["item-uri", "uri"], + "guid" => ["item-uri", "guid"], + "type" => ["post-category", "type"], + "tid" => ["post-category", "tid"], + "name" => ["tag", "name"], + "url" => ["tag", "url"], + ], + "query" => "FROM `post-category` + INNER JOIN `item-uri` ON `item-uri`.id = `post-category`.`uri-id` + LEFT JOIN `tag` ON `post-category`.`tid` = `tag`.`id`" + ], "tag-view" => [ "fields" => [ "uri-id" => ["post-tag", "uri-id"], From 4fa26c81e895abfcaffd19e66be9ba28d2dbbfda Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 3 May 2020 15:44:39 +0000 Subject: [PATCH 2/8] Fix for wrong table and wrong field name --- src/Model/Post/Category.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Model/Post/Category.php b/src/Model/Post/Category.php index 680e32e3a..d4a8469d4 100644 --- a/src/Model/Post/Category.php +++ b/src/Model/Post/Category.php @@ -49,12 +49,12 @@ class Category { $file_text = ''; - $tags = DBA::selectToArray('post-category', ['type', 'name'], ['uri-id' => $uri_id, 'uid' => $uid]); + $tags = DBA::selectToArray('category-view', ['type', 'name'], ['uri-id' => $uri_id, 'uid' => $uid]); foreach ($tags as $tag) { if ($tag['type'] == self::CATEGORY) { - $file_text .= '<' . $tag['term'] . '>'; + $file_text .= '<' . $tag['name'] . '>'; } else { - $file_text .= '[' . $tag['term'] . ']'; + $file_text .= '[' . $tag['name'] . ']'; } } From d5edc9e3aeb82006c17fe305cc9d507b5976197b Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 3 May 2020 16:02:51 +0000 Subject: [PATCH 3/8] Fix tests --- tests/datasets/api.fixture.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/datasets/api.fixture.php b/tests/datasets/api.fixture.php index c9c16a33f..efcb32bec 100644 --- a/tests/datasets/api.fixture.php +++ b/tests/datasets/api.fixture.php @@ -103,6 +103,7 @@ return [ 'item' => [ [ 'id' => 1, + 'uri-id' => 1, 'visible' => 1, 'contact-id' => 42, 'author-id' => 42, @@ -123,6 +124,7 @@ return [ ], [ 'id' => 2, + 'uri-id' => 2, 'visible' => 1, 'contact-id' => 42, 'author-id' => 42, @@ -140,6 +142,7 @@ return [ [ 'id' => 3, + 'uri-id' => 3, 'visible' => 1, 'contact-id' => 43, 'author-id' => 43, @@ -156,6 +159,7 @@ return [ ], [ 'id' => 4, + 'uri-id' => 4, 'visible' => 1, 'contact-id' => 44, 'author-id' => 44, @@ -173,6 +177,7 @@ return [ [ 'id' => 5, + 'uri-id' => 5, 'visible' => 1, 'contact-id' => 42, 'author-id' => 42, @@ -193,6 +198,7 @@ return [ ], [ 'id' => 6, + 'uri-id' => 6, 'visible' => 1, 'contact-id' => 44, 'author-id' => 44, From b261df14af642fc1225df0153dbc9967688e5108 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 4 May 2020 15:23:45 +0000 Subject: [PATCH 4/8] Updated database versions --- database.sql | 55 +++++++++++++++++++++++++++++++++-- static/dbstructure.config.php | 2 +- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/database.sql b/database.sql index a68f3d4e5..3ae2374fd 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 2020.06-dev (Red Hot Poker) --- DB_UPDATE_VERSION 1341 +-- DB_UPDATE_VERSION 1346 -- ------------------------------------------ @@ -666,7 +666,8 @@ CREATE TABLE IF NOT EXISTS `item` ( INDEX `uid_eventid` (`uid`,`event-id`), INDEX `icid` (`icid`), INDEX `iaid` (`iaid`), - INDEX `psid_wall` (`psid`,`wall`) + INDEX `psid_wall` (`psid`,`wall`), + INDEX `uri-id` (`uri-id`) ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Structure for all posts'; -- @@ -1187,6 +1188,18 @@ CREATE TABLE IF NOT EXISTS `tag` ( INDEX `url` (`url`) ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='tags and mentions'; +-- +-- TABLE post-category +-- +CREATE TABLE IF NOT EXISTS `post-category` ( + `uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri', + `uid` mediumint unsigned NOT NULL DEFAULT 0 COMMENT 'User id', + `type` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '', + `tid` int unsigned NOT NULL DEFAULT 0 COMMENT '', + PRIMARY KEY(`uri-id`,`uid`,`type`,`tid`), + INDEX `uri-id` (`tid`) +) DEFAULT COLLATE utf8mb4_general_ci COMMENT='post relation to categories'; + -- -- TABLE post-tag -- @@ -1386,6 +1399,23 @@ CREATE TABLE IF NOT EXISTS `storage` ( PRIMARY KEY(`id`) ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Data stored by Database storage backend'; +-- +-- VIEW category-view +-- +DROP VIEW IF EXISTS `category-view`; +CREATE VIEW `category-view` AS SELECT + `post-category`.`uri-id` AS `uri-id`, + `post-category`.`uid` AS `uid`, + `item-uri`.`uri` AS `uri`, + `item-uri`.`guid` AS `guid`, + `post-category`.`type` AS `type`, + `post-category`.`tid` AS `tid`, + `tag`.`name` AS `name`, + `tag`.`url` AS `url` + FROM `post-category` + INNER JOIN `item-uri` ON `item-uri`.id = `post-category`.`uri-id` + LEFT JOIN `tag` ON `post-category`.`tid` = `tag`.`id`; + -- -- VIEW tag-view -- @@ -1577,6 +1607,27 @@ CREATE VIEW `pending-view` AS SELECT INNER JOIN `contact` ON `register`.`uid` = `contact`.`uid` INNER JOIN `user` ON `register`.`uid` = `user`.`uid`; +-- +-- VIEW tag-search-view +-- +DROP VIEW IF EXISTS `tag-search-view`; +CREATE VIEW `tag-search-view` AS SELECT + `post-tag`.`uri-id` AS `uri-id`, + `item`.`id` AS `iid`, + `item`.`uri` AS `uri`, + `item`.`guid` AS `guid`, + `item`.`uid` AS `uid`, + `item`.`private` AS `private`, + `item`.`wall` AS `wall`, + `item`.`origin` AS `origin`, + `item`.`gravity` AS `gravity`, + `item`.`received` AS `received`, + `tag`.`name` AS `name` + FROM `post-tag` + INNER JOIN `tag` ON `tag`.`id` = `post-tag`.`tid` + INNER JOIN `item` ON `item`.`uri-id` = `post-tag`.`uri-id` + WHERE `post-tag`.`type` = 1; + -- -- VIEW workerqueue-view -- diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 788ff7d13..a9b0682ef 100755 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -51,7 +51,7 @@ use Friendica\Database\DBA; if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1345); + define('DB_UPDATE_VERSION', 1346); } return [ From b7e25d16e7a617b94d7a348f395e61eb3f244bad Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 5 May 2020 05:11:59 +0000 Subject: [PATCH 5/8] Removed usage of "tag" field --- mod/photos.php | 18 ++--------- src/Model/Contact.php | 4 +-- src/Protocol/ActivityPub/Processor.php | 42 +------------------------- src/Protocol/DFRN.php | 2 +- src/Protocol/OStatus.php | 11 +------ src/Worker/Notifier.php | 16 ++++------ 6 files changed, 13 insertions(+), 80 deletions(-) diff --git a/mod/photos.php b/mod/photos.php index 50210d686..8f968d0c0 100644 --- a/mod/photos.php +++ b/mod/photos.php @@ -425,13 +425,11 @@ function photos_post(App $a) $item = Item::selectFirst(['tag', 'inform', 'uri-id'], ['id' => $item_id, 'uid' => $page_owner_uid]); if (DBA::isResult($item)) { - $old_tag = $item['tag']; $old_inform = $item['inform']; } } if (strlen($rawtags)) { - $str_tags = ''; $inform = ''; // if the new tag doesn't have a namespace specifier (@foo or #foo) give it a hashtag @@ -513,15 +511,10 @@ function photos_post(App $a) if (!empty($contact)) { $taginfo[] = [$newname, $profile, $notify, $contact, '@[url=' . str_replace(',', '%2c', $profile) . ']' . $newname . '[/url]']; } else { - $taginfo[] = [$newname, $profile, $notify, null, $str_tags .= '@[url=' . $profile . ']' . $newname . '[/url]']; - } - - if (strlen($str_tags)) { - $str_tags .= ','; + $taginfo[] = [$newname, $profile, $notify, null, '@[url=' . $profile . ']' . $newname . '[/url]']; } $profile = str_replace(',', '%2c', $profile); - $str_tags .= '@[url=' . $profile . ']' . $newname . '[/url]'; if (!empty($item['uri-id'])) { Tag::store($item['uri-id'], Tag::MENTION, $newname, $profile); @@ -529,7 +522,6 @@ function photos_post(App $a) } } elseif (strpos($tag, '#') === 0) { $tagname = substr($tag, 1); - $str_tags .= '#[url=' . DI::baseUrl() . "/search?tag=" . $tagname . ']' . $tagname . '[/url],'; if (!empty($item['uri-id'])) { Tag::store($item['uri-id'], Tag::HASHTAG, $tagname); } @@ -537,19 +529,13 @@ function photos_post(App $a) } } - $newtag = $old_tag ?? ''; - if (strlen($newtag) && strlen($str_tags)) { - $newtag .= ','; - } - $newtag .= $str_tags; - $newinform = $old_inform ?? ''; if (strlen($newinform) && strlen($inform)) { $newinform .= ','; } $newinform .= $inform; - $fields = ['tag' => $newtag, 'inform' => $newinform, 'edited' => DateTimeFormat::utcNow(), 'changed' => DateTimeFormat::utcNow()]; + $fields = ['inform' => $newinform, 'edited' => DateTimeFormat::utcNow(), 'changed' => DateTimeFormat::utcNow()]; $condition = ['id' => $item_id]; Item::update($fields, $condition); diff --git a/src/Model/Contact.php b/src/Model/Contact.php index becc80c0c..0d321189f 100644 --- a/src/Model/Contact.php +++ b/src/Model/Contact.php @@ -847,7 +847,7 @@ class Contact $item['body'] = ''; $item['title'] = ''; $item['guid'] = ''; - $item['tag'] = ''; + $item['uri-id'] = 0; $item['attach'] = ''; $slap = OStatus::salmon($item, $user); @@ -2457,7 +2457,7 @@ class Contact $item['body'] = ''; $item['title'] = ''; $item['guid'] = ''; - $item['tag'] = ''; + $item['uri-id'] = 0; $item['attach'] = ''; $slap = OStatus::salmon($item, $owner); diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index 479d8d55c..197fb2baa 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -79,35 +79,6 @@ class Processor return $body; } - /** - * Constructs a string with tags for a given tag array - * - * @param array $tags - * @param boolean $sensitive - * @return string with tags - */ - private static function constructTagString(array $tags = null, $sensitive = false) - { - if (empty($tags)) { - return ''; - } - - $tag_text = ''; - foreach ($tags as $tag) { - if (in_array($tag['type'] ?? '', ['Mention', 'Hashtag'])) { - if (!empty($tag_text)) { - $tag_text .= ','; - } - - $tag_text .= substr($tag['name'], 0, 1) . '[url=' . $tag['href'] . ']' . substr($tag['name'], 1) . '[/url]'; - } - } - - /// @todo add nsfw for $sensitive - - return $tag_text; - } - /** * Add attachment data to the item array * @@ -263,16 +234,7 @@ class Processor } Tag::store($item['uri-id'], Tag::HASHTAG, $activity['object_content'], $activity['object_id']); - - // To-Do: - // - Check if "blocktag" is set - // - Check if actor is a contact - - if (!stristr($item['tag'], trim($activity['object_content']))) { - $tag = $item['tag'] . (strlen($item['tag']) ? ',' : '') . '#[url=' . $activity['object_id'] . ']'. $activity['object_content'] . '[/url]'; - Item::update(['tag' => $tag], ['id' => $item['id']]); - Logger::info('Tagged item', ['id' => $item['id'], 'tag' => $activity['object_content'], 'uri' => $activity['target_id'], 'actor' => $activity['actor']]); - } + Logger::info('Tagged item', ['id' => $item['id'], 'tag' => $activity['object_content'], 'uri' => $activity['target_id'], 'actor' => $activity['actor']]); } } @@ -405,8 +367,6 @@ class Processor $item['body'] = $content; } - $item['tag'] = self::constructTagString($activity['tags'], $activity['sensitive']); - self::storeFromBody($item); self::storeTags($item['uri-id'], $activity['tags']); diff --git a/src/Protocol/DFRN.php b/src/Protocol/DFRN.php index 5060a7d56..74352906d 100644 --- a/src/Protocol/DFRN.php +++ b/src/Protocol/DFRN.php @@ -2001,7 +2001,7 @@ class DFRN } $fields = ['title' => $item['title'] ?? '', 'body' => $item['body'] ?? '', - 'tag' => $item['tag'] ?? '', 'changed' => DateTimeFormat::utcNow(), + 'changed' => DateTimeFormat::utcNow(), 'edited' => DateTimeFormat::utc($item["edited"])]; $condition = ["`uri` = ? AND `uid` IN (0, ?)", $item["uri"], $importer["importer_uid"]]; diff --git a/src/Protocol/OStatus.php b/src/Protocol/OStatus.php index 8ab14c078..6eeadb46c 100644 --- a/src/Protocol/OStatus.php +++ b/src/Protocol/OStatus.php @@ -655,17 +655,8 @@ class OStatus foreach ($categories as $category) { foreach ($category->attributes as $attributes) { if ($attributes->name == 'term') { - $term = $attributes->textContent; - if (!empty($item['tag'])) { - $item['tag'] .= ','; - } else { - $item['tag'] = ''; - } - - $item['tag'] .= '#[url=' . DI::baseUrl() . '/search?tag=' . $term . ']' . $term . '[/url]'; - // Store the hashtag - Tag::store($item['uri-id'], Tag::HASHTAG, $term); + Tag::store($item['uri-id'], Tag::HASHTAG, $attributes->textContent); } } } diff --git a/src/Worker/Notifier.php b/src/Worker/Notifier.php index 35a228fce..09470c318 100644 --- a/src/Worker/Notifier.php +++ b/src/Worker/Notifier.php @@ -34,6 +34,7 @@ use Friendica\Model\Group; use Friendica\Model\Item; use Friendica\Model\ItemDeliveryData; use Friendica\Model\PushSubscriber; +use Friendica\Model\Tag; use Friendica\Model\User; use Friendica\Network\Probe; use Friendica\Protocol\ActivityPub; @@ -367,16 +368,11 @@ class Notifier } // Send a salmon notification to every person we mentioned in the post - $arr = explode(',',$target_item['tag']); - foreach ($arr as $x) { - //Logger::log('Checking tag '.$x, Logger::DEBUG); - $matches = null; - if (preg_match('/@\[url=([^\]]*)\]/',$x,$matches)) { - $probed_contact = Probe::uri($matches[1]); - if ($probed_contact["notify"] != "") { - Logger::log('Notify mentioned user '.$probed_contact["url"].': '.$probed_contact["notify"]); - $url_recipients[$probed_contact["notify"]] = $probed_contact["notify"]; - } + foreach (Tag::getByURIId($target_item['uri-id'], [Tag::MENTION, Tag::EXCLUSIVE_MENTION, Tag::IMPLICIT_MENTION]) as $tag) { + $probed_contact = Probe::uri($tag['url']); + if ($probed_contact["notify"] != "") { + Logger::log('Notify mentioned user '.$probed_contact["url"].': '.$probed_contact["notify"]); + $url_recipients[$probed_contact["notify"]] = $probed_contact["notify"]; } } From 57257840553a2f91c85e4ad37143fffd50b1cc65 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 5 May 2020 15:47:35 +0000 Subject: [PATCH 6/8] Update database definition, cleaned code --- database.sql | 3 +++ src/Database/PostUpdate.php | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/database.sql b/database.sql index 5f6fc2932..a26b6f2bf 100644 --- a/database.sql +++ b/database.sql @@ -815,6 +815,8 @@ CREATE TABLE IF NOT EXISTS `notify` ( `link` varchar(255) NOT NULL DEFAULT '' COMMENT '', `iid` int unsigned NOT NULL DEFAULT 0 COMMENT 'item.id', `parent` int unsigned NOT NULL DEFAULT 0 COMMENT '', + `uri-id` int unsigned COMMENT 'Item-uri id of the related post', + `parent-uri-id` int unsigned COMMENT 'Item-uri id of the parent of the related post', `seen` boolean NOT NULL DEFAULT '0' COMMENT '', `verb` varchar(100) NOT NULL DEFAULT '' COMMENT '', `otype` varchar(10) NOT NULL DEFAULT '' COMMENT '', @@ -833,6 +835,7 @@ CREATE TABLE IF NOT EXISTS `notify-threads` ( `id` int unsigned NOT NULL auto_increment COMMENT 'sequential ID', `notify-id` int unsigned NOT NULL DEFAULT 0 COMMENT '', `master-parent-item` int unsigned NOT NULL DEFAULT 0 COMMENT '', + `master-parent-uri-id` int unsigned COMMENT 'Item-uri id of the parent of the related post', `parent-item` int unsigned NOT NULL DEFAULT 0 COMMENT '', `receiver-uid` mediumint unsigned NOT NULL DEFAULT 0 COMMENT 'User id', PRIMARY KEY(`id`) diff --git a/src/Database/PostUpdate.php b/src/Database/PostUpdate.php index 25cf99142..514956b22 100644 --- a/src/Database/PostUpdate.php +++ b/src/Database/PostUpdate.php @@ -688,6 +688,7 @@ class PostUpdate } $id = DI::config()->get('system', 'post_update_version_1345_id', 0); + Logger::info('Start', ['item' => $id]); $rows = 0; @@ -726,8 +727,6 @@ class PostUpdate return false; } - - /** * Fill the "tag" table with tags and mentions from the "term" table * From f67c8a73153a09826feda49a9c5fca030c7c1a19 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 5 May 2020 16:42:29 +0000 Subject: [PATCH 7/8] Fix test --- tests/include/ApiTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/include/ApiTest.php b/tests/include/ApiTest.php index 145619a75..e4443404e 100644 --- a/tests/include/ApiTest.php +++ b/tests/include/ApiTest.php @@ -3849,7 +3849,7 @@ class ApiTest extends DatabaseTest $assertXml=<< - + XML; $this->assertXmlStringEqualsXmlString($assertXml, $result); From 349e1d20151304ec31283ff8f04c1feb28f36da8 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 5 May 2020 17:32:11 +0000 Subject: [PATCH 8/8] Tests could work again --- tests/include/ApiTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/include/ApiTest.php b/tests/include/ApiTest.php index e4443404e..9970ced24 100644 --- a/tests/include/ApiTest.php +++ b/tests/include/ApiTest.php @@ -3849,7 +3849,7 @@ class ApiTest extends DatabaseTest $assertXml=<< - + XML; $this->assertXmlStringEqualsXmlString($assertXml, $result);