keith/friendica
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #10396 from annando/public-timeline

Browse source 
API: improved speed of the public timeline
This commit is contained in:
Hypolite Petovan 2021-06-16 17:42:24 -04:00 committed by GitHub
commit 5419d1d426
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 223 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

64
database.sql
View file

@ -1,6 +1,6 @@
-- ------------------------------------------
-- Friendica 2021.06-rc (Siberian Iris)
-- DB_UPDATE_VERSION 1421
-- DB_UPDATE_VERSION 1422
-- ------------------------------------------
@ -1874,7 +1874,9 @@ CREATE VIEW `post-thread-user-view` AS SELECT
`parent-post`.`author-id` AS `parent-author-id`,
`parent-post-author`.`url` AS `parent-author-link`,
`parent-post-author`.`name` AS `parent-author-name`,
`parent-post-author`.`network` AS `parent-author-network`
`parent-post-author`.`network` AS `parent-author-network`,
`parent-post-author`.`blocked` AS `parent-author-blocked`,
`parent-post-author`.`hidden` AS `parent-author-hidden`
FROM `post-thread-user`
INNER JOIN `post-user` ON `post-user`.`id` = `post-thread-user`.`post-user-id`
STRAIGHT_JOIN `contact` ON `contact`.`id` = `post-thread-user`.`contact-id`
@ -1938,6 +1940,31 @@ CREATE VIEW `post-view` AS SELECT
`post-content`.`target-type` AS `target-type`,
`post-content`.`target` AS `target`,
`post-content`.`resource-id` AS `resource-id`,
`post`.`author-id` AS `contact-id`,
`author`.`url` AS `contact-link`,
`author`.`addr` AS `contact-addr`,
`author`.`name` AS `contact-name`,
`author`.`nick` AS `contact-nick`,
`author`.`thumb` AS `contact-avatar`,
`author`.`network` AS `contact-network`,
`author`.`blocked` AS `contact-blocked`,
`author`.`hidden` AS `contact-hidden`,
`author`.`readonly` AS `contact-readonly`,
`author`.`archive` AS `contact-archive`,
`author`.`pending` AS `contact-pending`,
`author`.`rel` AS `contact-rel`,
`author`.`uid` AS `contact-uid`,
`author`.`contact-type` AS `contact-contact-type`,
IF (`post`.`network` IN ('apub', 'dfrn', 'dspr', 'stat'), true, `author`.`writable`) AS `writable`,
false AS `self`,
`author`.`id` AS `cid`,
`author`.`alias` AS `alias`,
`author`.`photo` AS `photo`,
`author`.`name-date` AS `name-date`,
`author`.`uri-date` AS `uri-date`,
`author`.`avatar-date` AS `avatar-date`,
`author`.`thumb` AS `thumb`,
`author`.`dfrn-id` AS `dfrn-id`,
`post`.`author-id` AS `author-id`,
`author`.`url` AS `author-link`,
`author`.`addr` AS `author-addr`,
@ -1973,7 +2000,9 @@ CREATE VIEW `post-view` AS SELECT
`parent-post`.`author-id` AS `parent-author-id`,
`parent-post-author`.`url` AS `parent-author-link`,
`parent-post-author`.`name` AS `parent-author-name`,
`parent-post-author`.`network` AS `parent-author-network`
`parent-post-author`.`network` AS `parent-author-network`,
`parent-post-author`.`blocked` AS `parent-author-blocked`,
`parent-post-author`.`hidden` AS `parent-author-hidden`
FROM `post`
STRAIGHT_JOIN `post-thread` ON `post-thread`.`uri-id` = `post`.`parent-uri-id`
STRAIGHT_JOIN `contact` AS `author` ON `author`.`id` = `post`.`author-id`
@ -2033,6 +2062,31 @@ CREATE VIEW `post-thread-view` AS SELECT
`post-content`.`target-type` AS `target-type`,
`post-content`.`target` AS `target`,
`post-content`.`resource-id` AS `resource-id`,
`post-thread`.`author-id` AS `contact-id`,
`author`.`url` AS `contact-link`,
`author`.`addr` AS `contact-addr`,
`author`.`name` AS `contact-name`,
`author`.`nick` AS `contact-nick`,
`author`.`thumb` AS `contact-avatar`,
`author`.`network` AS `contact-network`,
`author`.`blocked` AS `contact-blocked`,
`author`.`hidden` AS `contact-hidden`,
`author`.`readonly` AS `contact-readonly`,
`author`.`archive` AS `contact-archive`,
`author`.`pending` AS `contact-pending`,
`author`.`rel` AS `contact-rel`,
`author`.`uid` AS `contact-uid`,
`author`.`contact-type` AS `contact-contact-type`,
IF (`post`.`network` IN ('apub', 'dfrn', 'dspr', 'stat'), true, `author`.`writable`) AS `writable`,
false AS `self`,
`author`.`id` AS `cid`,
`author`.`alias` AS `alias`,
`author`.`photo` AS `photo`,
`author`.`name-date` AS `name-date`,
`author`.`uri-date` AS `uri-date`,
`author`.`avatar-date` AS `avatar-date`,
`author`.`thumb` AS `thumb`,
`author`.`dfrn-id` AS `dfrn-id`,
`post-thread`.`author-id` AS `author-id`,
`author`.`url` AS `author-link`,
`author`.`addr` AS `author-addr`,
@ -2068,7 +2122,9 @@ CREATE VIEW `post-thread-view` AS SELECT
`parent-post`.`author-id` AS `parent-author-id`,
`parent-post-author`.`url` AS `parent-author-link`,
`parent-post-author`.`name` AS `parent-author-name`,
`parent-post-author`.`network` AS `parent-author-network`
`parent-post-author`.`network` AS `parent-author-network`,
`parent-post-author`.`blocked` AS `parent-author-blocked`,
`parent-post-author`.`hidden` AS `parent-author-hidden`
FROM `post-thread`
INNER JOIN `post` ON `post`.`uri-id` = `post-thread`.`uri-id`
STRAIGHT_JOIN `contact` AS `author` ON `author`.`id` = `post-thread`.`author-id`

6
src/Factory/Api/Mastodon/Status.php
View file

@ -72,9 +72,9 @@ class Status extends BaseFactory
$account = DI::mstdnAccount()->createFromContactId($item['author-id']);
$counts = new \Friendica\Object\Api\Mastodon\Status\Counts(
Post::count(['thr-parent-id' => $uriId, 'gravity' => GRAVITY_COMMENT, 'deleted' => false], [], false),
Post::count(['thr-parent-id' => $uriId, 'gravity' => GRAVITY_ACTIVITY, 'vid' => Verb::getID(Activity::ANNOUNCE), 'deleted' => false], [], false),
Post::count(['thr-parent-id' => $uriId, 'gravity' => GRAVITY_ACTIVITY, 'vid' => Verb::getID(Activity::LIKE), 'deleted' => false], [], false)
Post::countPosts(['thr-parent-id' => $uriId, 'gravity' => GRAVITY_COMMENT, 'deleted' => false], []),
Post::countPosts(['thr-parent-id' => $uriId, 'gravity' => GRAVITY_ACTIVITY, 'vid' => Verb::getID(Activity::ANNOUNCE), 'deleted' => false], []),
Post::countPosts(['thr-parent-id' => $uriId, 'gravity' => GRAVITY_ACTIVITY, 'vid' => Verb::getID(Activity::LIKE), 'deleted' => false], [])
);
$userAttributes = new \Friendica\Object\Api\Mastodon\Status\UserAttributes(

120
src/Model/Post.php
View file

@ -124,24 +124,22 @@ class Post
}
/**
* Check if post data exists
* Check if post-user-view records exists
*
* @param array $condition array of fields for condition
* @param bool $user_mode true = post-user-view, false = post-view
*
* @return boolean Are there rows for that condition?
* @throws \Exception
*/
public static function exists($condition, bool $user_mode = true) {
return DBA::exists($user_mode ? 'post-user-view' : 'post-view', $condition);
public static function exists($condition) {
return DBA::exists('post-user-view', $condition);
}
/**
* Counts the posts satisfying the provided condition
* Counts the post-user-view records satisfying the provided condition
*
* @param array $condition array of fields for condition
* @param array $params Array of several parameters
* @param bool $user_mode true = post-user-view, false = post-view
*
* @return int
*
@ -153,13 +151,34 @@ class Post
* $count = Post::count($condition);
* @throws \Exception
*/
public static function count(array $condition = [], array $params = [], bool $user_mode = true)
public static function count(array $condition = [], array $params = [])
{
return DBA::count($user_mode ? 'post-user-view' : 'post-view', $condition, $params);
return DBA::count('post-user-view', $condition, $params);
}
/**
* Retrieve a single record from the post table and returns it in an associative array
* Counts the post-view records satisfying the provided condition
*
* @param array $condition array of fields for condition
* @param array $params Array of several parameters
*
* @return int
*
* Example:
* $condition = ["network" => 'dspr'];
* or:
* $condition = ["`network` IN (?, ?)", 1, 'dfrn', 'dspr'];
*
* $count = Post::count($condition);
* @throws \Exception
*/
public static function countPosts(array $condition = [], array $params = [])
{
return DBA::count('post-view', $condition, $params);
}
/**
* Retrieve a single record from the post-user-view view and returns it in an associative array
*
* @param array $fields
* @param array $condition
@ -169,11 +188,11 @@ class Post
* @throws \Exception
* @see DBA::select
*/
public static function selectFirst(array $fields = [], array $condition = [], $params = [], bool $user_mode = true)
public static function selectFirst(array $fields = [], array $condition = [], $params = [])
{
$params['limit'] = 1;
$result = self::select($fields, $condition, $params, $user_mode);
$result = self::select($fields, $condition, $params);
if (is_bool($result)) {
return $result;
@ -185,7 +204,32 @@ class Post
}
/**
* Retrieve a single record from the post-thread table and returns it in an associative array
* Retrieve a single record from the post-view view and returns it in an associative array
*
* @param array $fields
* @param array $condition
* @param array $params
* @return bool|array
* @throws \Exception
* @see DBA::select
*/
public static function selectFirstPost(array $fields = [], array $condition = [], $params = [])
{
$params['limit'] = 1;
$result = self::selectPosts($fields, $condition, $params);
if (is_bool($result)) {
return $result;
} else {
$row = self::fetch($result);
DBA::close($result);
return $row;
}
}
/**
* Retrieve a single record from the post-thread-user-view view and returns it in an associative array
*
* @param array $fields
* @param array $condition
@ -210,7 +254,7 @@ class Post
}
/**
* Select rows from the post table and returns them as an array
* Select rows from the post-user-view view and returns them as an array
*
* @param array $selected Array of selected fields, empty for all
* @param array $condition Array of fields for condition
@ -263,23 +307,37 @@ class Post
}
/**
* Select rows from the post table
* Select rows from the post-user-view view
*
* @param array $selected Array of selected fields, empty for all
* @param array $condition Array of fields for condition
* @param array $params Array of several parameters
* @param bool $user_mode true = post-user-view, false = post-view
*
* @return boolean|object
* @throws \Exception
*/
public static function select(array $selected = [], array $condition = [], $params = [], bool $user_mode = true)
public static function select(array $selected = [], array $condition = [], $params = [])
{
return self::selectView($user_mode ? 'post-user-view' : 'post-view', $selected, $condition, $params);
return self::selectView('post-user-view', $selected, $condition, $params);
}
/**
* Select rows from the post table
* Select rows from the post-view view
*
* @param array $selected Array of selected fields, empty for all
* @param array $condition Array of fields for condition
* @param array $params Array of several parameters
*
* @return boolean|object
* @throws \Exception
*/
public static function selectPosts(array $selected = [], array $condition = [], $params = [])
{
return self::selectView('post-view', $selected, $condition, $params);
}
/**
* Select rows from the post-thread-user-view view
*
* @param array $selected Array of selected fields, empty for all
* @param array $condition Array of fields for condition
@ -336,7 +394,7 @@ class Post
}
/**
* Select rows from the post view for a given user
* Select rows from the post-user-view view for a given user
*
* @param integer $uid User ID
* @param array $selected Array of selected fields, empty for all
@ -351,8 +409,24 @@ class Post
return self::selectViewForUser('post-user-view', $uid, $selected, $condition, $params);
}
/**
* Select rows from the post view for a given user
/**
* Select rows from the post-view view for a given user
*
* @param integer $uid User ID
* @param array $selected Array of selected fields, empty for all
* @param array $condition Array of fields for condition
* @param array $params Array of several parameters
*
* @return boolean|object
* @throws \Exception
*/
public static function selectPostsForUser($uid, array $selected = [], array $condition = [], $params = [])
{
return self::selectViewForUser('post-view', $uid, $selected, $condition, $params);
}
/**
* Select rows from the post-thread-user-view view for a given user
*
* @param integer $uid User ID
* @param array $selected Array of selected fields, empty for all
@ -368,7 +442,7 @@ class Post
}
/**
* Retrieve a single record from the post view for a given user and returns it in an associative array
* Retrieve a single record from the post-user-view view for a given user and returns it in an associative array
*
* @param integer $uid User ID
* @param array $selected
@ -394,7 +468,7 @@ class Post
}
/**
* Select pinned rows from the item table for a given user
* Select pinned rows from the post-thread-user table for a given user
*
* @param integer $uid User ID
* @param array $selected Array of selected fields, empty for all

2
src/Module/Api/Mastodon/Preferences.php
View file

@ -52,7 +52,7 @@ class Preferences extends BaseApi
$sensitive = false;
$language = $user['language'];
$media = DI::pConfig()->get($uid, 'nsfw', 'disable') ? 'show_all' : 'default';
$spoilers = DI::pConfig()->get($uid, 'system', 'disable_cw');
$spoilers = (bool)DI::pConfig()->get($uid, 'system', 'disable_cw');
$preferences = new \Friendica\Object\Api\Mastodon\Preferences($visibility, $sensitive, $language, $media, $spoilers);

2
src/Module/Api/Mastodon/Statuses/FavouritedBy.php
View file

@ -49,7 +49,7 @@ class FavouritedBy extends BaseApi
DI::mstdnError()->RecordNotFound();
}
$activities = Post::select(['author-id'], ['thr-parent-id' => $id, 'gravity' => GRAVITY_ACTIVITY, 'verb' => Activity::LIKE], [], false);
$activities = Post::selectPosts(['author-id'], ['thr-parent-id' => $id, 'gravity' => GRAVITY_ACTIVITY, 'verb' => Activity::LIKE]);
$accounts = [];

2
src/Module/Api/Mastodon/Statuses/RebloggedBy.php
View file

@ -49,7 +49,7 @@ class RebloggedBy extends BaseApi
DI::mstdnError()->RecordNotFound();
}
$activities = Post::select(['author-id'], ['thr-parent-id' => $id, 'gravity' => GRAVITY_ACTIVITY, 'verb' => Activity::ANNOUNCE], [], false);
$activities = Post::selectPosts(['author-id'], ['thr-parent-id' => $id, 'gravity' => GRAVITY_ACTIVITY, 'verb' => Activity::ANNOUNCE]);
$accounts = [];

6
src/Module/Api/Mastodon/Timelines/PublicTimeline.php
View file

@ -58,7 +58,7 @@ class PublicTimeline extends BaseApi
$params = ['order' => ['uri-id' => true], 'limit' => $request['limit']];
$condition = ['gravity' => [GRAVITY_PARENT, GRAVITY_COMMENT], 'private' => Item::PUBLIC,
'uid' => 0, 'network' => Protocol::FEDERATED, 'parent-author-blocked' => false, 'parent-author-hidden' => false];
'network' => Protocol::FEDERATED, 'parent-author-blocked' => false, 'parent-author-hidden' => false];
if ($request['local']) {
$condition = DBA::mergeConditions($condition, ["`uri-id` IN (SELECT `uri-id` FROM `post-user` WHERE `origin`)"]);
@ -95,12 +95,12 @@ class PublicTimeline extends BaseApi
["NOT EXISTS (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `cid` = `parent-author-id` AND (`blocked` OR `ignored`))", $uid]);
}
$items = Post::selectForUser($uid, ['uri-id', 'uid'], $condition, $params);
$items = Post::selectPostsForUser($uid, ['uri-id'], $condition, $params);
$statuses = [];
while ($item = Post::fetch($items)) {
self::setBoundaries($item['uri-id']);
$statuses[] = DI::mstdnStatus()->createFromUriId($item['uri-id'], $item['uid']);
$statuses[] = DI::mstdnStatus()->createFromUriId($item['uri-id'], $uid);
}
DBA::close($items);

2
static/dbstructure.config.php
View file

@ -55,7 +55,7 @@
use Friendica\Database\DBA;
if (!defined('DB_UPDATE_VERSION')) {
define('DB_UPDATE_VERSION', 1421);
define('DB_UPDATE_VERSION', 1422);
}
return [

56
static/dbview.config.php
View file

@ -356,6 +356,8 @@
"parent-author-link" => ["parent-post-author", "url"],
"parent-author-name" => ["parent-post-author", "name"],
"parent-author-network" => ["parent-post-author", "network"],
"parent-author-blocked" => ["parent-post-author", "blocked"],
"parent-author-hidden" => ["parent-post-author", "hidden"],
],
"query" => "FROM `post-thread-user`
INNER JOIN `post-user` ON `post-user`.`id` = `post-thread-user`.`post-user-id`
@ -417,6 +419,31 @@
"target-type" => ["post-content", "target-type"],
"target" => ["post-content", "target"],
"resource-id" => ["post-content", "resource-id"],
"contact-id" => ["post", "author-id"],
"contact-link" => ["author", "url"],
"contact-addr" => ["author", "addr"],
"contact-name" => ["author", "name"],
"contact-nick" => ["author", "nick"],
"contact-avatar" => ["author", "thumb"],
"contact-network" => ["author", "network"],
"contact-blocked" => ["author", "blocked"],
"contact-hidden" => ["author", "hidden"],
"contact-readonly" => ["author", "readonly"],
"contact-archive" => ["author", "archive"],
"contact-pending" => ["author", "pending"],
"contact-rel" => ["author", "rel"],
"contact-uid" => ["author", "uid"],
"contact-contact-type" => ["author", "contact-type"],
"writable" => "IF (`post`.`network` IN ('apub', 'dfrn', 'dspr', 'stat'), true, `author`.`writable`)",
"self" => "false",
"cid" => ["author", "id"],
"alias" => ["author", "alias"],
"photo" => ["author", "photo"],
"name-date" => ["author", "name-date"],
"uri-date" => ["author", "uri-date"],
"avatar-date" => ["author", "avatar-date"],
"thumb" => ["author", "thumb"],
"dfrn-id" => ["author", "dfrn-id"],
"author-id" => ["post", "author-id"],
"author-link" => ["author", "url"],
"author-addr" => ["author", "addr"],
@ -453,6 +480,8 @@
"parent-author-link" => ["parent-post-author", "url"],
"parent-author-name" => ["parent-post-author", "name"],
"parent-author-network" => ["parent-post-author", "network"],
"parent-author-blocked" => ["parent-post-author", "blocked"],
"parent-author-hidden" => ["parent-post-author", "hidden"],
],
"query" => "FROM `post`
STRAIGHT_JOIN `post-thread` ON `post-thread`.`uri-id` = `post`.`parent-uri-id`
@ -510,6 +539,31 @@
"target-type" => ["post-content", "target-type"],
"target" => ["post-content", "target"],
"resource-id" => ["post-content", "resource-id"],
"contact-id" => ["post-thread", "author-id"],
"contact-link" => ["author", "url"],
"contact-addr" => ["author", "addr"],
"contact-name" => ["author", "name"],
"contact-nick" => ["author", "nick"],
"contact-avatar" => ["author", "thumb"],
"contact-network" => ["author", "network"],
"contact-blocked" => ["author", "blocked"],
"contact-hidden" => ["author", "hidden"],
"contact-readonly" => ["author", "readonly"],
"contact-archive" => ["author", "archive"],
"contact-pending" => ["author", "pending"],
"contact-rel" => ["author", "rel"],
"contact-uid" => ["author", "uid"],
"contact-contact-type" => ["author", "contact-type"],
"writable" => "IF (`post`.`network` IN ('apub', 'dfrn', 'dspr', 'stat'), true, `author`.`writable`)",
"self" => "false",
"cid" => ["author", "id"],
"alias" => ["author", "alias"],
"photo" => ["author", "photo"],
"name-date" => ["author", "name-date"],
"uri-date" => ["author", "uri-date"],
"avatar-date" => ["author", "avatar-date"],
"thumb" => ["author", "thumb"],
"dfrn-id" => ["author", "dfrn-id"],
"author-id" => ["post-thread", "author-id"],
"author-link" => ["author", "url"],
"author-addr" => ["author", "addr"],
@ -546,6 +600,8 @@
"parent-author-link" => ["parent-post-author", "url"],
"parent-author-name" => ["parent-post-author", "name"],
"parent-author-network" => ["parent-post-author", "network"],
"parent-author-blocked" => ["parent-post-author", "blocked"],
"parent-author-hidden" => ["parent-post-author", "hidden"],
],
"query" => "FROM `post-thread`
INNER JOIN `post` ON `post`.`uri-id` = `post-thread`.`uri-id`