keith/friendica
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Several performance improvements

Browse source
This commit is contained in:
Michael 2022-08-15 13:23:01 +00:00
parent 9be09f7df5
commit 5b3145d7ce
15 changed files with 103 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
database.sql
View file

@ -1,6 +1,6 @@
-- ------------------------------------------
-- Friendica 2022.09-dev (Giant Rhubarb)
-- DB_UPDATE_VERSION 1480
-- DB_UPDATE_VERSION 1481
-- ------------------------------------------
@ -222,7 +222,7 @@ CREATE TABLE IF NOT EXISTS `contact` (
INDEX `uid_contact-type` (`uid`,`contact-type`),
INDEX `uid_self_contact-type` (`uid`,`self`,`contact-type`),
INDEX `self_network_uid` (`self`,`network`,`uid`),
INDEX `gsid` (`gsid`),
INDEX `gsid_uid_failed` (`gsid`,`uid`,`failed`),
INDEX `uri-id` (`uri-id`),
FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE,
FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
@ -920,6 +920,7 @@ CREATE TABLE IF NOT EXISTS `notification` (
INDEX `target-uri-id` (`target-uri-id`),
INDEX `parent-uri-id` (`parent-uri-id`),
INDEX `seen_uid` (`seen`,`uid`),
INDEX `uid_type_parent-uri-id_actor-id` (`uid`,`type`,`parent-uri-id`,`actor-id`),
FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE,
FOREIGN KEY (`vid`) REFERENCES `verb` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT,
FOREIGN KEY (`actor-id`) REFERENCES `contact` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
@ -1153,9 +1154,12 @@ CREATE TABLE IF NOT EXISTS `post-category` (
CREATE TABLE IF NOT EXISTS `post-collection` (
`uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri',
`type` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '0 - Featured',
`author-id` int unsigned COMMENT 'Author of the featured post',
PRIMARY KEY(`uri-id`,`type`),
INDEX `type` (`type`),
FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
INDEX `author-id` (`author-id`),
FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
FOREIGN KEY (`author-id`) REFERENCES `contact` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Collection of posts';
--
@ -2423,7 +2427,7 @@ DROP VIEW IF EXISTS `collection-view`;
CREATE VIEW `collection-view` AS SELECT
`post-collection`.`uri-id` AS `uri-id`,
`post-collection`.`type` AS `type`,
`post`.`author-id` AS `cid`,
`post-collection`.`author-id` AS `cid`,
`post`.`received` AS `received`,
`post`.`created` AS `created`,
`post-thread`.`commented` AS `commented`,
@ -2431,7 +2435,7 @@ CREATE VIEW `collection-view` AS SELECT
`post`.`visible` AS `visible`,
`post`.`deleted` AS `deleted`,
`post`.`thr-parent-id` AS `thr-parent-id`,
`post`.`author-id` AS `author-id`,
`post-collection`.`author-id` AS `author-id`,
`post`.`gravity` AS `gravity`
FROM `post-collection`
INNER JOIN `post` ON `post-collection`.`uri-id` = `post`.`uri-id`

2
doc/database/db_contact.md
View file

@ -121,7 +121,7 @@ Indexes
| uid_contact-type | uid, contact-type |
| uid_self_contact-type | uid, self, contact-type |
| self_network_uid | self, network, uid |
| gsid | gsid |
| gsid_uid_failed | gsid, uid, failed |
| uri-id | uri-id |
Foreign Keys

1
doc/database/db_notification.md
View file

@ -31,6 +31,7 @@ Indexes
| target-uri-id | target-uri-id |
| parent-uri-id | parent-uri-id |
| seen_uid | seen, uid |
| uid_type_parent-uri-id_actor-id | uid, type, parent-uri-id, actor-id |
Foreign Keys
------------

19
doc/database/db_post-collection.md
View file

@ -6,18 +6,20 @@ Collection of posts
Fields
------
| Field | Description | Type | Null | Key | Default | Extra |
| ------ | --------------------------------------------------------- | ---------------- | ---- | --- | ------- | ----- |
| uri-id | Id of the item-uri table entry that contains the item uri | int unsigned | NO | PRI | NULL | |
| type | 0 - Featured | tinyint unsigned | NO | PRI | 0 | |
| Field | Description | Type | Null | Key | Default | Extra |
| --------- | --------------------------------------------------------- | ---------------- | ---- | --- | ------- | ----- |
| uri-id | Id of the item-uri table entry that contains the item uri | int unsigned | NO | PRI | NULL | |
| type | 0 - Featured | tinyint unsigned | NO | PRI | 0 | |
| author-id | Author of the featured post | int unsigned | YES | | NULL | |
Indexes
------------
| Name | Fields |
| ------- | ------------ |
| PRIMARY | uri-id, type |
| type | type |
| Name | Fields |
| --------- | ------------ |
| PRIMARY | uri-id, type |
| type | type |
| author-id | author-id |
Foreign Keys
------------
@ -25,5 +27,6 @@ Foreign Keys
| Field | Target Table | Target Field |
|-------|--------------|--------------|
| uri-id | [item-uri](help/database/db_item-uri) | id |
| author-id | [contact](help/database/db_contact) | id |
Return to [database documentation](help/database)

36
src/Model/APContact.php
View file

@ -24,6 +24,7 @@ namespace Friendica\Model;
use Friendica\Content\Text\HTML;
use Friendica\Core\Cache\Enum\Duration;
use Friendica\Core\Logger;
use Friendica\Core\Protocol;
use Friendica\Core\System;
use Friendica\Database\DBA;
use Friendica\DI;
@ -352,17 +353,18 @@ class APContact
if (!empty($apcontact['outbox'])) {
if (!empty($local_owner)) {
$outbox = ActivityPub\Transmitter::getOutbox($local_owner);
$statuses_count = self::getStatusesCount($local_owner);
} else {
$outbox = ActivityPub::fetchContent($apcontact['outbox']);
$statuses_count = $outbox['totalItems'] ?? 0;
}
if (!empty($outbox['totalItems'])) {
if (!empty($statuses_count)) {
// Mastodon seriously allows for this condition?
// Jul 20 2021 - See https://chaos.social/@m11 for a negative posts count
if ($outbox['totalItems'] < 0) {
$outbox['totalItems'] = 0;
if ($statuses_count < 0) {
$statuses_count = 0;
}
$apcontact['statuses_count'] = $outbox['totalItems'];
$apcontact['statuses_count'] = $statuses_count;
}
}
@ -483,6 +485,30 @@ class APContact
return DBA::selectFirst('apcontact', [], ['url' => $apcontact['url']]) ?: [];
}
/**
* Fetch the number of statuses for the given owner
*
* @param array $owner
*
* @return integer
*/
private static function getStatusesCount(array $owner): int
{
$condition = [
'private' => [Item::PUBLIC, Item::UNLISTED],
'author-id' => Contact::getIdForURL($owner['url'], 0, false),
'gravity' => [GRAVITY_PARENT, GRAVITY_COMMENT],
'network' => Protocol::DFRN,
'parent-network' => Protocol::FEDERATED,
'deleted' => false,
'visible' => true
];
$count = Post::countPosts($condition);
return $count;
}
/**
* Mark the given AP Contact as "to archive"
*

2
src/Model/FContact.php
View file

@ -100,7 +100,7 @@ class FContact
$interacted = DBA::count('contact-relation', ["`cid` = ? AND NOT `follows` AND `last-interaction` > ?", $contact['id'], $last_interaction]);
$interacting = DBA::count('contact-relation', ["`relation-cid` = ? AND NOT `follows` AND `last-interaction` > ?", $contact['id'], $last_interaction]);
$posts = Post::countPosts(['author-id' => $contact['id'], 'gravity' => [GRAVITY_PARENT, GRAVITY_COMMENT]]);
$posts = DBA::count('post', ['author-id' => $contact['id'], 'gravity' => [GRAVITY_PARENT, GRAVITY_COMMENT]]);
}
$fields = [

12
src/Model/Post.php
View file

@ -400,12 +400,12 @@ class Post
AND (NOT `causer-blocked` OR `causer-id` = ? OR `causer-id` IS NULL) AND NOT `contact-blocked`
AND ((NOT `contact-readonly` AND NOT `contact-pending` AND (`contact-rel` IN (?, ?)))
OR `self` OR `gravity` != ? OR `contact-uid` = ?)
AND NOT EXISTS (SELECT `uri-id` FROM `post-user` WHERE `uid` = ? AND `uri-id` = `" . $view . "`.`uri-id` AND `hidden`)
AND NOT EXISTS (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `cid` = `author-id` AND `blocked`)
AND NOT EXISTS (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `cid` = `owner-id` AND `blocked`)
AND NOT EXISTS (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `cid` = `author-id` AND `ignored` AND `gravity` = ?)
AND NOT EXISTS (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `cid` = `owner-id` AND `ignored` AND `gravity` = ?)",
0, Contact::SHARING, Contact::FRIEND, GRAVITY_PARENT, 0, $uid, $uid, $uid, $uid, GRAVITY_PARENT, $uid, GRAVITY_PARENT]);
AND NOT `" . $view . "`.`uri-id` IN (SELECT `uri-id` FROM `post-user` WHERE `uid` = ? AND `hidden`)
AND NOT `author-id` IN (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `blocked`)
AND NOT `owner-id` IN (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `blocked`)
AND NOT (`gravity` = ? AND `author-id` IN (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `ignored`))
AND NOT (`gravity` = ? AND `owner-id` IN (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `ignored`))",
0, Contact::SHARING, Contact::FRIEND, GRAVITY_PARENT, 0, $uid, $uid, $uid, GRAVITY_PARENT, $uid, GRAVITY_PARENT, $uid]);
$select_string = implode(', ', array_map([DBA::class, 'quoteIdentifier'], $selected));

5
src/Model/Post/Collection.php
View file

@ -37,15 +37,16 @@ class Collection
*
* @param integer $uri_id
* @param integer $type
* @param integer $author_id
* @param integer $cache_uid If set to a non zero value, the featured cache is cleared
*/
public static function add(int $uri_id, int $type, int $cache_uid = 0)
public static function add(int $uri_id, int $type, int $author_id, int $cache_uid = 0)
{
if (empty($uri_id)) {
throw new BadMethodCallException('Empty URI_id');
}
DBA::insert('post-collection', ['uri-id' => $uri_id, 'type' => $type], Database::INSERT_IGNORE);
DBA::insert('post-collection', ['uri-id' => $uri_id, 'type' => $type, 'author-id' => $author_id], Database::INSERT_IGNORE);
if (!empty($cache_uid) && ($type == self::FEATURED)) {
DI::cache()->delete(ActivityPub\Transmitter::CACHEKEY_FEATURED . $cache_uid);

4
src/Module/Api/Mastodon/Statuses/Pin.php
View file

@ -41,12 +41,12 @@ class Pin extends BaseApi
DI::mstdnError()->UnprocessableEntity();
}
$item = Post::selectFirstForUser($uid, ['id', 'gravity'], ['uri-id' => $this->parameters['id'], 'uid' => [$uid, 0]]);
$item = Post::selectFirstForUser($uid, ['id', 'gravity', 'author-id'], ['uri-id' => $this->parameters['id'], 'uid' => [$uid, 0]]);
if (!DBA::isResult($item)) {
DI::mstdnError()->RecordNotFound();
}
Post\Collection::add($this->parameters['id'], Post\Collection::FEATURED, $uid);
Post\Collection::add($this->parameters['id'], Post\Collection::FEATURED, $item['author-id'], $uid);
System::jsonExit(DI::mstdnStatus()->createFromUriId($this->parameters['id'], $uid)->toArray());
}

4
src/Module/Item/Pin.php
View file

@ -48,7 +48,7 @@ class Pin extends BaseModule
$itemId = intval($this->parameters['id']);
$item = Post::selectFirst(['uri-id', 'uid', 'featured'], ['id' => $itemId]);
$item = Post::selectFirst(['uri-id', 'uid', 'featured', 'author-id'], ['id' => $itemId]);
if (!DBA::isResult($item)) {
throw new HTTPException\NotFoundException();
}
@ -60,7 +60,7 @@ class Pin extends BaseModule
$pinned = !$item['featured'];
if ($pinned) {
Post\Collection::add($item['uri-id'], Post\Collection::FEATURED, local_user());
Post\Collection::add($item['uri-id'], Post\Collection::FEATURED, $item['author-id'], local_user());
} else {
Post\Collection::remove($item['uri-id'], Post\Collection::FEATURED, local_user());
}

26
src/Protocol/ActivityPub/Processor.php
View file

@ -679,7 +679,7 @@ class Processor
* Fetch the Uri-Id of a post for the "featured" collection
*
* @param array $activity
* @return null|int
* @return null|array
*/
private static function getUriIdForFeaturedCollection(array $activity)
{
@ -697,7 +697,7 @@ class Processor
}
}
$parent = Post::selectFirst(['uri-id'], ['uri' => $activity['object_id']]);
$parent = Post::selectFirst(['uri-id', 'author-id'], ['uri' => $activity['object_id']]);
if (empty($parent['uri-id'])) {
if (self::fetchMissingActivity($activity['object_id'], $activity, '', Receiver::COMPLETION_AUTO)) {
$parent = Post::selectFirst(['uri-id'], ['uri' => $activity['object_id']]);
@ -705,7 +705,7 @@ class Processor
}
if (!empty($parent['uri-id'])) {
return $parent['uri-id'];
$parent;
}
return null;
@ -718,14 +718,14 @@ class Processor
*/
public static function addToFeaturedCollection(array $activity)
{
$uriid = self::getUriIdForFeaturedCollection($activity);
if (empty($uriid)) {
$post = self::getUriIdForFeaturedCollection($activity);
if (empty($post)) {
return;
}
Logger::debug('Add post to featured collection', ['uri-id' => $uriid]);
Logger::debug('Add post to featured collection', ['post' => $post]);
Post\Collection::add($uriid, Post\Collection::FEATURED);
Post\Collection::add($post['uri-id'], Post\Collection::FEATURED, $post['author-id']);
Queue::remove($activity);
}
@ -736,14 +736,14 @@ class Processor
*/
public static function removeFromFeaturedCollection(array $activity)
{
$uriid = self::getUriIdForFeaturedCollection($activity);
if (empty($uriid)) {
$post = self::getUriIdForFeaturedCollection($activity);
if (empty($post)) {
return;
}
Logger::debug('Remove post from featured collection', ['uri-id' => $uriid]);
Logger::debug('Remove post from featured collection', ['post' => $post]);
Post\Collection::remove($uriid, Post\Collection::FEATURED);
Post\Collection::remove($post['uri-id'], Post\Collection::FEATURED);
Queue::remove($activity);
}
@ -1335,10 +1335,10 @@ class Processor
}
$id = Item::fetchByLink($post['id']);
if (!empty($id)) {
$item = Post::selectFirst(['uri-id', 'featured'], ['id' => $id]);
$item = Post::selectFirst(['uri-id', 'featured', 'author-id'], ['id' => $id]);
if (!empty($item['uri-id'])) {
if (!$item['featured']) {
Post\Collection::add($item['uri-id'], Post\Collection::FEATURED);
Post\Collection::add($item['uri-id'], Post\Collection::FEATURED, $item['author-id']);
Logger::debug('Added featured post', ['uri-id' => $item['uri-id'], 'contact' => $url]);
$new++;
} else {

23
src/Protocol/ActivityPub/Transmitter.php
View file

@ -59,7 +59,6 @@ class Transmitter
{
const CACHEKEY_FEATURED = 'transmitter:getFeatured:';
const CACHEKEY_CONTACTS = 'transmitter:getContacts:';
const CACHEKEY_OUTBOX = 'transmitter:getOutbox:';
/**
* Add relay servers to the list of inboxes
@ -251,14 +250,6 @@ class Transmitter
*/
public static function getOutbox(array $owner, int $page = null, string $requester = '', bool $nocache = false): array
{
if (empty($page)) {
$cachekey = self::CACHEKEY_OUTBOX . $owner['uid'];
$result = DI::cache()->get($cachekey);
if (!$nocache && !is_null($result)) {
return $result;
}
}
$condition = ['private' => [Item::PUBLIC, Item::UNLISTED]];
if (!empty($requester)) {
@ -283,12 +274,12 @@ class Transmitter
'visible' => true
]);
$count = Post::count($condition);
$apcontact = APContact::getByURL($owner['url']);
$data = ['@context' => ActivityPub::CONTEXT];
$data['id'] = DI::baseUrl() . '/outbox/' . $owner['nickname'];
$data['type'] = 'OrderedCollection';
$data['totalItems'] = $count;
$data['totalItems'] = $apcontact['statuses_count'] ?? 0;
if (!empty($page)) {
$data['id'] .= '?' . http_build_query(['page' => $page]);
@ -316,15 +307,17 @@ class Transmitter
$data['next'] = DI::baseUrl() . '/outbox/' . $owner['nickname'] . '?page=' . ($page + 1);
}
// Fix the cached total item count when it is lower than the real count
$total = (($page - 1) * 20) + $data['totalItems'];
if ($total > $data['totalItems']) {
$data['totalItems'] = $total;
}
$data['partOf'] = DI::baseUrl() . '/outbox/' . $owner['nickname'];
$data['orderedItems'] = $list;
}
if (!empty($cachekey)) {
DI::cache()->set($cachekey, $data, Duration::DAY);
}
return $data;
}

7
static/dbstructure.config.php
View file

@ -55,7 +55,7 @@
use Friendica\Database\DBA;
if (!defined('DB_UPDATE_VERSION')) {
define('DB_UPDATE_VERSION', 1480);
define('DB_UPDATE_VERSION', 1481);
}
return [
@ -284,7 +284,7 @@ return [
"uid_contact-type" => ["uid", "contact-type"],
"uid_self_contact-type" => ["uid", "self", "contact-type"],
"self_network_uid" => ["self", "network", "uid"],
"gsid" => ["gsid"],
"gsid_uid_failed" => ["gsid", "uid", "failed"],
"uri-id" => ["uri-id"],
]
],
@ -971,6 +971,7 @@ return [
"target-uri-id" => ["target-uri-id"],
"parent-uri-id" => ["parent-uri-id"],
"seen_uid" => ["seen", "uid"],
"uid_type_parent-uri-id_actor-id" => ["uid", "type", "parent-uri-id", "actor-id"],
]
],
"notify" => [
@ -1187,10 +1188,12 @@ return [
"fields" => [
"uri-id" => ["type" => "int unsigned", "not null" => "1", "primary" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
"type" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "primary" => "1", "comment" => "0 - Featured"],
"author-id" => ["type" => "int unsigned", "foreign" => ["contact" => "id", "on delete" => "restrict"], "comment" => "Author of the featured post"],
],
"indexes" => [
"PRIMARY" => ["uri-id", "type"],
"type" => ["type"],
"author-id" => ["author-id"],
]
],
"post-content" => [

4
static/dbview.config.php
View file

@ -702,7 +702,7 @@
"fields" => [
"uri-id" => ["post-collection", "uri-id"],
"type" => ["post-collection", "type"],
"cid" => ["post", "author-id"],
"cid" => ["post-collection", "author-id"],
"received" => ["post", "received"],
"created" => ["post", "created"],
"commented" => ["post-thread", "commented"],
@ -710,7 +710,7 @@
"visible" => ["post", "visible"],
"deleted" => ["post", "deleted"],
"thr-parent-id" => ["post", "thr-parent-id"],
"author-id" => ["post", "author-id"],
"author-id" => ["post-collection", "author-id"],
"gravity" => ["post", "gravity"],
],
"query" => "FROM `post-collection`

10
update.php
View file

@ -1100,9 +1100,9 @@ function update_1451()
function update_1457()
{
$pinned = DBA::select('post-thread-user', ['uri-id'], ['pinned' => true]);
$pinned = DBA::select('post-thread-user', ['uri-id', 'author-id'], ['pinned' => true]);
while ($post = DBA::fetch($pinned)) {
Post\Collection::add($post['uri-id'], Post\Collection::FEATURED);
Post\Collection::add($post['uri-id'], Post\Collection::FEATURED, $post['author-id']);
}
DBA::close($pinned);
@ -1115,3 +1115,9 @@ function update_1480()
DBA::update('post', ['deleted' => false], ["`uri-id` IN (SELECT `uri-id` FROM `post-user` WHERE NOT `deleted`)"]);
return Update::SUCCESS;
}
function update_1481()
{
DBA::e("UPDATE `post-collection` INNER JOIN `post` ON `post`.`uri-id` = `post-collection`.`uri-id` SET `post-collection`.`author-id` = `post`.`author-id` WHERE `post-collection`.`author-id` IS null");
return Update::SUCCESS;
}