From c5effdadeca85194b844abc1ace44cf98580a032 Mon Sep 17 00:00:00 2001 From: Hank Grabowski Date: Thu, 16 Feb 2023 22:55:23 -0500 Subject: [PATCH 1/6] Add support to Mastodon Tag Trends for paging, local tags, and faster refresh = with phpcbf format auto-correction --- src/Model/Tag.php | 68 +++++++++++++++---------- src/Module/Api/Mastodon/Trends/Tags.php | 11 +++- 2 files changed, 51 insertions(+), 28 deletions(-) diff --git a/src/Model/Tag.php b/src/Model/Tag.php index d46680059..362df49ec 100644 --- a/src/Model/Tag.php +++ b/src/Model/Tag.php @@ -534,8 +534,11 @@ class Tag $searchpath = DI::baseUrl() . '/search?tag='; - $taglist = DBA::select('tag-view', ['type', 'name', 'url', 'cid'], - ['uri-id' => $item['uri-id'], 'type' => [self::HASHTAG, self::MENTION, self::EXCLUSIVE_MENTION, self::IMPLICIT_MENTION]]); + $taglist = DBA::select( + 'tag-view', + ['type', 'name', 'url', 'cid'], + ['uri-id' => $item['uri-id'], 'type' => [self::HASHTAG, self::MENTION, self::EXCLUSIVE_MENTION, self::IMPLICIT_MENTION]] + ); while ($tag = DBA::fetch($taglist)) { if ($tag['url'] == '') { $tag['url'] = $searchpath . rawurlencode($tag['name']); @@ -544,7 +547,7 @@ class Tag $orig_tag = $tag['url']; $prefix = self::TAG_CHARACTER[$tag['type']]; - switch($tag['type']) { + switch ($tag['type']) { case self::HASHTAG: if ($orig_tag != $tag['url']) { $item['body'] = str_replace($orig_tag, $tag['url'], $item['body']); @@ -639,17 +642,17 @@ class Tag * * @param int $period Period in hours to consider posts * @param int $limit Number of returned tags + * @param int $offset Page offset in results * @return array * @throws \Exception */ - public static function getGlobalTrendingHashtags(int $period, $limit = 10): array + public static function getGlobalTrendingHashtags(int $period, int $limit = 10, int $offset = 0): array { - $tags = DI::cache()->get('global_trending_tags-' . $period . '-' . $limit); - if (!empty($tags)) { - return $tags; - } else { - return self::setGlobalTrendingHashtags($period, $limit); + $tags = DI::cache()->get("global_trending_tags-$period"); + if (empty($tags)) { + $tags = self::setGlobalTrendingHashtags($period, 1000); } + return array_slice($tags, $limit * $offset, $limit); } /** @@ -665,7 +668,9 @@ class Tag } $blocked = explode(',', $blocked_txt); - array_walk($blocked, function(&$value) { $value = "'" . DBA::escape(trim($value)) . "'";}); + array_walk($blocked, function (&$value) { + $value = "'" . DBA::escape(trim($value)) . "'"; + }); return ' AND NOT `name` IN (' . implode(',', $blocked) . ')'; } @@ -683,8 +688,11 @@ class Tag * Get a uri-id that is at least X hours old. * We use the uri-id in the query for the hash tags since this is much faster */ - $post = Post::selectFirstThread(['uri-id'], ["`uid` = ? AND `received` < ?", 0, DateTimeFormat::utc('now - ' . $period . ' hour')], - ['order' => ['received' => true]]); + $post = Post::selectFirstThread( + ['uri-id'], + ["`uid` = ? AND `received` < ?", 0, DateTimeFormat::utc('now - ' . $period . ' hour')], + ['order' => ['received' => true]] + ); if (empty($post['uri-id'])) { return []; @@ -692,17 +700,20 @@ class Tag $block_sql = self::getBlockedSQL(); - $tagsStmt = DBA::p("SELECT `name` AS `term`, COUNT(*) AS `score`, COUNT(DISTINCT(`author-id`)) as `authors` + $tagsStmt = DBA::p( + "SELECT `name` AS `term`, COUNT(*) AS `score`, COUNT(DISTINCT(`author-id`)) as `authors` FROM `tag-search-view` WHERE `private` = ? AND `uid` = ? AND `uri-id` > ? $block_sql GROUP BY `term` ORDER BY `authors` DESC, `score` DESC LIMIT ?", - Item::PUBLIC, 0, $post['uri-id'], + Item::PUBLIC, + 0, + $post['uri-id'], $limit ); if (DBA::isResult($tagsStmt)) { $tags = DBA::toArray($tagsStmt); - DI::cache()->set('global_trending_tags-' . $period . '-' . $limit, $tags, Duration::DAY); + DI::cache()->set("global_trending_tags-$period", $tags, Duration::HOUR); return $tags; } @@ -714,17 +725,17 @@ class Tag * * @param int $period Period in hours to consider posts * @param int $limit Number of returned tags + * @param int $offset Page offset in results * @return array * @throws \Exception */ - public static function getLocalTrendingHashtags(int $period, $limit = 10): array + public static function getLocalTrendingHashtags(int $period, $limit = 10, int $offset = 0): array { - $tags = DI::cache()->get('local_trending_tags-' . $period . '-' . $limit); - if (!empty($tags)) { - return $tags; - } else { - return self::setLocalTrendingHashtags($period, $limit); + $tags = DI::cache()->get("local_trending_tags-$period"); + if (empty($tags)) { + $tags = self::setLocalTrendingHashtags($period, 1000); } + return array_slice($tags, $limit * $offset, $limit); } /** @@ -739,25 +750,30 @@ class Tag { // Get a uri-id that is at least X hours old. // We use the uri-id in the query for the hash tags since this is much faster - $post = Post::selectFirstThread(['uri-id'], ["`uid` = ? AND `received` < ?", 0, DateTimeFormat::utc('now - ' . $period . ' hour')], - ['order' => ['received' => true]]); + $post = Post::selectFirstThread( + ['uri-id'], + ["`uid` = ? AND `received` < ?", 0, DateTimeFormat::utc('now - ' . $period . ' hour')], + ['order' => ['received' => true]] + ); if (empty($post['uri-id'])) { return []; } $block_sql = self::getBlockedSQL(); - $tagsStmt = DBA::p("SELECT `name` AS `term`, COUNT(*) AS `score`, COUNT(DISTINCT(`author-id`)) as `authors` + $tagsStmt = DBA::p( + "SELECT `name` AS `term`, COUNT(*) AS `score`, COUNT(DISTINCT(`author-id`)) as `authors` FROM `tag-search-view` WHERE `private` = ? AND `wall` AND `origin` AND `uri-id` > ? $block_sql GROUP BY `term` ORDER BY `authors` DESC, `score` DESC LIMIT ?", - Item::PUBLIC, $post['uri-id'], + Item::PUBLIC, + $post['uri-id'], $limit ); if (DBA::isResult($tagsStmt)) { $tags = DBA::toArray($tagsStmt); - DI::cache()->set('local_trending_tags-' . $period . '-' . $limit, $tags, Duration::DAY); + DI::cache()->set("local_trending_tags-$period", $tags, Duration::HOUR); return $tags; } diff --git a/src/Module/Api/Mastodon/Trends/Tags.php b/src/Module/Api/Mastodon/Trends/Tags.php index 810ab002d..b084797c8 100644 --- a/src/Module/Api/Mastodon/Trends/Tags.php +++ b/src/Module/Api/Mastodon/Trends/Tags.php @@ -37,11 +37,18 @@ class Tags extends BaseApi protected function rawContent(array $request = []) { $request = $this->getRequest([ - 'limit' => 20, // Maximum number of results to return. Defaults to 10. + 'limit' => 20, // Maximum number of results to return. Defaults to 20. + 'offset' => 0, + 'friendica_local' => false, ], $request); $trending = []; - $tags = Tag::getGlobalTrendingHashtags(24, 20); + if ($request['friendica_local']) { + $tags = Tag::getLocalTrendingHashtags(24, $request['limit'], $request['offset']); + } else { + $tags = Tag::getGlobalTrendingHashtags(24, $request['limit'], $request['offset']); + } + foreach ($tags as $tag) { $tag['name'] = $tag['term']; $history = [['day' => (string)time(), 'uses' => (string)$tag['score'], 'accounts' => (string)$tag['authors']]]; From 6a94632131060bade068717095cc484b9d51652f Mon Sep 17 00:00:00 2001 From: Hank Grabowski Date: Fri, 17 Feb 2023 11:25:25 -0500 Subject: [PATCH 2/6] Add documentation to new trending tags endpoint QPs --- src/Module/Api/Mastodon/Trends/Tags.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Module/Api/Mastodon/Trends/Tags.php b/src/Module/Api/Mastodon/Trends/Tags.php index b084797c8..21e078b23 100644 --- a/src/Module/Api/Mastodon/Trends/Tags.php +++ b/src/Module/Api/Mastodon/Trends/Tags.php @@ -38,8 +38,8 @@ class Tags extends BaseApi { $request = $this->getRequest([ 'limit' => 20, // Maximum number of results to return. Defaults to 20. - 'offset' => 0, - 'friendica_local' => false, + 'offset' => 0, // Offset page. Defaults to 0. + 'friendica_local' => false, // Whether to return local tag trends instead of global, defaults to false ], $request); $trending = []; From e6c93d31c17760c76909f0569176a1ec3050dc78 Mon Sep 17 00:00:00 2001 From: Hank Grabowski Date: Fri, 17 Feb 2023 11:59:30 -0500 Subject: [PATCH 3/6] Add offset parameter to Mastodon trending Links and Statuses endpoints --- src/Module/Api/Mastodon/Trends/Links.php | 3 ++- src/Module/Api/Mastodon/Trends/Statuses.php | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Module/Api/Mastodon/Trends/Links.php b/src/Module/Api/Mastodon/Trends/Links.php index 5cf2ed471..e2a160087 100644 --- a/src/Module/Api/Mastodon/Trends/Links.php +++ b/src/Module/Api/Mastodon/Trends/Links.php @@ -41,6 +41,7 @@ class Links extends BaseApi { $request = $this->getRequest([ 'limit' => 10, // Maximum number of results to return. Defaults to 10. + 'offset' => 0, // Offset page, Defaults to 0. ], $request); $condition = ["EXISTS(SELECT `id` FROM `post-media` WHERE `post-media`.`uri-id` = `post-thread-view`.`uri-id` AND `type` = ? AND NOT `name` IS NULL AND NOT `description` IS NULL) AND NOT `private` AND `commented` > ? AND `created` > ?", @@ -48,7 +49,7 @@ class Links extends BaseApi $condition = DBA::mergeConditions($condition, ['network' => Protocol::FEDERATED]); $trending = []; - $statuses = Post::selectPostThread(['uri-id', 'total-comments', 'total-actors'], $condition, ['limit' => $request['limit'], 'order' => ['total-actors' => true]]); + $statuses = Post::selectPostThread(['uri-id', 'total-comments', 'total-actors'], $condition, ['limit' => [$request['offset'], $request['limit']], 'offset' => $request['offset'], 'order' => ['total-actors' => true]]); while ($status = Post::fetch($statuses)) { $history = [['day' => (string)time(), 'uses' => (string)$status['total-comments'], 'accounts' => (string)$status['total-actors']]]; $trending[] = DI::mstdnCard()->createFromUriId($status['uri-id'], $history)->toArray(); diff --git a/src/Module/Api/Mastodon/Trends/Statuses.php b/src/Module/Api/Mastodon/Trends/Statuses.php index cf287e59c..6ae9c83dd 100644 --- a/src/Module/Api/Mastodon/Trends/Statuses.php +++ b/src/Module/Api/Mastodon/Trends/Statuses.php @@ -44,6 +44,7 @@ class Statuses extends BaseApi $request = $this->getRequest([ 'limit' => 10, // Maximum number of results to return. Defaults to 10. + 'offset' => 0, // Offset page, Defaults to 0. ], $request); $condition = ["NOT `private` AND `commented` > ? AND `created` > ?", DateTimeFormat::utc('now -1 day'), DateTimeFormat::utc('now -1 week')]; @@ -52,7 +53,7 @@ class Statuses extends BaseApi $display_quotes = self::appSupportsQuotes(); $trending = []; - $statuses = Post::selectPostThread(['uri-id'], $condition, ['limit' => $request['limit'], 'order' => ['total-actors' => true]]); + $statuses = Post::selectPostThread(['uri-id'], $condition, ['limit' => [$request['offset'], $request['limit']], 'order' => ['total-actors' => true]]); while ($status = Post::fetch($statuses)) { try { $trending[] = DI::mstdnStatus()->createFromUriId($status['uri-id'], $uid, $display_quotes); From df4af8da9bac95aba6839487d7dd851227fea30c Mon Sep 17 00:00:00 2001 From: Hank Grabowski Date: Fri, 17 Feb 2023 13:23:34 -0500 Subject: [PATCH 4/6] Fix offset is absolute in set not a page to be consistent with SQL --- src/Model/Tag.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Model/Tag.php b/src/Model/Tag.php index 362df49ec..0e891f9c6 100644 --- a/src/Model/Tag.php +++ b/src/Model/Tag.php @@ -652,7 +652,7 @@ class Tag if (empty($tags)) { $tags = self::setGlobalTrendingHashtags($period, 1000); } - return array_slice($tags, $limit * $offset, $limit); + return array_slice($tags, $offset, $limit); } /** @@ -735,7 +735,7 @@ class Tag if (empty($tags)) { $tags = self::setLocalTrendingHashtags($period, 1000); } - return array_slice($tags, $limit * $offset, $limit); + return array_slice($tags, $offset, $limit); } /** From 2754cdc5d6661d27351f7c0d320f715f0b6105ba Mon Sep 17 00:00:00 2001 From: Hank Grabowski Date: Fri, 17 Feb 2023 13:24:00 -0500 Subject: [PATCH 5/6] Add Link headers by offset/limit capability to BaseApi --- src/Module/BaseApi.php | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/Module/BaseApi.php b/src/Module/BaseApi.php index 6a4119847..f8524b524 100644 --- a/src/Module/BaseApi.php +++ b/src/Module/BaseApi.php @@ -168,6 +168,34 @@ class BaseApi extends BaseModule return 'Link: <' . $next . '>; rel="next", <' . $prev . '>; rel="prev"'; } + /** + * Get the "link" header with "next" and "prev" links for an offset/limit type call + * @return string + */ + protected static function getOffsetAndLimitLinkHeader(int $offset, int $limit): string + { + $request = self::$request; + + unset($request['offset']); + $request['limit'] = $limit; + + $prev_request = $next_request = $request; + + $prev_request['offset'] = $offset - $limit; + $next_request['offset'] = $offset + $limit; + + $command = DI::baseUrl() . '/' . DI::args()->getCommand(); + + $prev = $command . '?' . http_build_query($prev_request); + $next = $command . '?' . http_build_query($next_request); + + if ($prev_request['offset'] >= 0) { + return 'Link: <' . $next . '>; rel="next", <' . $prev . '>; rel="prev"'; + } else { + return 'Link: <' . $next . '>; rel="next"'; + } + } + /** * Set the "link" header with "next" and "prev" links * @return void @@ -180,6 +208,18 @@ class BaseApi extends BaseModule } } + /** + * Set the "link" header with "next" and "prev" links + * @return void + */ + protected static function setLinkHeaderByOffsetLimit(int $offset, int $limit) + { + $header = self::getOffsetAndLimitLinkHeader($offset, $limit); + if (!empty($header)) { + header($header); + } + } + /** * Check if the app is known to support quoted posts * From 9187723263c5b0d87b78bfbdcfa8b8234ee958ce Mon Sep 17 00:00:00 2001 From: Hank Grabowski Date: Fri, 17 Feb 2023 13:24:11 -0500 Subject: [PATCH 6/6] Add link headers to Mastodon trending endpoints --- src/Module/Api/Mastodon/Trends/Links.php | 6 +++++- src/Module/Api/Mastodon/Trends/Statuses.php | 6 +++++- src/Module/Api/Mastodon/Trends/Tags.php | 8 ++++++-- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/Module/Api/Mastodon/Trends/Links.php b/src/Module/Api/Mastodon/Trends/Links.php index e2a160087..bfb353147 100644 --- a/src/Module/Api/Mastodon/Trends/Links.php +++ b/src/Module/Api/Mastodon/Trends/Links.php @@ -41,7 +41,7 @@ class Links extends BaseApi { $request = $this->getRequest([ 'limit' => 10, // Maximum number of results to return. Defaults to 10. - 'offset' => 0, // Offset page, Defaults to 0. + 'offset' => 0, // Offset in set, Defaults to 0. ], $request); $condition = ["EXISTS(SELECT `id` FROM `post-media` WHERE `post-media`.`uri-id` = `post-thread-view`.`uri-id` AND `type` = ? AND NOT `name` IS NULL AND NOT `description` IS NULL) AND NOT `private` AND `commented` > ? AND `created` > ?", @@ -56,6 +56,10 @@ class Links extends BaseApi } DBA::close($statuses); + if (!empty($trending)) { + self::setLinkHeaderByOffsetLimit($request['offset'], $request['limit']); + } + System::jsonExit($trending); } } diff --git a/src/Module/Api/Mastodon/Trends/Statuses.php b/src/Module/Api/Mastodon/Trends/Statuses.php index 6ae9c83dd..884319aa4 100644 --- a/src/Module/Api/Mastodon/Trends/Statuses.php +++ b/src/Module/Api/Mastodon/Trends/Statuses.php @@ -44,7 +44,7 @@ class Statuses extends BaseApi $request = $this->getRequest([ 'limit' => 10, // Maximum number of results to return. Defaults to 10. - 'offset' => 0, // Offset page, Defaults to 0. + 'offset' => 0, // Offset in set, Defaults to 0. ], $request); $condition = ["NOT `private` AND `commented` > ? AND `created` > ?", DateTimeFormat::utc('now -1 day'), DateTimeFormat::utc('now -1 week')]; @@ -63,6 +63,10 @@ class Statuses extends BaseApi } DBA::close($statuses); + if (!empty($trending)) { + self::setLinkHeaderByOffsetLimit($request['offset'], $request['limit']); + } + System::jsonExit($trending); } } diff --git a/src/Module/Api/Mastodon/Trends/Tags.php b/src/Module/Api/Mastodon/Trends/Tags.php index 21e078b23..2190a2e3c 100644 --- a/src/Module/Api/Mastodon/Trends/Tags.php +++ b/src/Module/Api/Mastodon/Trends/Tags.php @@ -38,7 +38,7 @@ class Tags extends BaseApi { $request = $this->getRequest([ 'limit' => 20, // Maximum number of results to return. Defaults to 20. - 'offset' => 0, // Offset page. Defaults to 0. + 'offset' => 0, // Offset in set. Defaults to 0. 'friendica_local' => false, // Whether to return local tag trends instead of global, defaults to false ], $request); @@ -56,6 +56,10 @@ class Tags extends BaseApi $trending[] = $hashtag->toArray(); } - System::jsonExit(array_slice($trending, 0, $request['limit'])); + if (!empty($trending)) { + self::setLinkHeaderByOffsetLimit($request['offset'], $request['limit']); + } + + System::jsonExit($trending); } }