Fix the erratic page update behaviour

doc/database/db_post-thread-user.md

@@ -49,6 +49,9 @@ Indexes
 | psid              | psid                |
 | post-user-id      | post-user-id        |
 | commented         | commented           |
+| received          | received            |
+| wall              | wall                |
+| origin            | origin              |
 | uid_received      | uid, received       |
 | uid_wall_received | uid, wall, received |
 | uid_commented     | uid, commented      |

doc/database/db_report.md

@@ -14,14 +14,14 @@ Fields
 | cid             | Reported contact                                             | int unsigned       | NO   |     | NULL                       |                |
 | gsid            | Reported contact server                                      | int unsigned       | YES  |     | NULL                       |                |
 | comment         | Report                                                       | text               | YES  |     | NULL                       |                |
-| category-id     | Report category, one of Entity\Report::CATEGORY_*            | int unsigned       | NO   |     | 1                          |                |
+| category-id     | Report category, one of Entity Report::CATEGORY_*            | int unsigned       | NO   |     | 1                          |                |
 | forward         | Forward the report to the remote server                      | boolean            | YES  |     | NULL                       |                |
 | public-remarks  | Remarks shared with the reporter                             | text               | YES  |     | NULL                       |                |
 | private-remarks | Remarks shared with the moderation team                      | text               | YES  |     | NULL                       |                |
 | last-editor-uid | Last editor user                                             | mediumint unsigned | YES  |     | NULL                       |                |
 | assigned-uid    | Assigned moderator user                                      | mediumint unsigned | YES  |     | NULL                       |                |
-| status          | Status of the report, one of Entity\Report::STATUS_*         | tinyint unsigned   | NO   |     | NULL                       |                |
+| status          | Status of the report, one of Entity Report::STATUS_*         | tinyint unsigned   | NO   |     | NULL                       |                |
-| resolution      | Resolution of the report, one of Entity\Report::RESOLUTION_* | tinyint unsigned   | YES  |     | NULL                       |                |
+| resolution      | Resolution of the report, one of Entity Report::RESOLUTION_* | tinyint unsigned   | YES  |     | NULL                       |                |
 | created         |                                                              | datetime(6)        | NO   |     | 0001-01-01 00:00:00.000000 |                |
 | edited          | Last time the report has been edited                         | datetime(6)        | YES  |     | NULL                       |                |
 

src/Database/DBStructure.php

@@ -284,7 +284,7 @@ class DBStructure
 					echo $sql;
 				}
 				if ($action) {
-					$r = DBA::e(str_replace('\\', '\\\\', $sql));
+					$r = DBA::e($sql);
 					if (!DBA::isResult($r)) {
 						$errors .= self::printUpdateError($name);
 					}
@@ -493,7 +493,7 @@ class DBStructure
 						DI::config()->set('system', 'maintenance_reason', DI::l10n()->t('%s: updating %s table.', DateTimeFormat::utcNow() . ' ' . date('e'), $name));
 					}
 
-					$r = DBA::e(str_replace('\\', '\\\\', $sql3));
+					$r = DBA::e($sql3);
 					if (!DBA::isResult($r)) {
 						$errors .= self::printUpdateError($sql3);
 					}

src/Module/Conversation/Community.php

@@ -126,13 +126,13 @@ class Community extends Timeline
 			return $o;
 		}
 
-		$o .= $this->conversation->render($items, Conversation::MODE_COMMUNITY, false, false, 'commented', $this->session->getLocalUserId());
+		$o .= $this->conversation->render($items, Conversation::MODE_COMMUNITY, false, false, 'received', $this->session->getLocalUserId());
 
 		$pager = new BoundariesPager(
 			$this->l10n,
 			$this->args->getQueryString(),
-			$items[0]['commented'],
+			$items[0]['received'],
-			$items[count($items) - 1]['commented'],
+			$items[count($items) - 1]['received'],
 			$this->itemsPerPage
 		);
 
@@ -196,6 +196,7 @@ class Community extends Timeline
 			}
 		}
 
-		$this->maxId = $request['last_commented'] ?? $this->maxId;
+		$this->maxId = $request['last_received'] ?? $this->maxId;
+		$this->minId = $request['first_received'] ?? $this->minId;
 	}
 }

src/Module/Conversation/Network.php

@@ -340,7 +340,7 @@ class Network extends Timeline
 			$this->order = $request['order'];
 			$this->star = false;
 			$this->mention = false;
-		} elseif (in_array($this->selectedTab, [TimelineEntity::RECEIVED, TimelineEntity::STAR])) {
+		} elseif (in_array($this->selectedTab, [TimelineEntity::RECEIVED, TimelineEntity::STAR]) || $this->timeline->isCommunity($this->selectedTab)) {
 			$this->order = 'received';
 		} elseif (($this->selectedTab == TimelineEntity::CREATED) || $this->timeline->isChannel($this->selectedTab)) {
 			$this->order = 'created';
@@ -348,6 +348,12 @@ class Network extends Timeline
 			$this->order = 'commented';
 		}
 
+		// Upon force (updates in the background) and order by last comment we order by receive date,
+		// since otherwise the feed will optically jump, when some already visible thread has been updated.
+		if ($this->force && ($this->selectedTab == TimelineEntity::COMMENTED)) {
+			$this->order = 'received';
+		}
+
 		$this->selectedTab = $this->selectedTab ?? $this->order;
 
 		// Prohibit combined usage of "star" and "mention"
@@ -368,16 +374,20 @@ class Network extends Timeline
 		switch ($this->order) {
 			case 'received':
 				$this->maxId = $request['last_received'] ?? $this->maxId;
+				$this->minId = $request['first_received'] ?? $this->minId;
 				break;
 			case 'created':
 				$this->maxId = $request['last_created'] ?? $this->maxId;
+				$this->minId = $request['first_created'] ?? $this->minId;
 				break;
 			case 'uriid':
 				$this->maxId = $request['last_uriid'] ?? $this->maxId;
+				$this->minId = $request['first_uriid'] ?? $this->minId;
 				break;
 			default:
 				$this->order = 'commented';
 				$this->maxId = $request['last_commented'] ?? $this->maxId;
+				$this->minId = $request['first_commented'] ?? $this->minId;
 		}
 	}
 

src/Module/Conversation/Timeline.php

@@ -62,6 +62,8 @@ class Timeline extends BaseModule
 	protected $itemsPerPage;
 	/** @var bool */
 	protected $noSharer;
+	/** @var bool */
+	protected $force;
 
 	/** @var App\Mode $mode */
 	protected $mode;
@@ -129,6 +131,7 @@ class Timeline extends BaseModule
 		$this->maxId = $request['max_id'] ?? null;
 
 		$this->noSharer = !empty($request['no_sharer']);
+		$this->force    = !empty($request['force']);
 	}
 
 	protected function getNoSharerWidget(string $base): string
@@ -191,31 +194,47 @@ class Timeline extends BaseModule
 	{
 		$items = $this->getRawChannelItems();
 
-		$contacts = $this->database->selectToArray('user-contact', ['cid'], ['channel-visibility' => Contact\User::VISIBILITY_REDUCED, 'cid' => array_column($items, 'owner-id')]);
+		$contacts = $this->database->selectToArray('user-contact', ['cid'], ['channel-frequency' => Contact\User::FREQUENCY_REDUCED, 'cid' => array_column($items, 'owner-id')]);
 		$reduced  = array_column($contacts, 'cid');
 
 		$maxpostperauthor = $this->config->get('channel', 'max_posts_per_author');
 
 		if ($maxpostperauthor != 0) {
 			$count          = 1;
-			$numposts       = [];
+			$owner_posts    = [];
 			$selected_items = [];
 
 			while (count($selected_items) < $this->itemsPerPage && ++$count < 50 && count($items) > 0) {
+				$maxposts = round((count($items) / $this->itemsPerPage) * $maxpostperauthor);
+				$minId = $items[array_key_first($items)]['created'];
+				$maxId = $items[array_key_last($items)]['created'];
+
 				foreach ($items as $item) {
-					$numposts[$item['owner-id']] = ($numposts[$item['owner-id']] ?? 0);
+					if (!in_array($item['owner-id'], $reduced)) {
-					if (!in_array($item['owner-id'], $reduced) || (($numposts[$item['owner-id']]++ < $maxpostperauthor)) && (count($selected_items) < $this->itemsPerPage)) {
+						continue;
-						$selected_items[] = $item;
+					}
+					$owner_posts[$item['owner-id']][$item['uri-id']] = (($item['comments'] * 100) + $item['activities']);
+				}
+				foreach ($owner_posts as $posts) {
+					if (count($posts) <= $maxposts) {
+						continue;
+					}
+					asort($posts);
+					while (count($posts) > $maxposts) {
+						$uri_id = array_key_first($posts);
+						unset($posts[$uri_id]);
+						unset($items[$uri_id]);
 					}
 				}
+				$selected_items = array_merge($selected_items, $items);
 
 				// If we're looking at a "previous page", the lookup continues forward in time because the list is
 				// sorted in chronologically decreasing order
-				if (isset($this->minId)) {
+				if (!empty($this->minId)) {
-					$this->minId = $items[0]['created'];
+					$this->minId = $minId;
 				} else {
 					// In any other case, the lookup continues backwards in time
-					$this->maxId = $items[count($items) - 1]['created'];
+					$this->maxId = $maxId;
 				}
 
 				if (count($selected_items) < $this->itemsPerPage) {
@@ -254,9 +273,9 @@ class Timeline extends BaseModule
 			$condition = [
 				"(`owner-id` IN (SELECT `cid` FROM `contact-relation` WHERE `relation-cid` = ? AND `relation-thread-score` > ?) OR
 				((`comments` >= ? OR `activities` >= ?) AND `owner-id` IN (SELECT `cid` FROM `contact-relation` WHERE `follows` AND `relation-cid` = ?)) OR
-				(`owner-id` IN (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND (`notify_new_posts` OR `channel-visibility` = ?))))",
+				(`owner-id` IN (SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND (`notify_new_posts` OR `channel-frequency` = ?))))",
 				$cid, $this->getMedianRelationThreadScore($cid, 4), $this->getMedianComments($uid, 4), $this->getMedianActivities($uid, 4), $cid,
-				$uid, Contact\User::VISIBILITY_ALWAYS
+				$uid, Contact\User::FREQUENCY_ALWAYS
 			];
 		} elseif ($this->selectedTab == TimelineEntity::FOLLOWERS) {
 			$condition = ["`owner-id` IN (SELECT `pid` FROM `account-user-view` WHERE `uid` = ? AND `rel` = ?)", $uid, Contact::FOLLOWER];
@@ -284,7 +303,7 @@ class Timeline extends BaseModule
 			$condition = $this->addLanguageCondition($uid, $condition);
 		}
 
-		$condition = DBA::mergeConditions($condition, ["NOT EXISTS(SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `cid` = `post-engagement`.`owner-id` AND (`ignored` OR `blocked` OR `collapsed` OR `is-blocked` OR `channel-visibility` = ?))", $uid, Contact\User::VISIBILITY_NEVER]);
+		$condition = DBA::mergeConditions($condition, ["NOT EXISTS(SELECT `cid` FROM `user-contact` WHERE `uid` = ? AND `cid` = `post-engagement`.`owner-id` AND (`ignored` OR `blocked` OR `collapsed` OR `is-blocked` OR `channel-frequency` = ?))", $uid, Contact\User::FREQUENCY_NEVER]);
 
 		if (($this->selectedTab != TimelineEntity::WHATSHOT) && !is_null($this->accountType)) {
 			$condition = DBA::mergeConditions($condition, ['contact-type' => $this->accountType]);
@@ -313,14 +332,20 @@ class Timeline extends BaseModule
 			}
 		}
 
-		$items = $this->database->selectToArray('post-engagement', ['uri-id', 'created', 'owner-id'], $condition, $params);
+		$items = [];
+		$result = $this->database->select('post-engagement', ['uri-id', 'created', 'owner-id', 'comments', 'activities'], $condition, $params);
+		while ($item = $this->database->fetch($result)) {
+			$items[$item['uri-id']] = $item;
+		}
+		$this->database->close($result);
+
 		if (empty($items)) {
 			return [];
 		}
 
 		// Previous page case: once we get the relevant items closest to min_id, we need to restore the expected display order
 		if (empty($this->itemUriId) && isset($this->minId) && !isset($this->maxId)) {
-			$items = array_reverse($items);
+			$items = array_reverse($items, true);
 		}
 
 		$condition = ['unseen' => true, 'uid' => $uid, 'parent-uri-id' => array_column($items, 'uri-id')];
@@ -451,10 +476,10 @@ class Timeline extends BaseModule
 				// If we're looking at a "previous page", the lookup continues forward in time because the list is
 				// sorted in chronologically decreasing order
 				if (isset($this->minId)) {
-					$this->minId = $items[0]['commented'];
+					$this->minId = $items[0]['received'];
 				} else {
 					// In any other case, the lookup continues backwards in time
-					$this->maxId = $items[count($items) - 1]['commented'];
+					$this->maxId = $items[count($items) - 1]['received'];
 				}
 
 				$items = $this->selectItems();
@@ -479,22 +504,18 @@ class Timeline extends BaseModule
 	private function selectItems()
 	{
 		if ($this->selectedTab == 'local') {
-			if (!is_null($this->accountType)) {
-				$condition = ["`wall` AND `origin` AND `private` = ? AND `owner-contact-type` = ?", Item::PUBLIC, $this->accountType];
-			} else {
 			$condition = ["`wall` AND `origin` AND `private` = ?", Item::PUBLIC];
-			}
 		} elseif ($this->selectedTab == 'global') {
-			if (!is_null($this->accountType)) {
-				$condition = ["`uid` = ? AND `private` = ? AND `owner-contact-type` = ?", 0, Item::PUBLIC, $this->accountType];
-			} else {
 			$condition = ["`uid` = ? AND `private` = ?", 0, Item::PUBLIC];
-			}
 		} else {
 			return [];
 		}
 
-		$params = ['order' => ['commented' => true], 'limit' => $this->itemsPerPage];
+		if (!is_null($this->accountType)) {
+			$condition = DBA::mergeConditions($condition, ['owner-contact-type' => $this->accountType]);
+		}
+
+		$params = ['order' => ['received' => true], 'limit' => $this->itemsPerPage];
 
 		if (!empty($this->itemUriId)) {
 			$condition = DBA::mergeConditions($condition, ['uri-id' => $this->itemUriId]);
@@ -504,20 +525,20 @@ class Timeline extends BaseModule
 			}
 
 			if (isset($this->maxId)) {
-				$condition = DBA::mergeConditions($condition, ["`commented` < ?", $this->maxId]);
+				$condition = DBA::mergeConditions($condition, ["`received` < ?", $this->maxId]);
 			}
 
 			if (isset($this->minId)) {
-				$condition = DBA::mergeConditions($condition, ["`commented` > ?", $this->minId]);
+				$condition = DBA::mergeConditions($condition, ["`received` > ?", $this->minId]);
 
 				// Previous page case: we want the items closest to min_id but for that we need to reverse the query order
 				if (!isset($this->maxId)) {
-					$params['order']['commented'] = false;
+					$params['order']['received'] = false;
 				}
 			}
 		}
 
-		$r = Post::selectThreadForUser($this->session->getLocalUserId() ?: 0, ['uri-id', 'commented', 'author-link'], $condition, $params);
+		$r = Post::selectThreadForUser($this->session->getLocalUserId() ?: 0, ['uri-id', 'received', 'author-link'], $condition, $params);
 
 		$items = Post::toArray($r);
 		if (empty($items)) {

static/dbstructure.config.php

@@ -1578,6 +1578,9 @@ return [
 			"psid" => ["psid"],
 			"post-user-id" => ["post-user-id"],
 			"commented" => ["commented"],
+			"received" => ["received"],
+			"wall" => ["wall"],
+			"origin" => ["origin"],
 			"uid_received" => ["uid", "received"],
 			"uid_wall_received" => ["uid", "wall", "received"],
 			"uid_commented" => ["uid", "commented"],
@@ -1728,14 +1731,14 @@ return [
 			"cid" => ["type" => "int unsigned", "not null" => "1", "foreign" => ["contact" => "id"], "comment" => "Reported contact"],
 			"gsid" => ["type" => "int unsigned", "foreign" => ["gserver" => "id"], "comment" => "Reported contact server"],
 			"comment" => ["type" => "text", "comment" => "Report"],
-			"category-id" => ["type" => "int unsigned", "not null" => 1, "default" => \Friendica\Moderation\Entity\Report::CATEGORY_OTHER, "comment" => "Report category, one of Entity\Report::CATEGORY_*"],
+			"category-id" => ["type" => "int unsigned", "not null" => 1, "default" => \Friendica\Moderation\Entity\Report::CATEGORY_OTHER, "comment" => "Report category, one of Entity Report::CATEGORY_*"],
 			"forward" => ["type" => "boolean", "comment" => "Forward the report to the remote server"],
 			"public-remarks" => ["type" => "text", "comment" => "Remarks shared with the reporter"],
 			"private-remarks" => ["type" => "text", "comment" => "Remarks shared with the moderation team"],
 			"last-editor-uid" => ["type" => "mediumint unsigned", "foreign" => ["user" => "uid"], "comment" => "Last editor user"],
 			"assigned-uid" => ["type" => "mediumint unsigned", "foreign" => ["user" => "uid"], "comment" => "Assigned moderator user"],
-			"status" => ["type" => "tinyint unsigned", "not null" => "1", "comment" => "Status of the report, one of Entity\Report::STATUS_*"],
+			"status" => ["type" => "tinyint unsigned", "not null" => "1", "comment" => "Status of the report, one of Entity Report::STATUS_*"],
-			"resolution" => ["type" => "tinyint unsigned", "comment" => "Resolution of the report, one of Entity\Report::RESOLUTION_*"],
+			"resolution" => ["type" => "tinyint unsigned", "comment" => "Resolution of the report, one of Entity Report::RESOLUTION_*"],
 			"created" => ["type" => "datetime(6)", "not null" => "1", "default" => DBA::NULL_DATETIME6, "comment" => ""],
 			"edited" => ["type" => "datetime(6)", "comment" => "Last time the report has been edited"],
 		],
@@ -1857,7 +1860,7 @@ return [
 			"collapsed" => ["type" => "boolean", "comment" => "Posts from this contact are collapsed"],
 			"hidden" => ["type" => "boolean", "comment" => "This contact is hidden from the others"],
 			"is-blocked" => ["type" => "boolean", "comment" => "User is blocked by this contact"],
-			"channel-visibility" => ["type" => "tinyint unsigned", "comment" => "Controls the visibility in channels"],
+			"channel-frequency" => ["type" => "tinyint unsigned", "comment" => "Controls the frequency of the appearance of this contact in channels"],
 			"pending" => ["type" => "boolean", "comment" => ""],
 			"rel" => ["type" => "tinyint unsigned", "comment" => "The kind of the relation between the user and the contact"],
 			"info" => ["type" => "mediumtext", "comment" => ""],

view/js/main.js

@@ -608,6 +608,26 @@ function liveUpdate(src) {
 		update_url += '&max_id=' + getUrlParameter('max_id');
 	}
 
+	match = $("span.received").first();
+	if (match.length > 0) {
+		update_url += '&first_received=' + match[0].innerHTML;
+	}
+
+	match = $("span.created").first();
+	if (match.length > 0) {
+		update_url += '&first_created=' + match[0].innerHTML;
+	}
+
+	match = $("span.commented").first();
+	if (match.length > 0) {
+		update_url += '&first_commented=' + match[0].innerHTML;
+	}
+
+	match = $("span.uriid").first();
+	if (match.length > 0) {
+		update_url += '&first_uriid=' + match[0].innerHTML;
+	}
+
 	$.get(update_url, function(data) {
 		in_progress = false;
 		update_item = 0;

view/lang/C/messages.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: 2023.09-dev\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2023-09-15 19:05+0000\n"
+"POT-Creation-Date: 2023-09-16 04:18+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -6738,15 +6738,15 @@ msgstr ""
 msgid "Network feed not available."
 msgstr ""
 
-#: src/Module/Conversation/Timeline.php:156
+#: src/Module/Conversation/Timeline.php:155
 msgid "Own Contacts"
 msgstr ""
 
-#: src/Module/Conversation/Timeline.php:160
+#: src/Module/Conversation/Timeline.php:159
 msgid "Include"
 msgstr ""
 
-#: src/Module/Conversation/Timeline.php:161
+#: src/Module/Conversation/Timeline.php:160
 msgid "Hide"
 msgstr ""