Merge branch '1702-detect-server' of github.com:annando/friendica into 1702-detect-server

CHANGELOG

@@ -33,7 +33,6 @@ Version 3.5.1
     ping now works with JSON as well [Hypolite]
     On pending registrations, an email is now send to inform the user about it [tobiasd]
     On systems where the registration needs approval, a note for the admin can now be written [tobiasd]
-    Theme developers can now announce if their theme does support the RichText editor or not [heluecht, rabuzarus]
     Meta Information for HTML descriptions is now limited to 160 character [rabuzarus]
     Removed very old deprecated themes from the repository [silke]
     Marked frost and frost mobile as deprecated [silke]
@@ -64,7 +63,7 @@ Version 3.5.1
     2850, 2858, 2865, 2892, 2894, 2895, 2907, 2908, 2914, 2015, 2926,
     2948, 2955, 2958, 2963, 2964, 2968, 2987, 2993, 3020, 3052, 3062,
     3066, 3091, 3108, 3113, 3116, 3117, 3118, 3126, 3130, 3135, 3155,
-    3163
+    3160, 3163, 3187, 3196
 
 Version 3.5 (2016-09-13)
   Friendica Core:

boot.php

@@ -40,7 +40,7 @@ define ( 'FRIENDICA_PLATFORM',     'Friendica');
 define ( 'FRIENDICA_CODENAME',     'Asparagus');
 define ( 'FRIENDICA_VERSION',      '3.5.1-rc' );
 define ( 'DFRN_PROTOCOL_VERSION',  '2.23'    );
-define ( 'DB_UPDATE_VERSION',      1214      );
+define ( 'DB_UPDATE_VERSION',      1215      );
 
 /**
  * @brief Constant with a HTML line break.

database.sql

@@ -1,6 +1,6 @@
 -- ------------------------------------------
--- Friendica 3.5.1-dev (Asparagus)
+-- Friendica 3.5.1-rc (Asparagus)
--- DB_UPDATE_VERSION 1213
+-- DB_UPDATE_VERSION 1215
 -- ------------------------------------------
 
 
@@ -532,6 +532,7 @@ CREATE TABLE IF NOT EXISTS `item` (
 	 INDEX `parent-uri` (`parent-uri`),
 	 INDEX `extid` (`extid`),
 	 INDEX `uid_id` (`uid`,`id`),
+	 INDEX `uid_contactid_id` (`uid`,`contact-id`,`id`),
 	 INDEX `uid_created` (`uid`,`created`),
 	 INDEX `uid_unseen_contactid` (`uid`,`unseen`,`contact-id`),
 	 INDEX `uid_network_received` (`uid`,`network`,`received`),
@@ -1016,7 +1017,8 @@ CREATE TABLE IF NOT EXISTS `thread` (
 	 INDEX `uid_contactid_commented` (`uid`,`contact-id`,`commented`),
 	 INDEX `uid_contactid_created` (`uid`,`contact-id`,`created`),
 	 INDEX `uid_created` (`uid`,`created`),
-	 INDEX `uid_commented` (`uid`,`commented`)
+	 INDEX `uid_commented` (`uid`,`commented`),
+	 INDEX `uid_wall_created` (`uid`,`wall`,`created`)
 ) DEFAULT CHARSET=utf8mb4;
 
 --

include/dbstructure.php

@@ -1526,6 +1526,7 @@ function db_definition($charset) {
 					"uid_contactid_created" => array("uid","contact-id","created"),
 					"uid_created" => array("uid","created"),
 					"uid_commented" => array("uid","commented"),
+					"uid_wall_created" => array("uid","wall","created"),
 					)
 			);
 	$database["tokens"] = array(

include/discover_poco.php

@@ -186,6 +186,10 @@ function discover_directory($search) {
  */
 function gs_search_user($search) {
 
+	// Currently disabled, since the service isn't available anymore.
+	// It is not removed since I hope that there will be a successor.
+	return false;
+
 	$a = get_app();
 
 	$url = "http://gstools.org/api/users_search/".urlencode($search);

include/socgraph.php

@@ -309,7 +309,18 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 
 	logger("profile-check generation: ".$generation." Network: ".$network." URL: ".$profile_url." name: ".$name." avatar: ".$profile_photo, LOGGER_DEBUG);
 
-	poco_check_server($server_url, $network);
+	// We check the server url to be sure that it is a real one
+	$server_url2 = poco_detect_server($profile_url);
+
+	// We are no sure that it is a correct URL. So we use it in the future
+	if ($server_url2 != "") {
+		$server_url = $server_url2;
+	}
+
+	// The server URL doesn't seem to be valid, so we don't store it.
+	if (!poco_check_server($server_url, $network)) {
+		$server_url = "";
+	}
 
 	$gcontact = array("url" => $profile_url,
 			"addr" => $addr,
@@ -401,13 +412,47 @@ function poco_detect_server($profile) {
 
 	// Mastodon
 	if ($server_url == "") {
-		$red = preg_replace("=(https?://)(.*)/users/(.*)=ism", "$1$2", $profile);
+		$mastodon = preg_replace("=(https?://)(.*)/users/(.*)=ism", "$1$2", $profile);
-		if ($red != $profile) {
+		if ($mastodon != $profile) {
-			$server_url = $red;
+			$server_url = $mastodon;
 			$network = NETWORK_OSTATUS;
 		}
 	}
 
+	// Numeric OStatus variant
+	if ($server_url == "") {
+		$ostatus = preg_replace("=(https?://)(.*)/user/(.*)=ism", "$1$2", $profile);
+		if ($ostatus != $profile) {
+			$server_url = $ostatus;
+			$network = NETWORK_OSTATUS;
+		}
+	}
+
+	// Wild guess
+	if ($server_url == "") {
+		$base = preg_replace("=(https?://)(.*?)/(.*)=ism", "$1$2", $profile);
+		if (base != $profile) {
+			$server_url = $base;
+			$network = NETWORK_PHANTOM;
+		}
+	}
+
+	if ($server_url == "") {
+		return "";
+	}
+
+	$r = q("SELECT `id` FROM `gserver` WHERE `nurl` = '%s' AND `last_contact` > `last_failure`",
+		dbesc(normalise_link($server_url)));
+	if (dbm::is_result($r)) {
+		return $server_url;
+	}
+
+	// Fetch the host-meta to check if this really is a server
+	$serverret = z_fetch_url($server_url."/.well-known/host-meta");
+	if (!$serverret["success"]) {
+		return "";
+	}
+
 	return $server_url;
 }
 
@@ -424,10 +469,12 @@ function poco_last_updated($profile, $force = false) {
 		q("UPDATE `gcontact` SET `created` = '%s' WHERE `nurl` = '%s'",
 			dbesc(datetime_convert()), dbesc(normalise_link($profile)));
 
-	if ($gcontacts[0]["server_url"] != "")
+	if ($gcontacts[0]["server_url"] != "") {
 		$server_url = $gcontacts[0]["server_url"];
-	else
+	}
+	if (($server_url == '') OR ($gcontacts[0]["server_url"] == $gcontacts[0]["nurl"])) {
 		$server_url = poco_detect_server($profile);
+	}
 
 	if (!in_array($gcontacts[0]["network"], array(NETWORK_DFRN, NETWORK_DIASPORA, NETWORK_FEED, NETWORK_OSTATUS, ""))) {
 		logger("Profile ".$profile.": Network type ".$gcontacts[0]["network"]." can't be checked", LOGGER_DEBUG);
@@ -680,7 +727,49 @@ function poco_to_boolean($val) {
 	return ($val);
 }
 
-function poco_detect_friendica_server($body) {
+/**
+ * @brief Detect server type (Hubzilla or Friendica) via the poco data
+ *
+ * @param object $data POCO data
+ * @return array Server data
+ */
+function poco_detect_poco_data($data) {
+	$server = false;
+
+	if (!isset($data->entry)) {
+		return false;
+	}
+
+	if (count($data->entry) == 0) {
+		return false;
+	}
+
+	if (!isset($data->entry[0]->urls)) {
+		return false;
+	}
+
+	if (count($data->entry[0]->urls) == 0) {
+		return false;
+	}
+
+	foreach ($data->entry[0]->urls AS $url) {
+		if ($url->type == 'zot') {
+			$server = array();
+			$server["platform"] = 'Hubzilla';
+			$server["network"] = NETWORK_DIASPORA;
+			return $server;
+		}
+	}
+	return false;
+}
+
+/**
+ * @brief Detect server type (Hubzilla or Friendica) via the front page body
+ *
+ * @param string $body Front page of the server
+ * @return array Server data
+ */
+function poco_detect_server_type($body) {
 	$server = false;
 
 	$doc = new \DOMDocument();
@@ -709,6 +798,27 @@ function poco_detect_friendica_server($body) {
 		}
 	}
 
+	if (!$server) {
+		$list = $xpath->query("//meta[@property]");
+
+		foreach ($list as $node) {
+			$attr = array();
+			if ($node->attributes->length) {
+				foreach ($node->attributes as $attribute) {
+					$attr[$attribute->name] = $attribute->value;
+				}
+			}
+			if ($attr['property'] == 'generator') {
+				if (in_array($attr['content'], array("hubzilla", "BlaBlaNet"))) {
+					$server = array();
+					$server["platform"] = $attr['content'];
+					$server["version"] = "";
+					$server["network"] = NETWORK_DIASPORA;
+				}
+			}
+		}
+	}
+
 	if (!$server) {
 		return false;
 	}
@@ -793,14 +903,52 @@ function poco_check_server($server_url, $network = "", $force = false) {
 	} elseif ($network == NETWORK_DIASPORA)
 		$last_contact = datetime_convert();
 
+	// If the server has no possible failure we reset the cached data
+	if (!$possible_failure) {
+		$version = "";
+		$platform = "";
+		$site_name = "";
+		$info = "";
+		$register_policy = -1;
+	}
+
+	// Look for poco
 	if (!$failure) {
-		// Test for Diaspora
+		$serverret = z_fetch_url($server_url."/poco");
+		if ($serverret["success"]) {
+			$data = json_decode($serverret["body"]);
+			if (isset($data->totalResults)) {
+				$poco = $server_url."/poco";
+				$last_contact = datetime_convert();
+
+				$server = poco_detect_poco_data($data);
+				if ($server) {
+					$platform = $server['platform'];
+					$network = $server['network'];
+					$version = '';
+					$site_name = '';
+				}
+			}
+		}
+	}
+
+	if (!$failure) {
+		// Test for Diaspora, Hubzilla, Mastodon or older Friendica servers
 		$serverret = z_fetch_url($server_url);
 
 		if (!$serverret["success"] OR ($serverret["body"] == "")) {
 			$last_failure = datetime_convert();
 			$failure = true;
 		} else {
+			$server = poco_detect_server_type($serverret["body"]);
+			if ($server) {
+				$platform = $server['platform'];
+				$network = $server['network'];
+				$version = $server['version'];
+				$site_name = $server['site_name'];
+				$last_contact = datetime_convert();
+			}
+
 			$lines = explode("\n",$serverret["header"]);
 			if(count($lines)) {
 				foreach($lines as $line) {
@@ -824,19 +972,10 @@ function poco_check_server($server_url, $network = "", $force = false) {
 					}
 				}
 			}
-
-			$friendica_server = poco_detect_friendica_server($serverret["body"]);
-			if ($friendica_server) {
-				$platform = $friendica_server['platform'];
-				$network = $friendica_server['network'];
-				$version = $friendica_server['version'];
-				$site_name = $friendica_server['site_name'];
-				$last_contact = datetime_convert();
-			}
 		}
 	}
 
-	if (!$failure) {
+	if (!$failure AND ($poco == "")) {
 		// Test for Statusnet
 		// Will also return data for Friendica and GNU Social - but it will be overwritten later
 		// The "not implemented" is a special treatment for really, really old Friendica versions
@@ -844,7 +983,9 @@ function poco_check_server($server_url, $network = "", $force = false) {
 		if ($serverret["success"] AND ($serverret["body"] != '{"error":"not implemented"}') AND
 			($serverret["body"] != '') AND (strlen($serverret["body"]) < 30)) {
 			$platform = "StatusNet";
-			$version = trim($serverret["body"], '"');
+			// Remove junk that some GNU Social servers return
+			$version = str_replace(chr(239).chr(187).chr(191), "", $serverret["body"]);
+			$version = trim($version, '"');
 			$network = NETWORK_OSTATUS;
 			$last_contact = datetime_convert();
 		}
@@ -854,17 +995,32 @@ function poco_check_server($server_url, $network = "", $force = false) {
 		if ($serverret["success"] AND ($serverret["body"] != '{"error":"not implemented"}') AND
 			($serverret["body"] != '') AND (strlen($serverret["body"]) < 30)) {
 			$platform = "GNU Social";
-			$version = trim($serverret["body"], '"');
+			// Remove junk that some GNU Social servers return
+			$version = str_replace(chr(239).chr(187).chr(191), "", $serverret["body"]);
+			$version = trim($version, '"');
 			$network = NETWORK_OSTATUS;
 			$last_contact = datetime_convert();
 		}
+	}
 
+	if (!$failure) {
+		// Test for Hubzilla, Redmatrix or Friendica
 		$serverret = z_fetch_url($server_url."/api/statusnet/config.json");
 		if ($serverret["success"]) {
 			$data = json_decode($serverret["body"]);
 			if (isset($data->site->server)) {
 				$last_contact = datetime_convert();
 
+				if (isset($data->site->platform)) {
+					$platform = $data->site->platform->PLATFORM_NAME;
+					$version = $data->site->platform->STD_VERSION;
+					$network = NETWORK_DIASPORA;
+				}
+				if (isset($data->site->BlaBlaNet)) {
+					$platform = $data->site->BlaBlaNet->PLATFORM_NAME;
+					$version = $data->site->BlaBlaNet->STD_VERSION;
+					$network = NETWORK_DIASPORA;
+				}
 				if (isset($data->site->hubzilla)) {
 					$platform = $data->site->hubzilla->PLATFORM_NAME;
 					$version = $data->site->hubzilla->RED_VERSION;
@@ -965,18 +1121,6 @@ function poco_check_server($server_url, $network = "", $force = false) {
 		}
 	}
 
-	// Look for poco
-	if (!$failure) {
-		$serverret = z_fetch_url($server_url."/poco");
-		if ($serverret["success"]) {
-			$data = json_decode($serverret["body"]);
-			if (isset($data->totalResults)) {
-				$poco = $server_url."/poco";
-				$last_contact = datetime_convert();
-			}
-		}
-	}
-
 	if ($possible_failure AND !$failure) {
 		$last_failure = datetime_convert();
 		$failure = true;
@@ -1002,7 +1146,7 @@ function poco_check_server($server_url, $network = "", $force = false) {
 	$info = strip_tags($info);
 	$platform = strip_tags($platform);
 
-	if ($servers)
+	if ($servers) {
 		 q("UPDATE `gserver` SET `url` = '%s', `version` = '%s', `site_name` = '%s', `info` = '%s', `register_policy` = %d, `poco` = '%s', `noscrape` = '%s',
 			`network` = '%s', `platform` = '%s', `last_contact` = '%s', `last_failure` = '%s' WHERE `nurl` = '%s'",
 			dbesc($server_url),
@@ -1018,7 +1162,7 @@ function poco_check_server($server_url, $network = "", $force = false) {
 			dbesc($last_failure),
 			dbesc(normalise_link($server_url))
 		);
-	else
+	} elseif (!$failure)
 		q("INSERT INTO `gserver` (`url`, `nurl`, `version`, `site_name`, `info`, `register_policy`, `poco`, `noscrape`, `network`, `platform`, `created`, `last_contact`, `last_failure`)
 					VALUES ('%s', '%s', '%s', '%s', '%s', %d, '%s', '%s', '%s', '%s', '%s', '%s', '%s')",
 				dbesc($server_url),
@@ -1316,18 +1460,20 @@ function poco_discover_federation() {
 			poco_check_server("https://".$server->host);
 	}
 
-	// Discover GNU Social Servers
+	// Currently disabled, since the service isn't available anymore.
-	if (!get_config('system','ostatus_disabled')) {
+	// It is not removed since I hope that there will be a successor.
-		$serverdata = "http://gstools.org/api/get_open_instances/";
+	// Discover GNU Social Servers.
+	//if (!get_config('system','ostatus_disabled')) {
+	//	$serverdata = "http://gstools.org/api/get_open_instances/";
 
-		$result = z_fetch_url($serverdata);
+	//	$result = z_fetch_url($serverdata);
-		if ($result["success"]) {
+	//	if ($result["success"]) {
-			$servers = json_decode($result["body"]);
+	//		$servers = json_decode($result["body"]);
 
-			foreach($servers->data AS $server)
+	//		foreach($servers->data AS $server)
-				poco_check_server($server->instance_address);
+	//			poco_check_server($server->instance_address);
-		}
+	//	}
-	}
+	//}
 
 	set_config('poco','last_federation_discovery', time());
 }
@@ -1930,4 +2076,25 @@ function gs_discover() {
 		q("UPDATE `gserver` SET `last_poco_query` = '%s' WHERE `nurl` = '%s'", dbesc(datetime_convert()), dbesc($server["nurl"]));
 	}
 }
+
+/**
+ * @brief Returns a list of all known servers
+ * @return array List of server urls
+ */
+function poco_serverlist() {
+	$r = q("SELECT `id`, `url`, `site_name` AS `displayName`, `network`, `platform`, `version` FROM `gserver`
+		WHERE `network` IN ('%s', '%s', '%s') AND `last_contact` > `last_failure`
+		ORDER BY `last_contact`
+		LIMIT 1000",
+		dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_OSTATUS));
+	if (!dbm::is_result($r)) {
+		return false;
+	}
+	$list = array();
+	foreach ($r AS $server) {
+		$server['id'] = (int)$server['id'];
+		$list[] = $server;
+	}
+	return $list;
+}
 ?>

mod/notifications.php

@@ -194,10 +194,14 @@ function notifications_content(App $a) {
 						if($it['network'] === NETWORK_DFRN) {
 							$lbl_knowyou = t('Claims to be known to you: ');
 							$knowyou = (($it['knowyou']) ? t('yes') : t('no'));
-							$helptext = t('Shall your connection be bidirectional or not? "Friend" implies that you allow to read and you subscribe to their posts. "Fan/Admirer" means that you allow to read but you do not want to read theirs. Approve as: ');
+							$helptext = t('Shall your connection be bidirectional or not?');
+							$helptext2 = sprintf(t('Accepting %s as a friend allows %s to subscribe to your posts, and you will also receive updates from them in your news feed.'), $it['name'], $it['name']);
+							$helptext3 = sprintf(t('Accepting %s as a subscriber allows them to subscribe to your posts, but you will not receive updates from them in your news feed.'), $it['name']);
 						} else {
 							$knowyou = '';
-							$helptext = t('Shall your connection be bidirectional or not? "Friend" implies that you allow to read and you subscribe to their posts. "Sharer" means that you allow to read but you do not want to read theirs. Approve as: ');
+							$helptext = t('Shall your connection be bidirectional or not?');
+							$helptext2 = sprintf(t('Accepting %s as a friend allows %s to subscribe to your posts, and you will also receive updates from them in your news feed.'), $it['name'], $it['name']);
+							$helptext3 = sprintf(t('Accepting %s as a sharer allows them to subscribe to your posts, but you will not receive updates from them in your news feed.'), $it['name']);
 						}
 					}
 
@@ -205,9 +209,11 @@ function notifications_content(App $a) {
 						'$intro_id' => $it['intro_id'],
 						'$friend_selected' => $friend_selected,
 						'$fan_selected' => $fan_selected,
-						'$approve_as' => $helptext,
+						'$approve_as1' => $helptext,
+						'$approve_as2' => $helptext2,
+						'$approve_as3' => $helptext3,
 						'$as_friend' => t('Friend'),
-						'$as_fan' => (($it['network'] == NETWORK_DIASPORA) ? t('Sharer') : t('Fan/Admirer'))
+						'$as_fan' => (($it['network'] == NETWORK_DIASPORA) ? t('Sharer') : t('Subscriber'))
 					));
 
 					$header = $it["name"];

mod/poco.php

@@ -27,6 +27,12 @@ function poco_init(App $a) {
 	$justme = false;
 	$global = false;
 
+	if($a->argc > 1 && $a->argv[1] === '@server') {
+		$ret = poco_serverlist();
+		header('Content-type: application/json');
+		echo json_encode($ret);
+		killme();
+	}
 	if($a->argc > 1 && $a->argv[1] === '@global') {
 		$global = true;
 		$update_limit = date("Y-m-d H:i:s", time() - 30 * 86400);
@@ -314,11 +320,9 @@ function poco_init(App $a) {
 
 				$ret['entry'][] = $entry;
 			}
-		}
+		} else
-		else
 			$ret['entry'][] = array();
-	}
+	} else
-	else
 		http_status_exit(500);
 
 	logger("End of poco", LOGGER_DEBUG);
@@ -332,8 +336,7 @@ function poco_init(App $a) {
 		header('Content-type: application/json');
 		echo json_encode($ret);
 		killme();
-	}
+	} else
-	else
 		http_status_exit(500);
 
 

mod/profile.php

@@ -240,6 +240,17 @@ function profile_content(App $a, $update = 0) {
 			$sql_extra2 .= protect_sprintf(sprintf(" AND `thread`.`created` >= '%s' ", dbesc(datetime_convert(date_default_timezone_get(),'',$datequery2))));
 		}
 
+		// Belongs the profile page to a forum?
+		// If not then we can improve the performance with an additional condition
+		$r = q("SELECT `uid` FROM `user` WHERE `uid` = %d AND `page-flags` IN (%d, %d)",
+			intval($a->profile['profile_uid']),
+			intval(PAGE_COMMUNITY),
+			intval(PAGE_PRVGROUP));
+
+		if (!dbm::is_result($r)) {
+			$sql_extra3 = sprintf(" AND `thread`.`contact-id` = %d ", intval(intval($a->profile['contact_id'])));
+		}
+
 		if(get_config('system', 'old_pager')) {
 		    $r = q("SELECT COUNT(*) AS `total`
 			    FROM `thread` INNER JOIN `item` ON `item`.`id` = `thread`.`iid`
@@ -248,7 +259,7 @@ function profile_content(App $a, $update = 0) {
 			    WHERE `thread`.`uid` = %d AND `thread`.`visible` = 1 AND `thread`.`deleted` = 0
 			    and `thread`.`moderated` = 0
 			    AND `thread`.`wall` = 1
-			    $sql_extra $sql_extra2 ",
+			    $sql_extra3 $sql_extra $sql_extra2 ",
 			    intval($a->profile['profile_uid'])
 			);
 
@@ -282,14 +293,12 @@ function profile_content(App $a, $update = 0) {
 			STRAIGHT_JOIN `contact` ON `contact`.`id` = `thread`.`contact-id`
 				AND NOT `contact`.`blocked` AND NOT `contact`.`pending`
 			WHERE `thread`.`uid` = %d AND `thread`.`visible`
-				AND `thread`.`contact-id` = %d
 				AND NOT `thread`.`deleted`
 				AND NOT `thread`.`moderated`
 				AND `thread`.`wall`
-				$sql_extra $sql_extra2
+				$sql_extra3 $sql_extra $sql_extra2
 			ORDER BY `thread`.`created` DESC $pager_sql",
-			intval($a->profile['profile_uid']),
+			intval($a->profile['profile_uid'])
-			intval($a->profile['contact_id'])
 		);
 	}
 

update.php

@@ -1,6 +1,6 @@
 <?php
 
-define('UPDATE_VERSION' , 1214);
+define('UPDATE_VERSION' , 1215);
 
 /**
  *

view/templates/netfriend.tpl

@@ -1,5 +1,9 @@
 
-<div class="intro-approve-as-friend-desc">{{$approve_as}}</div>
+<div class="intro-approve-as-friend-desc">
+  <p>{{$approve_as1}}</p>
+  <p>{{$approve_as2}}</p>
+  <p>{{$approve_as3}}</p>
+</div>
 
 <div class="intro-approve-as-friend-wrapper">
 	<label class="intro-approve-as-friend-label" for="intro-approve-as-friend-{{$intro_id}}">{{$as_friend}}</label>