Merge pull request #2836 from Hypolite/Issue-#2816-2

Fix Issue #2816 - Unable to save config items present in .htconfig.php but not in DB

boot.php

@@ -38,7 +38,7 @@ define ( 'FRIENDICA_PLATFORM',     'Friendica');
 define ( 'FRIENDICA_CODENAME',     'Asparagus');
 define ( 'FRIENDICA_VERSION',      '3.5.1-dev' );
 define ( 'DFRN_PROTOCOL_VERSION',  '2.23'    );
-define ( 'DB_UPDATE_VERSION',      1204      );
+define ( 'DB_UPDATE_VERSION',      1205      );
 
 /**
  * @brief Constant with a HTML line break.

database.sql

@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 3.5.1-dev (Asparagus)
--- DB_UPDATE_VERSION 1204
+-- DB_UPDATE_VERSION 1205
 -- ------------------------------------------
 
 
@@ -97,7 +97,7 @@ CREATE TABLE IF NOT EXISTS `config` (
 	`k` varchar(255) NOT NULL DEFAULT '',
 	`v` text,
 	 PRIMARY KEY(`id`),
-	 INDEX `cat_k` (`cat`(30),`k`(30))
+	 UNIQUE INDEX `cat_k` (`cat`(30),`k`(30))
 ) DEFAULT CHARSET=utf8mb4;
 
 --
@@ -707,7 +707,7 @@ CREATE TABLE IF NOT EXISTS `pconfig` (
 	`k` varchar(255) NOT NULL DEFAULT '',
 	`v` mediumtext,
 	 PRIMARY KEY(`id`),
-	 INDEX `uid_cat_k` (`uid`,`cat`(30),`k`(30))
+	 UNIQUE INDEX `uid_cat_k` (`uid`,`cat`(30),`k`(30))
 ) DEFAULT CHARSET=utf8mb4;
 
 --

doc/upgrade.md

@@ -0,0 +1,34 @@
+# Considerations before upgrading Friendica
+
+* [Home](help)
+
+## MySQL >= 5.7.4
+
+Starting from MySQL version 5.7.4, the IGNORE keyword in ALTER TABLE statements is ignored.
+This prevents automatic table deduplication if a UNIQUE index is added to a Friendica table's structure.
+If a DB update fails for you while creating a UNIQUE index, make sure to manually deduplicate the table before trying the update again.
+
+### Manual deduplication
+
+There are two main ways of doing it, either by manually removing the duplicates or by recreating the table.
+Manually removing the duplicates is usually faster if they're not too numerous.
+To manually remove the duplicates, you need to know the UNIQUE index columns available in `database.sql`.
+
+```SQL
+SELECT GROUP_CONCAT(id), <index columns>, count(*) as count FROM users
+GROUP BY <index columns> HAVING count >= 2;
+
+/* delete or merge duplicate from above query */;
+```
+
+If there are too many rows to handle manually, you can create a new table with the same structure as the table with duplicates and insert the existing content with INSERT IGNORE.
+To recreate the table you need to know the table structure available in `database.sql`.
+
+```SQL
+CREATE TABLE <table_name>_new <rest of the CREATE TABLE>;
+INSERT IGNORE INTO <table_name>_new SELECT * FROM <table_name>;
+DROP TABLE <table_name>;
+RENAME TABLE <table_name>_new TO <table_name>;
+```
+
+This method is slower overall, but it is better suited for large numbers of duplicates.

include/Core/Config.php

@@ -126,37 +126,19 @@ class Config {
 	public static function set($family,$key,$value) {
 		global $a;
 
-		// If $a->config[$family] has been previously set to '!<unset>!', then
+		$a->config[$family][$key] = $value;
-		// $a->config[$family][$key] will evaluate to $a->config[$family][0], and
-		// $a->config[$family][$key] = $value will be equivalent to
-		// $a->config[$family][0] = $value[0] (this causes infuriating bugs),
-		// so unset the family before assigning a value to a family's key
-		if($a->config[$family] === '!<unset>!')
-			unset($a->config[$family]);
 
 		// manage array value
 		$dbvalue = (is_array($value)?serialize($value):$value);
 		$dbvalue = (is_bool($dbvalue) ? intval($dbvalue) : $dbvalue);
-		if(is_null(self::get($family,$key,null,true))) {
-			$a->config[$family][$key] = $value;
-			$ret = q("INSERT INTO `config` ( `cat`, `k`, `v` ) VALUES ( '%s', '%s', '%s' ) ",
-				dbesc($family),
-				dbesc($key),
-				dbesc($dbvalue)
-			);
-			if($ret)
-				return $value;
-			return $ret;
-		}
 
-		$ret = q("UPDATE `config` SET `v` = '%s' WHERE `cat` = '%s' AND `k` = '%s'",
+		$ret = q("INSERT INTO `config` ( `cat`, `k`, `v` ) VALUES ( '%s', '%s', '%s' )
-			dbesc($dbvalue),
+ON DUPLICATE KEY UPDATE `v` = '%s'",
 			dbesc($family),
-			dbesc($key)
+			dbesc($key),
+			dbesc($dbvalue),
+			dbesc($dbvalue)
 		);
-
-		$a->config[$family][$key] = $value;
-
 		if($ret)
 			return $value;
 		return $ret;

include/Core/PConfig.php

@@ -126,27 +126,16 @@ class PConfig {
 		// manage array value
 		$dbvalue = (is_array($value)?serialize($value):$value);
 
-		if(is_null(self::get($uid,$family,$key,null, true))) {
-			$a->config[$uid][$family][$key] = $value;
-			$ret = q("INSERT INTO `pconfig` ( `uid`, `cat`, `k`, `v` ) VALUES ( %d, '%s', '%s', '%s' ) ",
-				intval($uid),
-				dbesc($family),
-				dbesc($key),
-				dbesc($dbvalue)
-			);
-			if($ret)
-				return $value;
-			return $ret;
-		}
-		$ret = q("UPDATE `pconfig` SET `v` = '%s' WHERE `uid` = %d AND `cat` = '%s' AND `k` = '%s'",
-			dbesc($dbvalue),
-			intval($uid),
-			dbesc($family),
-			dbesc($key)
-		);
-
 		$a->config[$uid][$family][$key] = $value;
 
+		$ret = q("INSERT INTO `pconfig` ( `uid`, `cat`, `k`, `v` ) VALUES ( %d, '%s', '%s', '%s' )
+ON DUPLICATE KEY UPDATE `v` = '%s'",
+			intval($uid),
+			dbesc($family),
+			dbesc($key),
+			dbesc($dbvalue),
+			dbesc($dbvalue)
+		);
 		if($ret)
 			return $value;
 		return $ret;

include/dba.php

@@ -91,6 +91,23 @@ class dba {
 		return $this->db;
 	}
 
+	/**
+	 * @brief Returns the MySQL server version string
+	 * 
+	 * This function discriminate between the deprecated mysql API and the current
+	 * object-oriented mysqli API. Example of returned string: 5.5.46-0+deb8u1
+	 *
+	 * @return string
+	 */
+	public function server_info() {
+		if ($this->mysqli) {
+			$return = $this->db->server_info;
+		} else {
+			$return = mysql_get_server_info($this->db);
+		}
+		return $return;
+	}
+
 	public function q($sql, $onlyquery = false) {
 		global $a;
 

include/dbstructure.php

@@ -78,6 +78,10 @@ function table_structure($table) {
 			if ($index["Index_type"] == "FULLTEXT")
 				continue;
 
+			if ($index['Key_name'] != 'PRIMARY' && $index['Non_unique'] == '0' && !isset($indexdata[$index["Key_name"]])) {
+				$indexdata[$index["Key_name"]] = array('UNIQUE');
+			}
+
 			$column = $index["Column_name"];
 			// On utf8mb4 a varchar index can only have a length of 191
 			// To avoid the need to add this to every index definition we just ignore it here.
@@ -154,6 +158,19 @@ function update_structure($verbose, $action, $tables=null, $definition=null) {
 	if (is_null($definition))
 		$definition = db_definition($charset);
 
+	// Ensure index conversion to unique removes duplicates
+	$sql_config = "SET session old_alter_table=1;";
+	if ($verbose)
+		echo $sql_config."\n";
+	if ($action)
+		@$db->q($sql_config);
+
+	// MySQL >= 5.7.4 doesn't support the IGNORE keyword in ALTER TABLE statements
+	if (version_compare($db->server_info(), '5.7.4') >= 0) {
+		$ignore = '';
+	}else {
+		$ignore = ' IGNORE';
+	}
 
 	// Compare it
 	foreach ($definition AS $name => $structure) {
@@ -223,7 +240,7 @@ function update_structure($verbose, $action, $tables=null, $definition=null) {
 					$sql2=db_create_index($indexname, $fieldnames);
 					if ($sql2 != "") {
 						if ($sql3 == "")
-							$sql3 = "ALTER TABLE `".$name."` ".$sql2;
+							$sql3 = "ALTER" . $ignore . " TABLE `".$name."` ".$sql2;
 						else
 							$sql3 .= ", ".$sql2;
 					}
@@ -330,6 +347,11 @@ function db_create_index($indexname, $fieldnames, $method="ADD") {
 		killme();
 	}
 
+	if ($fieldnames[0] == "UNIQUE") {
+		array_shift($fieldnames);
+		$method .= ' UNIQUE';
+	}
+
 	$names = "";
 	foreach ($fieldnames AS $fieldname) {
 		if ($names != "")
@@ -457,7 +479,7 @@ function db_definition($charset) {
 					),
 			"indexes" => array(
 					"PRIMARY" => array("id"),
-					"cat_k" => array("cat(30)","k(30)"),
+					"cat_k" => array("UNIQUE", "cat(30)","k(30)"),
 					)
 			);
 	$database["contact"] = array(
@@ -1067,7 +1089,7 @@ function db_definition($charset) {
 					),
 			"indexes" => array(
 					"PRIMARY" => array("id"),
-					"uid_cat_k" => array("uid","cat(30)","k(30)"),
+					"uid_cat_k" => array("UNIQUE", "uid","cat(30)","k(30)"),
 					)
 			);
 	$database["photo"] = array(
@@ -1491,6 +1513,9 @@ function dbstructure_run(&$argv, &$argc) {
 
 	if ($argc==2) {
 		switch ($argv[1]) {
+			case "dryrun":
+				update_structure(true, false);
+				return;
 			case "update":
 				update_structure(true, true);
 
@@ -1523,7 +1548,8 @@ function dbstructure_run(&$argv, &$argc) {
 	// print help
 	echo $argv[0]." <command>\n";
 	echo "\n";
-	echo "commands:\n";
+	echo "Commands:\n";
+	echo "dryrun		show database update schema queries without running them\n";
 	echo "update		update database schema\n";
 	echo "dumpsql		dump database schema\n";
 	return;

update.php

@@ -1,6 +1,6 @@
 <?php
 
-define('UPDATE_VERSION' , 1204);
+define('UPDATE_VERSION' , 1205);
 
 /**
  *