From dbbe6efd27cde052402e1d9995f77feb95d1c265 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Wed, 12 Apr 2017 00:19:05 +0200 Subject: [PATCH 01/14] Fix ostatus bug related to only_full_group_by https://github.com/friendica/friendica/issues/3322 --- include/ostatus.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/ostatus.php b/include/ostatus.php index e0b33a1a4..ee405397c 100644 --- a/include/ostatus.php +++ b/include/ostatus.php @@ -720,7 +720,7 @@ class ostatus { $conversations = q("SELECT `term`.`oid`, `term`.`url`, `term`.`uid` FROM `term` STRAIGHT_JOIN `thread` ON `thread`.`iid` = `term`.`oid` AND `thread`.`uid` = `term`.`uid` WHERE `term`.`type` = 7 AND `term`.`term` > '%s' AND `thread`.`mention` - GROUP BY `term`.`url`, `term`.`uid` ORDER BY `term`.`term` DESC", dbesc($start)); + GROUP BY `term`.`url`, `term`.`uid`, `term`.`oid`, `term`.`term` ORDER BY `term`.`term` DESC", dbesc($start)); } else { $conversations = q("SELECT `oid`, `url`, `uid` FROM `term` WHERE `type` = 7 AND `term` > '%s' From 15a44d945b9219a75e519e6cb4b0563b6ea378f7 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Wed, 12 Apr 2017 01:06:08 +0200 Subject: [PATCH 02/14] Another GROUP BY fix for MySQL https://github.com/friendica/friendica/issues/3322 --- include/ostatus.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/ostatus.php b/include/ostatus.php index ee405397c..e0ed1df19 100644 --- a/include/ostatus.php +++ b/include/ostatus.php @@ -724,7 +724,7 @@ class ostatus { } else { $conversations = q("SELECT `oid`, `url`, `uid` FROM `term` WHERE `type` = 7 AND `term` > '%s' - GROUP BY `url`, `uid` ORDER BY `term` DESC", dbesc($start)); + GROUP BY `url`, `uid`, `oid`, `term` ORDER BY `term` DESC", dbesc($start)); } foreach ($conversations AS $conversation) { From c3e933642e6ad761c91888006b45efa69915d691 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Wed, 12 Apr 2017 14:17:16 +0200 Subject: [PATCH 03/14] Fix missing SQL group by in ping https://github.com/friendica/friendica/issues/3322 --- mod/ping.php | 1 + 1 file changed, 1 insertion(+) diff --git a/mod/ping.php b/mod/ping.php index b5330c7b3..0b82e71d3 100644 --- a/mod/ping.php +++ b/mod/ping.php @@ -201,6 +201,7 @@ function ping_init(App $a) if (is_null($ev)) { $ev = qu("SELECT count(`event`.`id`) AS total, type, start, adjust FROM `event` WHERE `event`.`uid` = %d AND `start` < '%s' AND `finish` > '%s' and `ignore` = 0 + GROUP BY type, start, adjust ORDER BY `start` ASC ", intval(local_user()), dbesc(datetime_convert('UTC', 'UTC', 'now + 7 days')), From 74b6d09e89bbe87e94ebb458d8a76415e324d1d3 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Wed, 12 Apr 2017 15:11:50 +0200 Subject: [PATCH 04/14] Fix GROUP BY in acl_selector, api, notifier, photos, messages https://github.com/friendica/friendica/issues/3322 --- include/acl_selectors.php | 4 ++-- include/api.php | 8 +++++--- include/notifier.php | 2 +- include/photos.php | 4 ++-- mod/message.php | 4 ++-- mod/photos.php | 2 +- mod/videos.php | 2 +- 7 files changed, 14 insertions(+), 12 deletions(-) diff --git a/include/acl_selectors.php b/include/acl_selectors.php index f4b644d68..87230a140 100644 --- a/include/acl_selectors.php +++ b/include/acl_selectors.php @@ -502,7 +502,7 @@ function acl_lookup(App $a, $out_type = 'json') { INNER JOIN `group_member` ON `group_member`.`gid`=`group`.`id` AND `group_member`.`uid` = `group`.`uid` WHERE NOT `group`.`deleted` AND `group`.`uid` = %d $sql_extra - GROUP BY `group`.`name` + GROUP BY `group`.`name`, `group`.`id` ORDER BY `group`.`name` LIMIT %d,%d", intval(local_user()), @@ -619,7 +619,7 @@ function acl_lookup(App $a, $out_type = 'json') { FROM `item` WHERE `parent` = %d AND (`author-name` LIKE '%%%s%%' OR `author-link` LIKE '%%%s%%') AND `author-link` NOT IN ('%s') - GROUP BY `author-link` + GROUP BY `author-link`, `author-avatar`, `author-name` ORDER BY `author-name` ASC ", intval($conv_id), diff --git a/include/api.php b/include/api.php index 9fc853340..477eb94b4 100644 --- a/include/api.php +++ b/include/api.php @@ -3064,7 +3064,7 @@ use \Friendica\Core\Config; function api_fr_photos_list($type) { if (api_user()===false) throw new ForbiddenException(); $r = q("select `resource-id`, max(scale) as scale, album, filename, type from photo - where uid = %d and album != 'Contact Photos' group by `resource-id`", + where uid = %d and album != 'Contact Photos' group by `resource-id`, album, filename, type", intval(local_user()) ); $typetoext = array( @@ -3102,8 +3102,10 @@ use \Friendica\Core\Config; $data_sql = ($scale === false ? "" : "data, "); $r = q("select %s `resource-id`, `created`, `edited`, `title`, `desc`, `album`, `filename`, - `type`, `height`, `width`, `datasize`, `profile`, min(`scale`) as minscale, max(`scale`) as maxscale - from photo where `uid` = %d and `resource-id` = '%s' %s group by `resource-id`", + `type`, `height`, `width`, `datasize`, `profile`, min(`scale`) as minscale, max(`scale`) as maxscale + from photo where `uid` = %d and `resource-id` = '%s' %s + group by `resource-id`, `created`, `edited`, `title`, `desc`, `album`, `filename`, + `type`, `height`, `width`, `datasize`, `profile`", $data_sql, intval(local_user()), dbesc($_REQUEST['photo_id']), diff --git a/include/notifier.php b/include/notifier.php index e3d7d10d6..ed34288e8 100644 --- a/include/notifier.php +++ b/include/notifier.php @@ -517,7 +517,7 @@ function notifier_run(&$argv, &$argc){ } $r1 = q("SELECT DISTINCT(`batch`), `id`, `name`,`network` FROM `contact` WHERE `network` = '%s' - AND `uid` = %d AND `rel` != %d AND NOT `blocked` AND NOT `pending` AND NOT `archive` GROUP BY `batch` ORDER BY rand()", + AND `uid` = %d AND `rel` != %d AND NOT `blocked` AND NOT `pending` AND NOT `archive` GROUP BY `batch`, `id` ORDER BY rand()", dbesc(NETWORK_DIASPORA), intval($owner['uid']), intval(CONTACT_IS_SHARING) diff --git a/include/photos.php b/include/photos.php index 9d8d3309c..02d1b1740 100644 --- a/include/photos.php +++ b/include/photos.php @@ -51,7 +51,7 @@ function photo_albums($uid, $update = false) { $albums = qu("SELECT COUNT(DISTINCT `resource-id`) AS `total`, `album` FROM `photo` WHERE `uid` = %d AND `album` != '%s' AND `album` != '%s' $sql_extra - GROUP BY `album` ORDER BY `created` DESC", + GROUP BY `album`, `created` ORDER BY `created` DESC", intval($uid), dbesc('Contact Photos'), dbesc(t('Contact Photos')) @@ -61,7 +61,7 @@ function photo_albums($uid, $update = false) { $albums = qu("SELECT DISTINCT(`album`), '' AS `total` FROM `photo` WHERE `uid` = %d AND `album` != '%s' AND `album` != '%s' $sql_extra - GROUP BY `album` ORDER BY `created` DESC", + GROUP BY `album`, `created` ORDER BY `created` DESC", intval($uid), dbesc('Contact Photos'), dbesc(t('Contact Photos')) diff --git a/mod/message.php b/mod/message.php index 9e9669146..05a8d36f6 100644 --- a/mod/message.php +++ b/mod/message.php @@ -350,7 +350,7 @@ function message_content(App $a) { $o .= $header; $r = q("SELECT count(*) AS `total` FROM `mail` - WHERE `mail`.`uid` = %d GROUP BY `parent-uri` ORDER BY `created` DESC", + WHERE `mail`.`uid` = %d GROUP BY `parent-uri`, `created` ORDER BY `created` DESC", intval(local_user()) ); @@ -533,7 +533,7 @@ function get_messages($user, $lstart, $lend) { `mail`.* , `contact`.`name`, `contact`.`url`, `contact`.`thumb` , `contact`.`network`, count( * ) as count FROM `mail` LEFT JOIN `contact` ON `mail`.`contact-id` = `contact`.`id` - WHERE `mail`.`uid` = %d GROUP BY `parent-uri` ORDER BY `mailcreated` DESC LIMIT %d , %d ", + WHERE `mail`.`uid` = %d GROUP BY `parent-uri`, `mail`.id ORDER BY `mailcreated` DESC LIMIT %d , %d ", intval($user), intval($lstart), intval($lend) ); } diff --git a/mod/photos.php b/mod/photos.php index 3acd39b2a..a24cee255 100644 --- a/mod/photos.php +++ b/mod/photos.php @@ -1241,7 +1241,7 @@ function photos_content(App $a) { } $r = q("SELECT `resource-id`, `id`, `filename`, type, max(`scale`) AS `scale`, `desc` FROM `photo` WHERE `uid` = %d AND `album` = '%s' - AND `scale` <= 4 $sql_extra GROUP BY `resource-id` ORDER BY `created` $order LIMIT %d , %d", + AND `scale` <= 4 $sql_extra GROUP BY `resource-id`, `id` ORDER BY `created` $order LIMIT %d , %d", intval($owner_uid), dbesc($album), intval($a->pager['start']), diff --git a/mod/videos.php b/mod/videos.php index 3828b8f1f..ea2e0d4a2 100644 --- a/mod/videos.php +++ b/mod/videos.php @@ -358,7 +358,7 @@ function videos_content(App $a) { $r = q("SELECT hash, `id`, `filename`, filetype FROM `attach` WHERE `uid` = %d AND filetype LIKE '%%video%%' - $sql_extra GROUP BY hash ORDER BY `created` DESC LIMIT %d , %d", + $sql_extra GROUP BY hash, `id` ORDER BY `created` DESC LIMIT %d , %d", intval($a->data['user']['uid']), intval($a->pager['start']), intval($a->pager['itemspage']) From 9fb4ba2b733d59b8a704cb8df846c3b43769503d Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Wed, 12 Apr 2017 18:49:29 +0200 Subject: [PATCH 05/14] Fix profile wrong DISTINCT + ORDER BY Fix: ERROR 3065 (HY000) Expression #1 of ORDER BY clause is not in SELECT list, references column 'friendica.item.created' which is not in SELECT list; this is incompatible with DISTINCT --- mod/profile.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mod/profile.php b/mod/profile.php index fbce509d2..a83cb076f 100644 --- a/mod/profile.php +++ b/mod/profile.php @@ -210,7 +210,7 @@ function profile_content(App $a, $update = 0) { if ($update) { - $r = q("SELECT distinct(parent) AS `item_id`, `item`.`network` AS `item_network` + $r = q("SELECT distinct(parent) AS `item_id`, `item`.`network` AS `item_network`, `item`.`created` FROM `item` INNER JOIN `contact` ON `contact`.`id` = `item`.`contact-id` AND `contact`.`blocked` = 0 AND `contact`.`pending` = 0 WHERE `item`.`uid` = %d AND `item`.`visible` = 1 AND From 534ea6aefcc480bcd01cfd478cc7f75d2f4373d2 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Wed, 12 Apr 2017 18:54:54 +0200 Subject: [PATCH 06/14] Fix ping_init SQL According to review https://github.com/friendica/friendica/pull/3323#pullrequestreview-32401628 --- mod/ping.php | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mod/ping.php b/mod/ping.php index 0b82e71d3..ba496a70b 100644 --- a/mod/ping.php +++ b/mod/ping.php @@ -199,9 +199,8 @@ function ping_init(App $a) $cachekey = "ping_init:".local_user(); $ev = Cache::get($cachekey); if (is_null($ev)) { - $ev = qu("SELECT count(`event`.`id`) AS total, type, start, adjust FROM `event` + $ev = qu("SELECT type, start, adjust FROM `event` WHERE `event`.`uid` = %d AND `start` < '%s' AND `finish` > '%s' and `ignore` = 0 - GROUP BY type, start, adjust ORDER BY `start` ASC ", intval(local_user()), dbesc(datetime_convert('UTC', 'UTC', 'now + 7 days')), @@ -213,7 +212,7 @@ function ping_init(App $a) } if (dbm::is_result($ev)) { - $all_events = intval($ev[0]['total']); + $all_events = count($ev); if ($all_events) { $str_now = datetime_convert('UTC', $a->timezone, 'now', 'Y-m-d'); From 38e7a0f7933461702275a846a85e9dc3ae5dd97e Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Wed, 12 Apr 2017 22:55:02 +0200 Subject: [PATCH 07/14] Fix GROUP BY for search https://github.com/friendica/friendica/issues/3322 Fix MySQL ERROR 1055 (42000): Expression #1 of SELECT list is not in GROUP BY clause and contains nonaggregated column 'friendica.item.author-id' which is not functionally dependent on columns in GROUP BY clause; this is incompatible with sql_mode=only_full_group_by --- mod/search.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mod/search.php b/mod/search.php index 7d588aa4d..f274b2a87 100644 --- a/mod/search.php +++ b/mod/search.php @@ -214,7 +214,7 @@ function search_content(App $a) { FROM `item` %s WHERE %s AND (`item`.`uid` = 0 OR (`item`.`uid` = %s AND NOT `item`.`global`)) $sql_extra - GROUP BY `item`.`uri` ORDER BY `item`.`id` DESC LIMIT %d , %d", + GROUP BY `item`.`uri`, `item`.`id` ORDER BY `item`.`id` DESC LIMIT %d , %d", item_fieldlists(), item_joins(), item_condition(), intval(local_user()), intval($a->pager['start']), intval($a->pager['itemspage'])); From 1058b28ceabcec802932fc33bb79e6ce7814f945 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sat, 15 Apr 2017 00:42:44 +0200 Subject: [PATCH 08/14] MySQL ANY_VALUE with fallback to MIN https://github.com/friendica/friendica/issues/3322 --- include/api.php | 13 +++++++------ include/dba.php | 10 ++++++++++ include/notifier.php | 5 +++-- include/photos.php | 8 ++++---- mod/message.php | 16 ++++++++++------ mod/photos.php | 6 ++++-- mod/videos.php | 6 ++++-- 7 files changed, 42 insertions(+), 22 deletions(-) diff --git a/include/api.php b/include/api.php index 477eb94b4..75f8ab069 100644 --- a/include/api.php +++ b/include/api.php @@ -3099,13 +3099,14 @@ use \Friendica\Core\Config; $scale = (x($_REQUEST, 'scale') ? intval($_REQUEST['scale']) : false); $scale_sql = ($scale === false ? "" : sprintf("and scale=%d",intval($scale))); - $data_sql = ($scale === false ? "" : "data, "); + $data_sql = ($scale === false ? "" : "ANY_VALUE(data) AS data,"); - $r = q("select %s `resource-id`, `created`, `edited`, `title`, `desc`, `album`, `filename`, - `type`, `height`, `width`, `datasize`, `profile`, min(`scale`) as minscale, max(`scale`) as maxscale - from photo where `uid` = %d and `resource-id` = '%s' %s - group by `resource-id`, `created`, `edited`, `title`, `desc`, `album`, `filename`, - `type`, `height`, `width`, `datasize`, `profile`", + $r = q("select %s ANY_VALUE(`resource-id`) AS `resource-id`, ANY_VALUE(`created`) AS `created`, + ANY_VALUE(`edited`) AS `edited`, ANY_VALUE(`title`) AS `title`, ANY_VALUE(`desc`) AS `desc`, + ANY_VALUE(`album`) AS `album`, ANY_VALUE(`filename`) AS `filename`, ANY_VALUE(`type`) AS `type`, + ANY_VALUE(`height`) AS `height`, ANY_VALUE(`width`) AS `width`, ANY_VALUE(`datasize`) AS `datasize`, + ANY_VALUE(`profile`) AS `profile`, min(`scale`) as minscale, max(`scale`) as maxscale + from photo where `uid` = %d and `resource-id` = '%s' %s", $data_sql, intval(local_user()), dbesc($_REQUEST['photo_id']), diff --git a/include/dba.php b/include/dba.php index 62728acae..96df072cf 100644 --- a/include/dba.php +++ b/include/dba.php @@ -504,6 +504,14 @@ function dbesc($str) { } } +function any_value_fallback($sql) { + //Considerations for Standard SQL, or MySQL with ONLY_FULL_GROUP_BY (default since 5.7.5). + //ANY_VALUE() is available from MySQL 5.7 https://dev.mysql.com/doc/refman/5.7/en/miscellaneous-functions.html + //A standard fallback is to use MIN(), or nothing () in old MySQL 5.6- + //TODO: Skip this line when we know we are on a platform supporting ANY_VALUE() + return str_ireplace('ANY_VALUE(', 'MIN(', $sql); +} + // Function: q($sql,$args); // Description: execute SQL query with printf style args. // Example: $r = q("SELECT * FROM `%s` WHERE `uid` = %d", @@ -514,6 +522,7 @@ function q($sql) { unset($args[0]); if ($db && $db->connected) { + $sql = any_value_fallback($sql); $stmt = @vsprintf($sql,$args); // Disabled warnings //logger("dba: q: $stmt", LOGGER_ALL); if ($stmt === false) @@ -550,6 +559,7 @@ function qu($sql) { unset($args[0]); if ($db && $db->connected) { + $sql = any_value_fallback($sql); $stmt = @vsprintf($sql,$args); // Disabled warnings if ($stmt === false) logger('dba: vsprintf error: ' . print_r(debug_backtrace(),true), LOGGER_DEBUG); diff --git a/include/notifier.php b/include/notifier.php index ed34288e8..370611b4e 100644 --- a/include/notifier.php +++ b/include/notifier.php @@ -516,8 +516,9 @@ function notifier_run(&$argv, &$argc){ $r0 = Diaspora::relay_list(); } - $r1 = q("SELECT DISTINCT(`batch`), `id`, `name`,`network` FROM `contact` WHERE `network` = '%s' - AND `uid` = %d AND `rel` != %d AND NOT `blocked` AND NOT `pending` AND NOT `archive` GROUP BY `batch`, `id` ORDER BY rand()", + $r1 = q("SELECT `batch`, ANY_VALUE(`id`) AS `id`, ANY_VALUE(`name`) AS `name`, ANY_VALUE(`network`) AS `network` + FROM `contact` WHERE `network` = '%s' + AND `uid` = %d AND `rel` != %d AND NOT `blocked` AND NOT `pending` AND NOT `archive` GROUP BY `batch` ORDER BY rand()", dbesc(NETWORK_DIASPORA), intval($owner['uid']), intval(CONTACT_IS_SHARING) diff --git a/include/photos.php b/include/photos.php index 02d1b1740..b273c1726 100644 --- a/include/photos.php +++ b/include/photos.php @@ -48,20 +48,20 @@ function photo_albums($uid, $update = false) { if (!Config::get('system', 'no_count', false)) { /// @todo This query needs to be renewed. It is really slow // At this time we just store the data in the cache - $albums = qu("SELECT COUNT(DISTINCT `resource-id`) AS `total`, `album` + $albums = qu("SELECT COUNT(DISTINCT `resource-id`) AS `total`, `album`, ANY_VALUE(`created`) AS `created` FROM `photo` WHERE `uid` = %d AND `album` != '%s' AND `album` != '%s' $sql_extra - GROUP BY `album`, `created` ORDER BY `created` DESC", + GROUP BY `album` ORDER BY `created` DESC", intval($uid), dbesc('Contact Photos'), dbesc(t('Contact Photos')) ); } else { // This query doesn't do the count and is much faster - $albums = qu("SELECT DISTINCT(`album`), '' AS `total` + $albums = qu("SELECT DISTINCT(`album`), '' AS `total`, ANY_VALUE(`created`) AS `created` FROM `photo` WHERE `uid` = %d AND `album` != '%s' AND `album` != '%s' $sql_extra - GROUP BY `album`, `created` ORDER BY `created` DESC", + GROUP BY `album` ORDER BY `created` DESC", intval($uid), dbesc('Contact Photos'), dbesc(t('Contact Photos')) diff --git a/mod/message.php b/mod/message.php index 05a8d36f6..dd13d58be 100644 --- a/mod/message.php +++ b/mod/message.php @@ -349,8 +349,8 @@ function message_content(App $a) { $o .= $header; - $r = q("SELECT count(*) AS `total` FROM `mail` - WHERE `mail`.`uid` = %d GROUP BY `parent-uri`, `created` ORDER BY `created` DESC", + $r = q("SELECT count(*) AS `total` FROM `mail`, ANY_VALUE(`created`) AS `created` + WHERE `mail`.`uid` = %d GROUP BY `parent-uri` ORDER BY `created` DESC", intval(local_user()) ); @@ -528,12 +528,16 @@ function message_content(App $a) { } function get_messages($user, $lstart, $lend) { - + //TODO: rewritte with a sub-query to get the first message of each private thread with certainty return q("SELECT max(`mail`.`created`) AS `mailcreated`, min(`mail`.`seen`) AS `mailseen`, - `mail`.* , `contact`.`name`, `contact`.`url`, `contact`.`thumb` , `contact`.`network`, - count( * ) as count + ANY_VALUE(`mail`.`id`), ANY_VALUE(`mail`.`uid`), ANY_VALUE(`mail`.`guid`), ANY_VALUE(`mail`.`from-name`), + ANY_VALUE(`mail`.`from-photo`), ANY_VALUE(`mail`.`from-url`), ANY_VALUE(`mail`.`contact-id`), + ANY_VALUE(`mail`.`convid`), ANY_VALUE(`mail`.`title`), ANY_VALUE(`mail`.`body`), ANY_VALUE(`mail`.`seen`), + ANY_VALUE(`mail`.`reply`), ANY_VALUE(`mail`.`replied`), ANY_VALUE(`mail`.`unknown`), + ANY_VALUE(`mail`.`uri`), ANY_VALUE(`mail`.`parent-uri`), ANY_VALUE(`mail`.`created`), + ANY_VALUE(`contact`.`name`), ANY_VALUE(`contact`.`url`), ANY_VALUE(`contact`.`thumb`), ANY_VALUE(`contact`.`network`), FROM `mail` LEFT JOIN `contact` ON `mail`.`contact-id` = `contact`.`id` - WHERE `mail`.`uid` = %d GROUP BY `parent-uri`, `mail`.id ORDER BY `mailcreated` DESC LIMIT %d , %d ", + WHERE `mail`.`uid` = %d GROUP BY `parent-uri` ORDER BY `mailcreated` DESC LIMIT %d , %d ", intval($user), intval($lstart), intval($lend) ); } diff --git a/mod/photos.php b/mod/photos.php index a24cee255..8e4782977 100644 --- a/mod/photos.php +++ b/mod/photos.php @@ -1240,8 +1240,10 @@ function photos_content(App $a) { $order = 'DESC'; } - $r = q("SELECT `resource-id`, `id`, `filename`, type, max(`scale`) AS `scale`, `desc` FROM `photo` WHERE `uid` = %d AND `album` = '%s' - AND `scale` <= 4 $sql_extra GROUP BY `resource-id`, `id` ORDER BY `created` $order LIMIT %d , %d", + $r = q("SELECT `resource-id`, ANY_VALUE(`id`) AS `id`, ANY_VALUE(`filename`) AS `filename`, + ANY_VALUE(`type`) AS `type`, max(`scale`) AS `scale`, ANY_VALUE(`desc`) as `desc` + FROM `photo` WHERE `uid` = %d AND `album` = '%s' + AND `scale` <= 4 $sql_extra GROUP BY `resource-id` ORDER BY `created` $order LIMIT %d , %d", intval($owner_uid), dbesc($album), intval($a->pager['start']), diff --git a/mod/videos.php b/mod/videos.php index ea2e0d4a2..269d53785 100644 --- a/mod/videos.php +++ b/mod/videos.php @@ -356,9 +356,11 @@ function videos_content(App $a) { $a->set_pager_itemspage(20); } - $r = q("SELECT hash, `id`, `filename`, filetype FROM `attach` + $r = q("SELECT hash, ANY_VALUE(`id`) AS `id`, ANY_VALUE(`created`) AS `created`, + ANY_VALUE(`filename`) AS `filename`, ANY_VALUE(`filetype`) as `filetype` + FROM `attach` WHERE `uid` = %d AND filetype LIKE '%%video%%' - $sql_extra GROUP BY hash, `id` ORDER BY `created` DESC LIMIT %d , %d", + $sql_extra GROUP BY hash ORDER BY `created` DESC LIMIT %d , %d", intval($a->data['user']['uid']), intval($a->pager['start']), intval($a->pager['itemspage']) From 6be6badd915800582ddeaf5b6f9162623205e868 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sat, 15 Apr 2017 00:45:30 +0200 Subject: [PATCH 09/14] SQL typo mod/message Mistake in previous commit --- mod/message.php | 1 + 1 file changed, 1 insertion(+) diff --git a/mod/message.php b/mod/message.php index dd13d58be..d3bd1558c 100644 --- a/mod/message.php +++ b/mod/message.php @@ -536,6 +536,7 @@ function get_messages($user, $lstart, $lend) { ANY_VALUE(`mail`.`reply`), ANY_VALUE(`mail`.`replied`), ANY_VALUE(`mail`.`unknown`), ANY_VALUE(`mail`.`uri`), ANY_VALUE(`mail`.`parent-uri`), ANY_VALUE(`mail`.`created`), ANY_VALUE(`contact`.`name`), ANY_VALUE(`contact`.`url`), ANY_VALUE(`contact`.`thumb`), ANY_VALUE(`contact`.`network`), + count( * ) as count FROM `mail` LEFT JOIN `contact` ON `mail`.`contact-id` = `contact`.`id` WHERE `mail`.`uid` = %d GROUP BY `parent-uri` ORDER BY `mailcreated` DESC LIMIT %d , %d ", intval($user), intval($lstart), intval($lend) From 5007be3cf55d5f4854a11b06811da9e50c256829 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sat, 15 Apr 2017 12:40:32 +0200 Subject: [PATCH 10/14] Use server_info to fallback from ANY_VALUE if needed --- include/dba.php | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/include/dba.php b/include/dba.php index 96df072cf..665580124 100644 --- a/include/dba.php +++ b/include/dba.php @@ -20,6 +20,7 @@ class dba { private $driver; public $connected = false; public $error = false; + private $_server_info = ''; function __construct($server, $user, $pass, $db, $install = false) { $a = get_app(); @@ -103,18 +104,20 @@ class dba { * @return string */ public function server_info() { - switch ($this->driver) { - case 'pdo': - $version = $this->db->getAttribute(PDO::ATTR_SERVER_VERSION); - break; - case 'mysqli': - $version = $this->db->server_info; - break; - case 'mysql': - $version = mysql_get_server_info($this->db); - break; + if ($this->_server_info == '') { + switch ($this->driver) { + case 'pdo': + $this->_server_info = $this->db->getAttribute(PDO::ATTR_SERVER_VERSION); + break; + case 'mysqli': + $this->_server_info = $this->db->server_info; + break; + case 'mysql': + $this->_server_info = mysql_get_server_info($this->db); + break; + } } - return $version; + return $this->_server_info; } /** @@ -505,11 +508,16 @@ function dbesc($str) { } function any_value_fallback($sql) { + global $db; + $server_info = $db->server_info(); //Considerations for Standard SQL, or MySQL with ONLY_FULL_GROUP_BY (default since 5.7.5). - //ANY_VALUE() is available from MySQL 5.7 https://dev.mysql.com/doc/refman/5.7/en/miscellaneous-functions.html - //A standard fallback is to use MIN(), or nothing () in old MySQL 5.6- - //TODO: Skip this line when we know we are on a platform supporting ANY_VALUE() - return str_ireplace('ANY_VALUE(', 'MIN(', $sql); + //ANY_VALUE() is available from MySQL 5.7.5 https://dev.mysql.com/doc/refman/5.7/en/miscellaneous-functions.html + //A standard fallback is to use MIN() + if (version_compare($server_info, '5.7.5', '<') || + (stripos($server_info, 'MariaDB') !== false)) { + $sql = str_ireplace('ANY_VALUE(', 'MIN(', $sql); + } + return $sql; } // Function: q($sql,$args); From 06f374b26b01674707c135843d0df5189a9e57cf Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sat, 15 Apr 2017 14:39:41 +0200 Subject: [PATCH 11/14] Document any_value_fallback() https://github.com/friendica/friendica/pull/3323#discussion_r111663767 --- include/dba.php | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/include/dba.php b/include/dba.php index 665580124..4352cde66 100644 --- a/include/dba.php +++ b/include/dba.php @@ -507,12 +507,20 @@ function dbesc($str) { } } +/** + * @brief Replaces ANY_VALUE() function by MIN() function, + * if the database server does not support ANY_VALUE(). + * + * Considerations for Standard SQL, or MySQL with ONLY_FULL_GROUP_BY (default since 5.7.5). + * ANY_VALUE() is available from MySQL 5.7.5 https://dev.mysql.com/doc/refman/5.7/en/miscellaneous-functions.html + * A standard fall-back is to use MIN(). + * + * @param string $sql An SQL string without the values + * @return string The input SQL string modified if necessary. + */ function any_value_fallback($sql) { global $db; $server_info = $db->server_info(); - //Considerations for Standard SQL, or MySQL with ONLY_FULL_GROUP_BY (default since 5.7.5). - //ANY_VALUE() is available from MySQL 5.7.5 https://dev.mysql.com/doc/refman/5.7/en/miscellaneous-functions.html - //A standard fallback is to use MIN() if (version_compare($server_info, '5.7.5', '<') || (stripos($server_info, 'MariaDB') !== false)) { $sql = str_ireplace('ANY_VALUE(', 'MIN(', $sql); From 062070058e48c584616493af38d9949eccd9a849 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sat, 15 Apr 2017 17:31:32 +0200 Subject: [PATCH 12/14] Move any_value_fallback() to dba class https://github.com/friendica/friendica/pull/3323#discussion_r111666245 --- include/dba.php | 45 ++++++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/include/dba.php b/include/dba.php index 4352cde66..5066dcd56 100644 --- a/include/dba.php +++ b/include/dba.php @@ -477,6 +477,26 @@ class dba { } } } + + /** + * @brief Replaces ANY_VALUE() function by MIN() function, + * if the database server does not support ANY_VALUE(). + * + * Considerations for Standard SQL, or MySQL with ONLY_FULL_GROUP_BY (default since 5.7.5). + * ANY_VALUE() is available from MySQL 5.7.5 https://dev.mysql.com/doc/refman/5.7/en/miscellaneous-functions.html + * A standard fall-back is to use MIN(). + * + * @param string $sql An SQL string without the values + * @return string The input SQL string modified if necessary. + */ + public function any_value_fallback($sql) { + $server_info = $this->server_info(); + if (version_compare($server_info, '5.7.5', '<') || + (stripos($server_info, 'MariaDB') !== false)) { + $sql = str_ireplace('ANY_VALUE(', 'MIN(', $sql); + } + return $sql; + } } function printable($s) { @@ -507,27 +527,6 @@ function dbesc($str) { } } -/** - * @brief Replaces ANY_VALUE() function by MIN() function, - * if the database server does not support ANY_VALUE(). - * - * Considerations for Standard SQL, or MySQL with ONLY_FULL_GROUP_BY (default since 5.7.5). - * ANY_VALUE() is available from MySQL 5.7.5 https://dev.mysql.com/doc/refman/5.7/en/miscellaneous-functions.html - * A standard fall-back is to use MIN(). - * - * @param string $sql An SQL string without the values - * @return string The input SQL string modified if necessary. - */ -function any_value_fallback($sql) { - global $db; - $server_info = $db->server_info(); - if (version_compare($server_info, '5.7.5', '<') || - (stripos($server_info, 'MariaDB') !== false)) { - $sql = str_ireplace('ANY_VALUE(', 'MIN(', $sql); - } - return $sql; -} - // Function: q($sql,$args); // Description: execute SQL query with printf style args. // Example: $r = q("SELECT * FROM `%s` WHERE `uid` = %d", @@ -538,7 +537,7 @@ function q($sql) { unset($args[0]); if ($db && $db->connected) { - $sql = any_value_fallback($sql); + $sql = $db->any_value_fallback($sql); $stmt = @vsprintf($sql,$args); // Disabled warnings //logger("dba: q: $stmt", LOGGER_ALL); if ($stmt === false) @@ -575,7 +574,7 @@ function qu($sql) { unset($args[0]); if ($db && $db->connected) { - $sql = any_value_fallback($sql); + $sql = $db->any_value_fallback($sql); $stmt = @vsprintf($sql,$args); // Disabled warnings if ($stmt === false) logger('dba: vsprintf error: ' . print_r(debug_backtrace(),true), LOGGER_DEBUG); From 8b3f623d29e7228115441d4824e2893a1585fe15 Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sat, 15 Apr 2017 23:42:10 +0200 Subject: [PATCH 13/14] ANY_VALUE in mod/admin The query was not standard, not deterministic, and was not doing what its description says. The patch makes it more standard and a bit more deterministic. The returned protocol type is still undeterministic. --- mod/admin.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mod/admin.php b/mod/admin.php index 2652b33c1..093f7c801 100644 --- a/mod/admin.php +++ b/mod/admin.php @@ -290,7 +290,8 @@ function admin_page_federation(App $a) { foreach ($platforms as $p) { // get a total count for the platform, the name and version of the // highest version and the protocol tpe - $c = qu('SELECT COUNT(*) AS `total`, `platform`, `network`, `version` FROM `gserver` + $c = qu('SELECT COUNT(*) AS `total`, ANY_VALUE(`platform`), + ANY_VALUE(`network`), MAX(`version`) FROM `gserver` WHERE `platform` LIKE "%s" AND `last_contact` >= `last_failure` ORDER BY `version` ASC;', $p); $total = $total + $c[0]['total']; From d076b39b648748f66b47f168cf3c7602ac2a085d Mon Sep 17 00:00:00 2001 From: Alexandre Alapetite Date: Sat, 15 Apr 2017 23:48:02 +0200 Subject: [PATCH 14/14] Forgotten AS in mod/admin patch --- mod/admin.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mod/admin.php b/mod/admin.php index 093f7c801..fb463d31c 100644 --- a/mod/admin.php +++ b/mod/admin.php @@ -290,8 +290,8 @@ function admin_page_federation(App $a) { foreach ($platforms as $p) { // get a total count for the platform, the name and version of the // highest version and the protocol tpe - $c = qu('SELECT COUNT(*) AS `total`, ANY_VALUE(`platform`), - ANY_VALUE(`network`), MAX(`version`) FROM `gserver` + $c = qu('SELECT COUNT(*) AS `total`, ANY_VALUE(`platform`) AS `platform`, + ANY_VALUE(`network`) AS `network`, MAX(`version`) AS `version` FROM `gserver` WHERE `platform` LIKE "%s" AND `last_contact` >= `last_failure` ORDER BY `version` ASC;', $p); $total = $total + $c[0]['total'];