From aefbf93541a97d877a0dc189239045de4cc22aca Mon Sep 17 00:00:00 2001 From: Fabrixxm Date: Wed, 20 Jan 2016 11:39:31 +0100 Subject: [PATCH 1/9] Bump version --- boot.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boot.php b/boot.php index af40027b0..c21a47f4c 100644 --- a/boot.php +++ b/boot.php @@ -17,7 +17,7 @@ require_once('include/dbstructure.php'); define ( 'FRIENDICA_PLATFORM', 'Friendica'); define ( 'FRIENDICA_CODENAME', 'Lily of the valley'); -define ( 'FRIENDICA_VERSION', '3.4.3' ); +define ( 'FRIENDICA_VERSION', '3.4.3-2' ); define ( 'DFRN_PROTOCOL_VERSION', '2.23' ); define ( 'DB_UPDATE_VERSION', 1191 ); define ( 'EOL', "
\r\n" ); From 2ca9d011fe2118293a53b0fda068b3ec27d34c4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Gonz=C3=A1lez?= Date: Sat, 30 Jan 2016 23:21:03 +0100 Subject: [PATCH 2/9] add simple option for HTTPS in .htaccess people have it easier to just uncomment 2 lines if they want such an important feature --- .htaccess | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.htaccess b/.htaccess index 1b63f9ad6..8c8a8861c 100644 --- a/.htaccess +++ b/.htaccess @@ -30,5 +30,9 @@ Deny from all RewriteCond %{REQUEST_FILENAME} !-d RewriteRule ^(.*)$ index.php?pagename=$1 [E=REMOTE_USER:%{HTTP:Authorization},L,QSA] + # If you want to restrict access to your Friendica site to HTTPS only, + # uncommment the following lines + RewriteCond %{HTTPS} !=on + RewriteRule .* https://%{SERVER_NAME}%{REQUEST_URI} [R=301,L] From e80d5a7f23143d10ff6043a00fc06f36e239e9ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Gonz=C3=A1lez?= Date: Sat, 30 Jan 2016 23:22:40 +0100 Subject: [PATCH 3/9] additional blank line --- .htaccess | 1 + 1 file changed, 1 insertion(+) diff --git a/.htaccess b/.htaccess index 8c8a8861c..a2c3c8914 100644 --- a/.htaccess +++ b/.htaccess @@ -34,5 +34,6 @@ Deny from all # uncommment the following lines RewriteCond %{HTTPS} !=on RewriteRule .* https://%{SERVER_NAME}%{REQUEST_URI} [R=301,L] + From 545e4f645e274af0e29a7e70159f0750b7ef8b30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Gonz=C3=A1lez?= Date: Sun, 31 Jan 2016 16:20:06 +0100 Subject: [PATCH 4/9] really uncomment the lines. --- .htaccess | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.htaccess b/.htaccess index a2c3c8914..d2aaa4e13 100644 --- a/.htaccess +++ b/.htaccess @@ -32,8 +32,8 @@ Deny from all # If you want to restrict access to your Friendica site to HTTPS only, # uncommment the following lines - RewriteCond %{HTTPS} !=on - RewriteRule .* https://%{SERVER_NAME}%{REQUEST_URI} [R=301,L] + #RewriteCond %{HTTPS} !=on + #RewriteRule .* https://%{SERVER_NAME}%{REQUEST_URI} [R=301,L] From fb447e84ee64243b00332329b72d85d726fd2119 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Fri, 22 Jul 2016 18:07:04 +0200 Subject: [PATCH 5/9] Some more information about the current queue status of the worker --- include/poller.php | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/include/poller.php b/include/poller.php index 9ea0aac1b..bf25549b5 100644 --- a/include/poller.php +++ b/include/poller.php @@ -39,7 +39,7 @@ function poller_run(&$argv, &$argc){ return; // Checking the number of workers - if (poller_too_much_workers(1)) { + if (poller_too_much_workers()) { poller_kill_stale_workers(); return; } @@ -58,7 +58,7 @@ function poller_run(&$argv, &$argc){ sleep(4); // Checking number of workers - if (poller_too_much_workers(2)) + if (poller_too_much_workers()) return; $cooldown = Config::get("system", "worker_cooldown", 0); @@ -76,7 +76,7 @@ function poller_run(&$argv, &$argc){ return; // Count active workers and compare them with a maximum value that depends on the load - if (poller_too_much_workers(3)) + if (poller_too_much_workers()) return; q("UPDATE `workerqueue` SET `executed` = '%s', `pid` = %d WHERE `id` = %d AND `executed` = '0000-00-00 00:00:00'", @@ -244,7 +244,7 @@ function poller_kill_stale_workers() { } } -function poller_too_much_workers($stage) { +function poller_too_much_workers() { $queues = get_config("system", "worker_queues"); @@ -267,7 +267,9 @@ function poller_too_much_workers($stage) { $slope = $maxworkers / pow($maxsysload, $exponent); $queues = ceil($slope * pow(max(0, $maxsysload - $load), $exponent)); - logger("Current load stage ".$stage.": ".$load." - maximum: ".$maxsysload." - current queues: ".$active." - maximum: ".$queues, LOGGER_DEBUG); + $s = q("SELECT COUNT(*) AS `total` FROM `workerqueue` WHERE `executed` = '0000-00-00 00:00:00'"); + + logger("Current load: ".$load." - maximum: ".$maxsysload." - current queues: ".$active."/".$s[0]["total"]." - maximum: ".$queues, LOGGER_DEBUG); } From 0b0b29dc9777d9958c9cdddbeddf15e4c363d819 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Fri, 22 Jul 2016 18:13:06 +0200 Subject: [PATCH 6/9] Reverted some strange stuff --- .htaccess | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.htaccess b/.htaccess index 1210dcf29..2348cdc38 100644 --- a/.htaccess +++ b/.htaccess @@ -37,10 +37,5 @@ AddType audio/ogg .oga RewriteCond %{REQUEST_FILENAME} !-d RewriteRule ^(.*)$ index.php?pagename=$1 [E=REMOTE_USER:%{HTTP:Authorization},L,QSA] - # If you want to restrict access to your Friendica site to HTTPS only, - # uncommment the following lines - #RewriteCond %{HTTPS} !=on - #RewriteRule .* https://%{SERVER_NAME}%{REQUEST_URI} [R=301,L] - From 7580c8ed1151c77f816cb8910ccbfe6549ad7c63 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sat, 23 Jul 2016 22:57:22 +0200 Subject: [PATCH 7/9] Processing of high priority processes --- include/poller.php | 51 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/include/poller.php b/include/poller.php index bf25549b5..3f1c1d308 100644 --- a/include/poller.php +++ b/include/poller.php @@ -39,7 +39,7 @@ function poller_run(&$argv, &$argc){ return; // Checking the number of workers - if (poller_too_much_workers()) { + if (poller_too_much_workers($entries, $queues, $maxqueues)) { poller_kill_stale_workers(); return; } @@ -58,13 +58,15 @@ function poller_run(&$argv, &$argc){ sleep(4); // Checking number of workers - if (poller_too_much_workers()) + if (poller_too_much_workers($entries, $queues, $maxqueues)) return; $cooldown = Config::get("system", "worker_cooldown", 0); $starttime = time(); + $delayed = 0; + while ($r = q("SELECT * FROM `workerqueue` WHERE `executed` = '0000-00-00 00:00:00' ORDER BY `created` LIMIT 1")) { // Constantly check the number of parallel database processes @@ -76,9 +78,37 @@ function poller_run(&$argv, &$argc){ return; // Count active workers and compare them with a maximum value that depends on the load - if (poller_too_much_workers()) + if (poller_too_much_workers($entries, $queues, $maxqueues)) return; + $argv = json_decode($r[0]["parameter"]); + + $argc = count($argv); + + $funcname = str_replace(".php", "", basename($argv[0]))."_run"; + + // Define the processes that have priority over any other process + /// @todo Better check for priority processes + $priority = array("delivery_run", "notifier_run", "pubsubpublish_run"); + + // Delay other processes if the system has a high load and there are many pending processes. + // The "delayed" value is a safety mechanism for the case when there are no high priority processes in the queue. + if ((($queues / $maxqueues) <= 0.5) AND (($entries > 100) AND ($delayed < 100)) AND !in_array($funcname, $priority)) { + q("UPDATE `workerqueue` SET `created` = '%s' WHERE `id` = %d", + dbesc(datetime_convert()), + intval($r[0]["id"])); + + // Count the number of delayed processes + ++$delayed; + logger("Delay call to '".$funcname."' for performance reasons (Delay ".$delayed.")", LOGGER_DEBUG); + continue; + } + + // If we delivered a process that has high priority we reset the delayed counter. + // When we reached the limit we will process any entry until we reach a high priority process. + if (in_array($funcname, $priority)) + $delayed = 0; + q("UPDATE `workerqueue` SET `executed` = '%s', `pid` = %d WHERE `id` = %d AND `executed` = '0000-00-00 00:00:00'", dbesc(datetime_convert()), intval(getmypid()), @@ -93,10 +123,6 @@ function poller_run(&$argv, &$argc){ continue; } - $argv = json_decode($r[0]["parameter"]); - - $argc = count($argv); - // Check for existance and validity of the include file $include = $argv[0]; @@ -108,8 +134,6 @@ function poller_run(&$argv, &$argc){ require_once($include); - $funcname=str_replace(".php", "", basename($argv[0]))."_run"; - if (function_exists($funcname)) { logger("Process ".getmypid()." - ID ".$r[0]["id"].": ".$funcname." ".$r[0]["parameter"]); $funcname($argv, $argc); @@ -244,13 +268,17 @@ function poller_kill_stale_workers() { } } -function poller_too_much_workers() { +function poller_too_much_workers(&$entries, &$queues, &$maxqueues) { + + $entries = 0; $queues = get_config("system", "worker_queues"); if ($queues == 0) $queues = 4; + $maxqueues = $queues; + $active = poller_active_workers(); // Decrease the number of workers at higher load @@ -268,8 +296,9 @@ function poller_too_much_workers() { $queues = ceil($slope * pow(max(0, $maxsysload - $load), $exponent)); $s = q("SELECT COUNT(*) AS `total` FROM `workerqueue` WHERE `executed` = '0000-00-00 00:00:00'"); + $entries = $s[0]["total"]; - logger("Current load: ".$load." - maximum: ".$maxsysload." - current queues: ".$active."/".$s[0]["total"]." - maximum: ".$queues, LOGGER_DEBUG); + logger("Current load: ".$load." - maximum: ".$maxsysload." - current queues: ".$active."/".$entries." - maximum: ".$queues."/".$maxqueues, LOGGER_DEBUG); } From 4be7e5a5819bc8c880a462d90633f957f1cb9c92 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 24 Jul 2016 11:22:44 +0200 Subject: [PATCH 8/9] "Fast lane" mechanism to reserve one queue for priority processes --- include/poller.php | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/include/poller.php b/include/poller.php index 3f1c1d308..f71872c57 100644 --- a/include/poller.php +++ b/include/poller.php @@ -39,7 +39,7 @@ function poller_run(&$argv, &$argc){ return; // Checking the number of workers - if (poller_too_much_workers($entries, $queues, $maxqueues)) { + if (poller_too_much_workers($entries, $queues, $maxqueues, $active)) { poller_kill_stale_workers(); return; } @@ -58,7 +58,7 @@ function poller_run(&$argv, &$argc){ sleep(4); // Checking number of workers - if (poller_too_much_workers($entries, $queues, $maxqueues)) + if (poller_too_much_workers($entries, $queues, $maxqueues, $active)) return; $cooldown = Config::get("system", "worker_cooldown", 0); @@ -78,7 +78,7 @@ function poller_run(&$argv, &$argc){ return; // Count active workers and compare them with a maximum value that depends on the load - if (poller_too_much_workers($entries, $queues, $maxqueues)) + if (poller_too_much_workers($entries, $queues, $maxqueues, $active)) return; $argv = json_decode($r[0]["parameter"]); @@ -91,16 +91,25 @@ function poller_run(&$argv, &$argc){ /// @todo Better check for priority processes $priority = array("delivery_run", "notifier_run", "pubsubpublish_run"); + // Reserve a speed lane + if (($active == ($queues - 1)) AND ($maxqueues >= 2) AND !in_array($funcname, $priority)) { + logger("Delay call to '".$funcname."' to reserve a speed lane for high priority processes", LOGGER_DEBUG); + $delay = true; + } + // Delay other processes if the system has a high load and there are many pending processes. // The "delayed" value is a safety mechanism for the case when there are no high priority processes in the queue. - if ((($queues / $maxqueues) <= 0.5) AND (($entries > 100) AND ($delayed < 100)) AND !in_array($funcname, $priority)) { - q("UPDATE `workerqueue` SET `created` = '%s' WHERE `id` = %d", - dbesc(datetime_convert()), - intval($r[0]["id"])); - + if (!$delay AND (($queues / $maxqueues) <= 0.5) AND (($entries > 100) AND ($delayed < 100)) AND !in_array($funcname, $priority)) { // Count the number of delayed processes ++$delayed; logger("Delay call to '".$funcname."' for performance reasons (Delay ".$delayed.")", LOGGER_DEBUG); + $delay = true; + } + + if ($delay) { + q("UPDATE `workerqueue` SET `created` = '%s' WHERE `id` = %d", + dbesc(datetime_convert()), + intval($r[0]["id"])); continue; } @@ -268,7 +277,7 @@ function poller_kill_stale_workers() { } } -function poller_too_much_workers(&$entries, &$queues, &$maxqueues) { +function poller_too_much_workers(&$entries, &$queues, &$maxqueues, &$active) { $entries = 0; From 5e81c105e0070f5cf409f9dac2ec1210fede6ee8 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 24 Jul 2016 20:25:11 +0200 Subject: [PATCH 9/9] We now work with the "priority" field - that's better --- boot.php | 21 +++++++++++++--- include/poller.php | 62 +++++++++++----------------------------------- 2 files changed, 32 insertions(+), 51 deletions(-) diff --git a/boot.php b/boot.php index 298010dd0..056e47b7d 100644 --- a/boot.php +++ b/boot.php @@ -1786,12 +1786,27 @@ function proc_run($cmd){ $found = q("SELECT `id` FROM `workerqueue` WHERE `parameter` = '%s'", dbesc($parameters)); + $funcname = str_replace(".php", "", basename($argv[0]))."_run"; + + // Define the processes that have priority over any other process + /// @todo Better check for priority processes + $high_priority = array("delivery_run", "notifier_run", "pubsubpublish_run"); + $low_priority = array("queue_run", "gprobe_run", "discover_poco_run"); + + if (in_array($funcname, $high_priority)) + $priority = 1; + elseif (in_array($funcname, $low_priority)) + $priority = 3; + else + $priority = 2; + if (!$found) - q("INSERT INTO `workerqueue` (`parameter`, `created`, `priority`) - VALUES ('%s', '%s', %d)", + q("INSERT INTO `workerqueue` (`function`, `parameter`, `created`, `priority`) + VALUES ('%s', '%s', '%s', %d)", + dbesc($funcname), dbesc($parameters), dbesc(datetime_convert()), - intval(0)); + intval($priority)); // Should we quit and wait for the poller to be called as a cronjob? if (get_config("system", "worker_dont_fork")) diff --git a/include/poller.php b/include/poller.php index f71872c57..ec6653b48 100644 --- a/include/poller.php +++ b/include/poller.php @@ -39,7 +39,7 @@ function poller_run(&$argv, &$argc){ return; // Checking the number of workers - if (poller_too_much_workers($entries, $queues, $maxqueues, $active)) { + if (poller_too_much_workers()) { poller_kill_stale_workers(); return; } @@ -58,16 +58,14 @@ function poller_run(&$argv, &$argc){ sleep(4); // Checking number of workers - if (poller_too_much_workers($entries, $queues, $maxqueues, $active)) + if (poller_too_much_workers()) return; $cooldown = Config::get("system", "worker_cooldown", 0); $starttime = time(); - $delayed = 0; - - while ($r = q("SELECT * FROM `workerqueue` WHERE `executed` = '0000-00-00 00:00:00' ORDER BY `created` LIMIT 1")) { + while ($r = q("SELECT * FROM `workerqueue` WHERE `executed` = '0000-00-00 00:00:00' ORDER BY `priority`, `created` LIMIT 1")) { // Constantly check the number of parallel database processes if ($a->max_processes_reached()) @@ -78,46 +76,9 @@ function poller_run(&$argv, &$argc){ return; // Count active workers and compare them with a maximum value that depends on the load - if (poller_too_much_workers($entries, $queues, $maxqueues, $active)) + if (poller_too_much_workers()) return; - $argv = json_decode($r[0]["parameter"]); - - $argc = count($argv); - - $funcname = str_replace(".php", "", basename($argv[0]))."_run"; - - // Define the processes that have priority over any other process - /// @todo Better check for priority processes - $priority = array("delivery_run", "notifier_run", "pubsubpublish_run"); - - // Reserve a speed lane - if (($active == ($queues - 1)) AND ($maxqueues >= 2) AND !in_array($funcname, $priority)) { - logger("Delay call to '".$funcname."' to reserve a speed lane for high priority processes", LOGGER_DEBUG); - $delay = true; - } - - // Delay other processes if the system has a high load and there are many pending processes. - // The "delayed" value is a safety mechanism for the case when there are no high priority processes in the queue. - if (!$delay AND (($queues / $maxqueues) <= 0.5) AND (($entries > 100) AND ($delayed < 100)) AND !in_array($funcname, $priority)) { - // Count the number of delayed processes - ++$delayed; - logger("Delay call to '".$funcname."' for performance reasons (Delay ".$delayed.")", LOGGER_DEBUG); - $delay = true; - } - - if ($delay) { - q("UPDATE `workerqueue` SET `created` = '%s' WHERE `id` = %d", - dbesc(datetime_convert()), - intval($r[0]["id"])); - continue; - } - - // If we delivered a process that has high priority we reset the delayed counter. - // When we reached the limit we will process any entry until we reach a high priority process. - if (in_array($funcname, $priority)) - $delayed = 0; - q("UPDATE `workerqueue` SET `executed` = '%s', `pid` = %d WHERE `id` = %d AND `executed` = '0000-00-00 00:00:00'", dbesc(datetime_convert()), intval(getmypid()), @@ -132,6 +93,10 @@ function poller_run(&$argv, &$argc){ continue; } + $argv = json_decode($r[0]["parameter"]); + + $argc = count($argv); + // Check for existance and validity of the include file $include = $argv[0]; @@ -143,16 +108,18 @@ function poller_run(&$argv, &$argc){ require_once($include); + $funcname = str_replace(".php", "", basename($argv[0]))."_run"; + if (function_exists($funcname)) { - logger("Process ".getmypid()." - ID ".$r[0]["id"].": ".$funcname." ".$r[0]["parameter"]); + logger("Process ".getmypid()." - Prio ".$r[0]["priority"]." - ID ".$r[0]["id"].": ".$funcname." ".$r[0]["parameter"]); $funcname($argv, $argc); if ($cooldown > 0) { - logger("Process ".getmypid()." - ID ".$r[0]["id"].": ".$funcname." - in cooldown for ".$cooldown." seconds"); + logger("Process ".getmypid()." - Prio ".$r[0]["priority"]." - ID ".$r[0]["id"].": ".$funcname." - in cooldown for ".$cooldown." seconds"); sleep($cooldown); } - logger("Process ".getmypid()." - ID ".$r[0]["id"].": ".$funcname." - done"); + logger("Process ".getmypid()." - Prio ".$r[0]["priority"]." - ID ".$r[0]["id"].": ".$funcname." - done"); q("DELETE FROM `workerqueue` WHERE `id` = %d", intval($r[0]["id"])); } else @@ -277,9 +244,8 @@ function poller_kill_stale_workers() { } } -function poller_too_much_workers(&$entries, &$queues, &$maxqueues, &$active) { +function poller_too_much_workers() { - $entries = 0; $queues = get_config("system", "worker_queues");