Merge pull request #7521 from nupplaphil/task/create_process

Add Process class / move run-settings to App\Mode
This commit is contained in:
Hypolite Petovan 2019-08-17 08:04:24 -04:00 committed by GitHub
commit dfd0f67b33
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 393 additions and 221 deletions

View file

@ -4,7 +4,6 @@
*/
namespace Friendica;
use Detection\MobileDetect;
use Exception;
use Friendica\App\Arguments;
use Friendica\App\BaseURL;
@ -67,7 +66,9 @@ class App
public $timezone;
public $interactive = true;
public $identities;
/** @deprecated 2019.09 - Use App\Mode->isMobile() instead */
public $is_mobile;
/** @deprecated 2019.09 - Use App\Mode->isTable() instead */
public $is_tablet;
public $theme_info = [];
public $category;
@ -86,11 +87,6 @@ class App
*/
private $mode;
/**
* @var App\Router
*/
private $router;
/**
* @var BaseURL
*/
@ -101,16 +97,6 @@ class App
*/
private $currentTheme;
/**
* @var bool check if request was an AJAX (xmlhttprequest) request
*/
private $isAjax;
/**
* @var MobileDetect
*/
public $mobileDetect;
/**
* @var Configuration The config
*/
@ -141,6 +127,11 @@ class App
*/
private $args;
/**
* @var Core\Process The process methods
*/
private $process;
/**
* Returns the current config cache of this node
*
@ -234,26 +225,24 @@ class App
* @param Database $database The Friendica Database
* @param Configuration $config The Configuration
* @param App\Mode $mode The mode of this Friendica app
* @param App\Router $router The router of this Friendica app
* @param BaseURL $baseURL The full base URL of this Friendica app
* @param LoggerInterface $logger The current app logger
* @param Profiler $profiler The profiler of this application
* @param L10n $l10n The translator instance
* @param App\Arguments $args The Friendica Arguments of the call
* @param MobileDetect $mobileDetect A mobile detection class
* @param Core\Process $process The process methods
*/
public function __construct(Database $database, Configuration $config, App\Mode $mode, App\Router $router, BaseURL $baseURL, LoggerInterface $logger, Profiler $profiler, L10n $l10n, Arguments $args, App\Module $module, App\Page $page, MobileDetect $mobileDetect)
public function __construct(Database $database, Configuration $config, App\Mode $mode, BaseURL $baseURL, LoggerInterface $logger, Profiler $profiler, L10n $l10n, Arguments $args, App\Module $module, App\Page $page, Core\Process $process)
{
$this->database = $database;
$this->config = $config;
$this->mode = $mode;
$this->router = $router;
$this->baseURL = $baseURL;
$this->profiler = $profiler;
$this->logger = $logger;
$this->l10n = $l10n;
$this->args = $args;
$this->mobileDetect = $mobileDetect;
$this->process = $process;
$this->cmd = $args->getCommand();
$this->argv = $args->getArgv();
@ -262,10 +251,8 @@ class App
$this->module = $module->getName();
$this->page = $page;
$this->is_mobile = $mobileDetect->isMobile();
$this->is_tablet = $mobileDetect->isTablet();
$this->isAjax = strtolower(defaults($_SERVER, 'HTTP_X_REQUESTED_WITH', '')) == 'xmlhttprequest';
$this->is_mobile = $mode->isMobile();
$this->is_tablet = $mode->isTablet();
$this->load();
}
@ -418,177 +405,27 @@ class App
}
/**
* Returns true, if the call is from a backend node (f.e. from a worker)
*
* @return bool Is it a known backend?
*
* @deprecated 2019.09 - use App\Mode->isBackend() instead
* @see App\Mode::isBackend()
* Use BaseObject::getClass(App\Mode::class) to get the global instance of Mode
*/
public function isBackend()
{
return $this->mode->isBackend();
}
/**
* @brief Checks if the maximum number of database processes is reached
*
* @return bool Is the limit reached?
* @deprecated 2019.09 - use Core\Process->isMaxProcessesReached() instead
*/
public function isMaxProcessesReached()
{
// Deactivated, needs more investigating if this check really makes sense
return false;
/*
* Commented out to suppress static analyzer issues
*
if ($this->is_backend()) {
$process = 'backend';
$max_processes = $this->config->get('system', 'max_processes_backend');
if (intval($max_processes) == 0) {
$max_processes = 5;
}
} else {
$process = 'frontend';
$max_processes = $this->config->get('system', 'max_processes_frontend');
if (intval($max_processes) == 0) {
$max_processes = 20;
}
}
$processlist = DBA::processlist();
if ($processlist['list'] != '') {
$this->logger->debug('Processcheck: Processes: ' . $processlist['amount'] . ' - Processlist: ' . $processlist['list']);
if ($processlist['amount'] > $max_processes) {
$this->logger->debug('Processcheck: Maximum number of processes for ' . $process . ' tasks (' . $max_processes . ') reached.');
return true;
}
}
return false;
*/
return $this->process->isMaxProcessesReached();
}
/**
* @brief Checks if the minimal memory is reached
*
* @return bool Is the memory limit reached?
* @throws HTTPException\InternalServerErrorException
* @deprecated 2019.09 - use Core\Process->isMinMemoryReached() instead
*/
public function isMinMemoryReached()
{
$min_memory = $this->config->get('system', 'min_memory', 0);
if ($min_memory == 0) {
return false;
}
if (!is_readable('/proc/meminfo')) {
return false;
}
$memdata = explode("\n", file_get_contents('/proc/meminfo'));
$meminfo = [];
foreach ($memdata as $line) {
$data = explode(':', $line);
if (count($data) != 2) {
continue;
}
list($key, $val) = $data;
$meminfo[$key] = (int)trim(str_replace('kB', '', $val));
$meminfo[$key] = (int)($meminfo[$key] / 1024);
}
if (!isset($meminfo['MemFree'])) {
return false;
}
$free = $meminfo['MemFree'];
$reached = ($free < $min_memory);
if ($reached) {
$this->logger->debug('Minimal memory reached.', ['free' => $free, 'memtotal' => $meminfo['MemTotal'], 'limit' => $min_memory]);
}
return $reached;
return $this->process->isMinMemoryReached();
}
/**
* @brief Checks if the maximum load is reached
*
* @return bool Is the load reached?
* @throws HTTPException\InternalServerErrorException
* @deprecated 2019.09 - use Core\Process->isMaxLoadReached() instead
*/
public function isMaxLoadReached()
{
if ($this->mode->isBackend()) {
$process = 'backend';
$maxsysload = intval($this->config->get('system', 'maxloadavg'));
if ($maxsysload < 1) {
$maxsysload = 50;
}
} else {
$process = 'frontend';
$maxsysload = intval($this->config->get('system', 'maxloadavg_frontend'));
if ($maxsysload < 1) {
$maxsysload = 50;
}
}
$load = Core\System::currentLoad();
if ($load) {
if (intval($load) > $maxsysload) {
$this->logger->info('system load for process too high.', ['load' => $load, 'process' => $process, 'maxsysload' => $maxsysload]);
return true;
}
}
return false;
}
/**
* Executes a child process with 'proc_open'
*
* @param string $command The command to execute
* @param array $args Arguments to pass to the command ( [ 'key' => value, 'key2' => value2, ... ]
*
* @throws HTTPException\InternalServerErrorException
*/
public function proc_run($command, $args)
{
if (!function_exists('proc_open')) {
return;
}
$cmdline = $this->config->get('config', 'php_path', 'php') . ' ' . escapeshellarg($command);
foreach ($args as $key => $value) {
if (!is_null($value) && is_bool($value) && !$value) {
continue;
}
$cmdline .= ' --' . $key;
if (!is_null($value) && !is_bool($value)) {
$cmdline .= ' ' . $value;
}
}
if ($this->isMinMemoryReached()) {
return;
}
if (strtoupper(substr(PHP_OS, 0, 3)) === 'WIN') {
$resource = proc_open('cmd /c start /b ' . $cmdline, [], $foo, $this->getBasePath());
} else {
$resource = proc_open($cmdline . ' &', [], $foo, $this->getBasePath());
}
if (!is_resource($resource)) {
$this->logger->debug('We got no resource for command.', ['cmd' => $cmdline]);
return;
}
proc_close($resource);
return $this->process->isMaxLoadReached();
}
/**
@ -705,13 +542,12 @@ class App
}
/**
* Check if request was an AJAX (xmlhttprequest) request.
*
* @return boolean true if it was an AJAX request
* @deprecated 2019.09 - use App\Mode->isAjax() instead
* @see App\Mode::isAjax()
*/
public function isAjax()
{
return $this->isAjax;
return $this->mode->isAjax();
}
/**
@ -764,7 +600,7 @@ class App
}
// Max Load Average reached: ERROR
if ($this->isMaxProcessesReached() || $this->isMaxLoadReached()) {
if ($this->process->isMaxProcessesReached() || $this->process->isMaxLoadReached()) {
header('Retry-After: 120');
header('Refresh: 120; url=' . $this->baseURL->get() . "/" . $this->args->getQueryString());

View file

@ -2,6 +2,7 @@
namespace Friendica\App;
use Detection\MobileDetect;
use Friendica\Core\Config\Cache\ConfigCache;
use Friendica\Database\Database;
use Friendica\Util\BasePath;
@ -29,10 +30,28 @@ class Mode
*/
private $isBackend;
public function __construct(int $mode = 0, bool $isBackend = false)
/**
* @var bool True, if the call is a ajax call
*/
private $isAjax;
/**
* @var bool True, if the call is from a mobile device
*/
private $isMobile;
/**
* @var bool True, if the call is from a tablet device
*/
private $isTablet;
public function __construct(int $mode = 0, bool $isBackend = false, bool $isAjax = false, bool $isMobile = false, bool $isTablet = false)
{
$this->mode = $mode;
$this->isBackend = $isBackend;
$this->isAjax = $isAjax;
$this->isMobile = $isMobile;
$this->isTablet = $isTablet;
}
/**
@ -81,7 +100,7 @@ class Mode
$mode |= Mode::MAINTENANCEDISABLED;
return new Mode($mode, $this->isBackend);
return new Mode($mode, $this->isBackend, $this->isAjax, $this->isMobile, $this->isTablet);
}
/**
@ -89,15 +108,19 @@ class Mode
*
* @param Module $module The pre-loaded module (just name, not class!)
* @param array $server The $_SERVER variable
* @param MobileDetect $mobileDetect The mobile detection library
*
* @return Mode returns the determined mode
*/
public function determineBackend(Module $module, array $server)
public function determineRunMode(Module $module, array $server, MobileDetect $mobileDetect)
{
$isBackend = basename(($server['PHP_SELF'] ?? ''), '.php') !== 'index' ||
$module->isBackend();
$isMobile = $mobileDetect->isMobile();
$isTablet = $mobileDetect->isTablet();
$isAjax = strtolower($server['HTTP_X_REQUESTED_WITH'] ?? '') == 'xmlhttprequest';
return new Mode($this->mode, $isBackend);
return new Mode($this->mode, $isBackend, $isAjax, $isMobile, $isTablet);
}
/**
@ -146,4 +169,34 @@ class Mode
{
return $this->isBackend;
}
/**
* Check if request was an AJAX (xmlhttprequest) request.
*
* @return bool true if it was an AJAX request
*/
public function isAjax()
{
return $this->isAjax;
}
/**
* Check if request was a mobile request.
*
* @return bool true if it was an mobile request
*/
public function isMobile()
{
return $this->isMobile;
}
/**
* Check if request was a tablet request.
*
* @return bool true if it was an tablet request
*/
public function isTablet()
{
return $this->isTablet;
}
}

View file

@ -245,11 +245,12 @@ class Page implements ArrayAccess
* - footer.tpl template
*
* @param App $app The Friendica App instance
* @param Mode $mode The Friendica runtime mode
* @param L10n $l10n The l10n instance
*
* @throws HTTPException\InternalServerErrorException
*/
private function initFooter(App $app, L10n $l10n)
private function initFooter(App $app, Mode $mode, L10n $l10n)
{
// If you're just visiting, let javascript take you home
if (!empty($_SESSION['visitor_home'])) {
@ -265,7 +266,7 @@ class Page implements ArrayAccess
/*
* Add a "toggle mobile" link if we're using a mobile device
*/
if ($app->is_mobile || $app->is_tablet) {
if ($mode->isMobile() || $mode->isTablet()) {
if (isset($_SESSION['show-mobile']) && !$_SESSION['show-mobile']) {
$link = 'toggle_mobile?address=' . urlencode(curPageURL());
} else {
@ -375,9 +376,9 @@ class Page implements ArrayAccess
/* Build the page ending -- this is stuff that goes right before
* the closing </body> tag
*/
$this->initFooter($app, $l10n);
$this->initFooter($app, $mode, $l10n);
if (!$app->isAjax()) {
if (!$mode->isAjax()) {
Hook::callAll('page_end', $this->page['content']);
}

203
src/Core/Process.php Normal file
View file

@ -0,0 +1,203 @@
<?php
namespace Friendica\Core;
use Friendica\App;
use Friendica\Core\Config\Configuration;
use Psr\Log\LoggerInterface;
/**
* Methods for interacting with the current process or create new process
*
* @todo 2019.12 Next release, this class holds all process relevant methods based on the big Worker class
* - Starting new processes (including checks)
* - Enabling multi-node processing (e.g. for docker service)
* - Using an process-id per node
* - Using memory locks for multi-node locking (redis, memcached, ..)
*/
final class Process
{
/**
* @var LoggerInterface
*/
private $logger;
/**
* @var App\Mode
*/
private $mode;
/**
* @var Configuration
*/
private $config;
/**
* @var string
*/
private $basePath;
public function __construct(LoggerInterface $logger, App\Mode $mode, Configuration $config, string $basepath)
{
$this->logger = $logger;
$this->mode = $mode;
$this->config = $config;
$this->basePath = $basepath;
}
/**
* @brief Checks if the maximum number of database processes is reached
*
* @return bool Is the limit reached?
*/
public function isMaxProcessesReached()
{
// Deactivated, needs more investigating if this check really makes sense
return false;
/*
* Commented out to suppress static analyzer issues
*
if ($this->mode->isBackend()) {
$process = 'backend';
$max_processes = $this->config->get('system', 'max_processes_backend');
if (intval($max_processes) == 0) {
$max_processes = 5;
}
} else {
$process = 'frontend';
$max_processes = $this->config->get('system', 'max_processes_frontend');
if (intval($max_processes) == 0) {
$max_processes = 20;
}
}
$processlist = DBA::processlist();
if ($processlist['list'] != '') {
$this->logger->debug('Processcheck: Processes: ' . $processlist['amount'] . ' - Processlist: ' . $processlist['list']);
if ($processlist['amount'] > $max_processes) {
$this->logger->debug('Processcheck: Maximum number of processes for ' . $process . ' tasks (' . $max_processes . ') reached.');
return true;
}
}
return false;
*/
}
/**
* @brief Checks if the minimal memory is reached
*
* @return bool Is the memory limit reached?
*/
public function isMinMemoryReached()
{
$min_memory = $this->config->get('system', 'min_memory', 0);
if ($min_memory == 0) {
return false;
}
if (!is_readable('/proc/meminfo')) {
return false;
}
$memdata = explode("\n", file_get_contents('/proc/meminfo'));
$meminfo = [];
foreach ($memdata as $line) {
$data = explode(':', $line);
if (count($data) != 2) {
continue;
}
list($key, $val) = $data;
$meminfo[$key] = (int)trim(str_replace('kB', '', $val));
$meminfo[$key] = (int)($meminfo[$key] / 1024);
}
if (!isset($meminfo['MemFree'])) {
return false;
}
$free = $meminfo['MemFree'];
$reached = ($free < $min_memory);
if ($reached) {
$this->logger->debug('Minimal memory reached.', ['free' => $free, 'memtotal' => $meminfo['MemTotal'], 'limit' => $min_memory]);
}
return $reached;
}
/**
* @brief Checks if the maximum load is reached
*
* @return bool Is the load reached?
*/
public function isMaxLoadReached()
{
if ($this->mode->isBackend()) {
$process = 'backend';
$maxsysload = intval($this->config->get('system', 'maxloadavg'));
if ($maxsysload < 1) {
$maxsysload = 50;
}
} else {
$process = 'frontend';
$maxsysload = intval($this->config->get('system', 'maxloadavg_frontend'));
if ($maxsysload < 1) {
$maxsysload = 50;
}
}
$load = System::currentLoad();
if ($load) {
if (intval($load) > $maxsysload) {
$this->logger->info('system load for process too high.', ['load' => $load, 'process' => $process, 'maxsysload' => $maxsysload]);
return true;
}
}
return false;
}
/**
* Executes a child process with 'proc_open'
*
* @param string $command The command to execute
* @param array $args Arguments to pass to the command ( [ 'key' => value, 'key2' => value2, ... ]
*/
public function run($command, $args)
{
if (!function_exists('proc_open')) {
return;
}
$cmdline = $this->config->get('config', 'php_path', 'php') . ' ' . escapeshellarg($command);
foreach ($args as $key => $value) {
if (!is_null($value) && is_bool($value) && !$value) {
continue;
}
$cmdline .= ' --' . $key;
if (!is_null($value) && !is_bool($value)) {
$cmdline .= ' ' . $value;
}
}
if ($this->isMinMemoryReached()) {
return;
}
if (strtoupper(substr(PHP_OS, 0, 3)) === 'WIN') {
$resource = proc_open('cmd /c start /b ' . $cmdline, [], $foo, $this->basePath);
} else {
$resource = proc_open($cmdline . ' &', [], $foo, $this->basePath);
}
if (!is_resource($resource)) {
$this->logger->debug('We got no resource for command.', ['cmd' => $cmdline]);
return;
}
proc_close($resource);
}
}

View file

@ -5,6 +5,7 @@
namespace Friendica\Core;
use Friendica\BaseObject;
use Friendica\Core;
use Friendica\Database\DBA;
use Friendica\Model\Process;
use Friendica\Util\DateTimeFormat;
@ -1082,7 +1083,9 @@ class Worker
$args = ['no_cron' => !$do_cron];
get_app()->proc_run($command, $args);
$a = get_app();
$process = new Core\Process($a->getLogger(), $a->getMode(), $a->getConfig(), $a->getBasePath());
$process->run($command, $args);
// after spawning we have to remove the flag.
if (Config::get('system', 'worker_daemon_mode', false)) {
@ -1124,7 +1127,7 @@ class Worker
$priority = PRIORITY_MEDIUM;
// Don't fork from frontend tasks by default
$dont_fork = Config::get("system", "worker_dont_fork", false) || !\get_app()->isBackend();
$dont_fork = Config::get("system", "worker_dont_fork", false) || !\get_app()->getMode()->isBackend();
$created = DateTimeFormat::utcNow();
$force_priority = false;

View file

@ -62,7 +62,7 @@ return [
],
App\Mode::class => [
'call' => [
['determineBackend', [$_SERVER], Dice::CHAIN_CALL],
['determineRunMode', [$_SERVER], Dice::CHAIN_CALL],
['determine', [], Dice::CHAIN_CALL],
],
],
@ -154,4 +154,9 @@ return [
['determineModule', [], Dice::CHAIN_CALL],
],
],
Friendica\Core\Process::class => [
'constructParams' => [
[Dice::INSTANCE => '$basepath'],
],
],
];

View file

@ -2,6 +2,7 @@
namespace Friendica\Test\src\App;
use Detection\MobileDetect;
use Friendica\App\Mode;
use Friendica\App\Module;
use Friendica\Core\Config;
@ -186,8 +187,9 @@ class ModeTest extends MockedTest
{
$server = ['PHP_SELF' => '/daemon.php'];
$module = new Module();
$mobileDetect = new MobileDetect();
$mode = (new Mode())->determineBackend($module, $server);
$mode = (new Mode())->determineRunMode($module, $server, $mobileDetect);
$this->assertTrue($mode->isBackend());
}
@ -199,8 +201,9 @@ class ModeTest extends MockedTest
{
$server = ['PHP_SELF' => '/index.php'];
$module = new Module(Module::DEFAULT, Module::DEFAULT_CLASS, true);
$mobileDetect = new MobileDetect();
$mode = (new Mode())->determineBackend($module, $server);
$mode = (new Mode())->determineRunMode($module, $server, $mobileDetect);
$this->assertTrue($mode->isBackend());
}
@ -212,9 +215,77 @@ class ModeTest extends MockedTest
{
$server = ['PHP_SELF' => '/index.php'];
$module = new Module(Module::DEFAULT, Module::DEFAULT_CLASS, false);
$mobileDetect = new MobileDetect();
$mode = (new Mode())->determineBackend($module, $server);
$mode = (new Mode())->determineRunMode($module, $server, $mobileDetect);
$this->assertFalse($mode->isBackend());
}
/**
* Test if the call is an ajax call
*/
public function testIsAjax()
{
// This is the server environment variable to determine ajax calls
$server = [
'HTTP_X_REQUESTED_WITH' => 'xmlhttprequest',
];
$module = new Module(Module::DEFAULT, Module::DEFAULT_CLASS, false);
$mobileDetect = new MobileDetect();
$mode = (new Mode())->determineRunMode($module, $server, $mobileDetect);
$this->assertTrue($mode->isAjax());
}
/**
* Test if the call is not nan ajax call
*/
public function testIsNotAjax()
{
$server = [];
$module = new Module(Module::DEFAULT, Module::DEFAULT_CLASS, false);
$mobileDetect = new MobileDetect();
$mode = (new Mode())->determineRunMode($module, $server, $mobileDetect);
$this->assertFalse($mode->isAjax());
}
/**
* Test if the call is a mobile and is a tablet call
*/
public function testIsMobileIsTablet()
{
$server = [];
$module = new Module(Module::DEFAULT, Module::DEFAULT_CLASS, false);
$mobileDetect = \Mockery::mock(MobileDetect::class);
$mobileDetect->shouldReceive('isMobile')->andReturn(true);
$mobileDetect->shouldReceive('isTablet')->andReturn(true);
$mode = (new Mode())->determineRunMode($module, $server, $mobileDetect);
$this->assertTrue($mode->isMobile());
$this->assertTrue($mode->isTablet());
}
/**
* Test if the call is not a mobile and is not a tablet call
*/
public function testIsNotMobileIsNotTablet()
{
$server = [];
$module = new Module(Module::DEFAULT, Module::DEFAULT_CLASS, false);
$mobileDetect = \Mockery::mock(MobileDetect::class);
$mobileDetect->shouldReceive('isMobile')->andReturn(false);
$mobileDetect->shouldReceive('isTablet')->andReturn(false);
$mode = (new Mode())->determineRunMode($module, $server, $mobileDetect);
$this->assertFalse($mode->isMobile());
$this->assertFalse($mode->isTablet());
}
}