Merge pull request #9755 from annando/no-frontend-worker

The frontend worker is removed
This commit is contained in:
Hypolite Petovan 2021-01-07 09:34:46 -05:00 committed by GitHub
commit 27691f64c0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 21 additions and 221 deletions

View File

@ -240,15 +240,9 @@ This section allows you to configure the background process that is triggered by
The process does check the available system resources before creating a new worker for a task. The process does check the available system resources before creating a new worker for a task.
Because of this, it may happen that the maximum number of worker processes you allow will not be reached. Because of this, it may happen that the maximum number of worker processes you allow will not be reached.
If your server setup does not allow you to use the `proc_open` function of PHP, please disable it in this section.
The tasks for the background process have priorities. The tasks for the background process have priorities.
To guarantee that important tasks are executed even though the system has a lot of work to do, it is useful to enable the *fastlane*. To guarantee that important tasks are executed even though the system has a lot of work to do, it is useful to enable the *fastlane*.
Should you not be able to run a cron job on your server, you can also activate the *frontend* worker.
If you have done so, you can call `example.com/worker` (replace example.com with your actual domain name) on a regular basis from an external service.
This will then trigger the execution of the background process.
### Relocate ### Relocate
## Users ## Users

View File

@ -227,15 +227,9 @@ In diesem Abschnitt kann der Hintergrund-Prozess konfiguriert werden.
Bevor ein neuer *Worker* Prozess gestartet wird, überprüft das System, dass die vorhandenen Resourchen ausrechend sind, Bevor ein neuer *Worker* Prozess gestartet wird, überprüft das System, dass die vorhandenen Resourchen ausrechend sind,
Aus diesem Grund kann es sein, dass die maximale Zahl der Hintergrungprozesse nicht erreicht wird. Aus diesem Grund kann es sein, dass die maximale Zahl der Hintergrungprozesse nicht erreicht wird.
Sollte die PHP Funktion `proc_open` auf dem Server nicht verfügbar sein, kann die Verwendung durch Friendica hier unterbunden werden.
Die Aufgaben die im Hintergrund erledigt werden, haben Prioritäten zugeteilt. Die Aufgaben die im Hintergrund erledigt werden, haben Prioritäten zugeteilt.
Um garantieren zu können, das wichtige Prozesse schnellst möglich abgearbeitet werden können, selbst wenn das System gerade stark belastet ist, sollte die *fastlane* aktiviert sein. Um garantieren zu können, das wichtige Prozesse schnellst möglich abgearbeitet werden können, selbst wenn das System gerade stark belastet ist, sollte die *fastlane* aktiviert sein.
Wenn es auf deinem Server nicht möglich ist, einen cron Job zu starten, kannst du den *frontend* Worker einschalten.
Nachdem dies geschehen ist, kannst du `example.com/worker` (tausche example.com mit dem echten Domainnamen aus) aufrufen werden.
Dadurch werden dann die Aufgaben aktiviert, die der cron Job sonst aktivieren würde.
### Umsiedeln ### Umsiedeln
## Nutzer ## Nutzer

View File

@ -445,11 +445,6 @@ class App
Core\Hook::callAll('init_1'); Core\Hook::callAll('init_1');
} }
// Exclude the backend processes from the session management
if ($this->mode->isBackend()) {
Core\Worker::executeIfIdle();
}
if ($this->mode->isNormal() && !$this->mode->isBackend()) { if ($this->mode->isNormal() && !$this->mode->isBackend()) {
$requester = HTTPSignature::getSigner('', $_SERVER); $requester = HTTPSignature::getSigner('', $_SERVER);
if (!empty($requester)) { if (!empty($requester)) {

View File

@ -463,6 +463,13 @@ class Installer
); );
$returnVal = $returnVal ? $status : false; $returnVal = $returnVal ? $status : false;
$status = $this->checkFunction('proc_open',
DI::l10n()->t('Program execution functions'),
DI::l10n()->t('Error: Program execution functions required but not enabled.'),
true
);
$returnVal = $returnVal ? $status : false;
$status = $this->checkFunction('json_encode', $status = $this->checkFunction('json_encode',
DI::l10n()->t('JSON PHP module'), DI::l10n()->t('JSON PHP module'),
DI::l10n()->t('Error: JSON PHP module required but not installed.'), DI::l10n()->t('Error: JSON PHP module required but not installed.'),

View File

@ -1085,95 +1085,6 @@ class Worker
self::$db_duration_write += (microtime(true) - $stamp); self::$db_duration_write += (microtime(true) - $stamp);
} }
/**
* Call the front end worker
*
* @return void
* @throws \Friendica\Network\HTTPException\InternalServerErrorException
*/
public static function callWorker()
{
if (!DI::config()->get("system", "frontend_worker")) {
return;
}
$url = DI::baseUrl() . '/worker';
DI::httpRequest()->fetch($url, 1);
}
/**
* Call the front end worker if there aren't any active
*
* @return void
* @throws \Friendica\Network\HTTPException\InternalServerErrorException
*/
public static function executeIfIdle()
{
self::checkDaemonState();
if (!DI::config()->get("system", "frontend_worker")) {
return;
}
// Do we have "proc_open"? Then we can fork the worker
if (function_exists("proc_open")) {
// When was the last time that we called the worker?
// Less than one minute? Then we quit
if ((time() - DI::config()->get("system", "worker_started")) < 60) {
return;
}
DI::config()->set("system", "worker_started", time());
// Do we have enough running workers? Then we quit here.
if (self::tooMuchWorkers()) {
// Cleaning dead processes
self::killStaleWorkers();
DI::modelProcess()->deleteInactive();
return;
}
self::runCron();
Logger::info('Call worker');
self::spawnWorker();
return;
}
// We cannot execute background processes.
// We now run the processes from the frontend.
// This won't work with long running processes.
self::runCron();
self::clearProcesses();
$workers = self::activeWorkers();
if ($workers == 0) {
self::callWorker();
}
}
/**
* Removes long running worker processes
*
* @return void
* @throws \Friendica\Network\HTTPException\InternalServerErrorException
*/
public static function clearProcesses()
{
$timeout = DI::config()->get("system", "frontend_worker_timeout", 10);
/// @todo We should clean up the corresponding workerqueue entries as well
$stamp = (float)microtime(true);
$condition = ["`created` < ? AND `command` = 'worker.php'",
DateTimeFormat::utc("now - ".$timeout." minutes")];
DBA::delete('process', $condition);
self::$db_duration = (microtime(true) - $stamp);
self::$db_duration_write += (microtime(true) - $stamp);
}
/** /**
* Runs the cron processes * Runs the cron processes
* *

View File

@ -205,9 +205,7 @@ class Site extends BaseAdmin
$check_new_version_url = (!empty($_POST['check_new_version_url']) ? Strings::escapeTags(trim($_POST['check_new_version_url'])) : 'none'); $check_new_version_url = (!empty($_POST['check_new_version_url']) ? Strings::escapeTags(trim($_POST['check_new_version_url'])) : 'none');
$worker_queues = (!empty($_POST['worker_queues']) ? intval($_POST['worker_queues']) : 10); $worker_queues = (!empty($_POST['worker_queues']) ? intval($_POST['worker_queues']) : 10);
$worker_dont_fork = !empty($_POST['worker_dont_fork']);
$worker_fastlane = !empty($_POST['worker_fastlane']); $worker_fastlane = !empty($_POST['worker_fastlane']);
$worker_frontend = !empty($_POST['worker_frontend']);
$relay_directly = !empty($_POST['relay_directly']); $relay_directly = !empty($_POST['relay_directly']);
$relay_server = (!empty($_POST['relay_server']) ? Strings::escapeTags(trim($_POST['relay_server'])) : ''); $relay_server = (!empty($_POST['relay_server']) ? Strings::escapeTags(trim($_POST['relay_server'])) : '');
@ -417,13 +415,7 @@ class Site extends BaseAdmin
DI::config()->set('system', 'only_tag_search' , $only_tag_search); DI::config()->set('system', 'only_tag_search' , $only_tag_search);
DI::config()->set('system', 'worker_queues' , $worker_queues); DI::config()->set('system', 'worker_queues' , $worker_queues);
if (function_exists('proc_open')) {
DI::config()->set('system', 'worker_dont_fork', $worker_dont_fork);
}
DI::config()->set('system', 'worker_fastlane' , $worker_fastlane); DI::config()->set('system', 'worker_fastlane' , $worker_fastlane);
DI::config()->set('system', 'frontend_worker' , $worker_frontend);
DI::config()->set('system', 'relay_directly' , $relay_directly); DI::config()->set('system', 'relay_directly' , $relay_directly);
DI::config()->set('system', 'relay_server' , $relay_server); DI::config()->set('system', 'relay_server' , $relay_server);
@ -582,14 +574,6 @@ class Site extends BaseAdmin
} }
} }
if (function_exists('proc_open')) {
$worker_dont_fork = DI::config()->get('system', 'worker_dont_fork');
$worker_dont_fork_disabled = '';
} else {
$worker_dont_fork = true;
$worker_dont_fork_disabled = 'disabled';
}
$t = Renderer::getMarkupTemplate('admin/site.tpl'); $t = Renderer::getMarkupTemplate('admin/site.tpl');
return Renderer::replaceMacros($t, [ return Renderer::replaceMacros($t, [
'$title' => DI::l10n()->t('Administration'), '$title' => DI::l10n()->t('Administration'),
@ -702,9 +686,7 @@ class Site extends BaseAdmin
'$rino' => ['rino', DI::l10n()->t('RINO Encryption'), intval(DI::config()->get('system', 'rino_encrypt')), DI::l10n()->t('Encryption layer between nodes.'), [0 => DI::l10n()->t('Disabled'), 1 => DI::l10n()->t('Enabled')]], '$rino' => ['rino', DI::l10n()->t('RINO Encryption'), intval(DI::config()->get('system', 'rino_encrypt')), DI::l10n()->t('Encryption layer between nodes.'), [0 => DI::l10n()->t('Disabled'), 1 => DI::l10n()->t('Enabled')]],
'$worker_queues' => ['worker_queues', DI::l10n()->t('Maximum number of parallel workers'), DI::config()->get('system', 'worker_queues'), DI::l10n()->t('On shared hosters set this to %d. On larger systems, values of %d are great. Default value is %d.', 5, 20, 10)], '$worker_queues' => ['worker_queues', DI::l10n()->t('Maximum number of parallel workers'), DI::config()->get('system', 'worker_queues'), DI::l10n()->t('On shared hosters set this to %d. On larger systems, values of %d are great. Default value is %d.', 5, 20, 10)],
'$worker_dont_fork' => ['worker_dont_fork', DI::l10n()->t('Don\'t use "proc_open" with the worker'), $worker_dont_fork, DI::l10n()->t('Enable this if your system doesn\'t allow the use of "proc_open". This can happen on shared hosters. If this is enabled you should increase the frequency of worker calls in your crontab.'), $worker_dont_fork_disabled],
'$worker_fastlane' => ['worker_fastlane', DI::l10n()->t('Enable fastlane'), DI::config()->get('system', 'worker_fastlane'), DI::l10n()->t('When enabed, the fastlane mechanism starts an additional worker if processes with higher priority are blocked by processes of lower priority.')], '$worker_fastlane' => ['worker_fastlane', DI::l10n()->t('Enable fastlane'), DI::config()->get('system', 'worker_fastlane'), DI::l10n()->t('When enabed, the fastlane mechanism starts an additional worker if processes with higher priority are blocked by processes of lower priority.')],
'$worker_frontend' => ['worker_frontend', DI::l10n()->t('Enable frontend worker'), DI::config()->get('system', 'frontend_worker'), DI::l10n()->t('When enabled the Worker process is triggered when backend access is performed (e.g. messages being delivered). On smaller sites you might want to call %s/worker on a regular basis via an external cron job. You should only enable this option if you cannot utilize cron/scheduled jobs on your server.', DI::baseUrl()->get())],
'$relay_subscribe' => ['relay_subscribe', DI::l10n()->t('Use relay servers'), DI::config()->get('system', 'relay_subscribe'), DI::l10n()->t('Enables the receiving of public posts from relay servers. They will be included in the search, subscribed tags and on the global community page.')], '$relay_subscribe' => ['relay_subscribe', DI::l10n()->t('Use relay servers'), DI::config()->get('system', 'relay_subscribe'), DI::l10n()->t('Enables the receiving of public posts from relay servers. They will be included in the search, subscribed tags and on the global community page.')],
'$relay_server' => ['relay_server', DI::l10n()->t('"Social Relay" server'), DI::config()->get('system', 'relay_server'), DI::l10n()->t('Address of the "Social Relay" server where public posts should be send to. For example %s. ActivityRelay servers are administrated via the "console relay" command line command.', 'https://social-relay.isurf.ca')], '$relay_server' => ['relay_server', DI::l10n()->t('"Social Relay" server'), DI::config()->get('system', 'relay_server'), DI::l10n()->t('Address of the "Social Relay" server where public posts should be send to. For example %s. ActivityRelay servers are administrated via the "console relay" command line command.', 'https://social-relay.isurf.ca')],

View File

@ -1,87 +0,0 @@
<?php
/**
* @copyright Copyright (C) 2020, Friendica
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
namespace Friendica\Module;
use Friendica\BaseModule;
use Friendica\Core\Process;
use Friendica\Core\System;
use Friendica\Core\Worker as WorkerCore;
use Friendica\Database\DBA;
use Friendica\DI;
use Friendica\Util\DateTimeFormat;
/**
* Module for starting the backend worker through a frontend call
*/
class Worker extends BaseModule
{
public static function rawContent(array $parameters = [])
{
if (!DI::config()->get("system", "frontend_worker")) {
return;
}
// Ensure that all "strtotime" operations do run timezone independent
date_default_timezone_set('UTC');
// We don't need the following lines if we can execute background jobs.
// So we just wake up the worker if it sleeps.
if (function_exists("proc_open")) {
WorkerCore::executeIfIdle();
return;
}
WorkerCore::clearProcesses();
$workers = DBA::count('process', ['command' => 'worker.php']);
if ($workers > DI::config()->get("system", "worker_queues", 4)) {
return;
}
DI::process()->start();
DI::logger()->notice('Front end worker started.', ['pid' => getmypid()]);
WorkerCore::callWorker();
if ($r = WorkerCore::workerProcess()) {
// On most configurations this parameter wouldn't have any effect.
// But since it doesn't destroy anything, we just try to get more execution time in any way.
set_time_limit(0);
$fields = ['executed' => DateTimeFormat::utcNow(), 'pid' => getmypid(), 'done' => false];
$condition = ['id' => $r[0]["id"], 'pid' => 0];
if (DBA::update('workerqueue', $fields, $condition)) {
WorkerCore::execute($r[0]);
}
}
WorkerCore::callWorker();
WorkerCore::unclaimProcess();
DI::process()->end();
System::httpExit(200, 'Frontend worker stopped.');
}
}

View File

@ -247,10 +247,6 @@ return [
// Number of "free" searches when system => permit_crawling is enabled. // Number of "free" searches when system => permit_crawling is enabled.
'free_crawls' => 10, 'free_crawls' => 10,
// frontend_worker_timeout (Integer)
// Value in minutes after we think that a frontend task was killed by the webserver.
'frontend_worker_timeout' => 10,
// groupedit_image_limit (Integer) // groupedit_image_limit (Integer)
// Number of contacts at which the group editor should switch from display the profile pictures of the contacts to only display the names. // Number of contacts at which the group editor should switch from display the profile pictures of the contacts to only display the names.
// This can alternatively be set on a per account basis in the pconfig table. // This can alternatively be set on a per account basis in the pconfig table.

View File

@ -93,6 +93,8 @@ class InstallerTest extends MockedTest
$this->mockL10nT('Error: JSON PHP module required but not installed.', 1); $this->mockL10nT('Error: JSON PHP module required but not installed.', 1);
$this->mockL10nT('File Information PHP module', 1); $this->mockL10nT('File Information PHP module', 1);
$this->mockL10nT('Error: File Information PHP module required but not installed.', 1); $this->mockL10nT('Error: File Information PHP module required but not installed.', 1);
$this->mockL10nT('Program execution functions', 1);
$this->mockL10nT('Error: Program execution functions required but not enabled.', 1);
} }
private function assertCheckExist($position, $title, $help, $status, $required, $assertionArray) private function assertCheckExist($position, $title, $help, $status, $required, $assertionArray)
@ -232,10 +234,20 @@ class InstallerTest extends MockedTest
$install->getChecks()); $install->getChecks());
$this->mockFunctionL10TCalls(); $this->mockFunctionL10TCalls();
$this->setFunctions(['json_encode' => false]); $this->setFunctions(['proc_open' => false]);
$install = new Installer(); $install = new Installer();
self::assertFalse($install->checkFunctions()); self::assertFalse($install->checkFunctions());
self::assertCheckExist(9, self::assertCheckExist(9,
'Program execution functions',
'Error: Program execution functions required but not enabled.',
false,
true,
$install->getChecks());
$this->mockFunctionL10TCalls();
$this->setFunctions(['json_encode' => false]);
$install = new Installer();
self::assertFalse($install->checkFunctions());
self::assertCheckExist(10,
'JSON PHP module', 'JSON PHP module',
'Error: JSON PHP module required but not installed.', 'Error: JSON PHP module required but not installed.',
false, false,
@ -246,7 +258,7 @@ class InstallerTest extends MockedTest
$this->setFunctions(['finfo_open' => false]); $this->setFunctions(['finfo_open' => false]);
$install = new Installer(); $install = new Installer();
self::assertFalse($install->checkFunctions()); self::assertFalse($install->checkFunctions());
self::assertCheckExist(10, self::assertCheckExist(11,
'File Information PHP module', 'File Information PHP module',
'Error: File Information PHP module required but not installed.', 'Error: File Information PHP module required but not installed.',
false, false,

View File

@ -122,9 +122,7 @@
{{include file="field_input.tpl" field=$maxloadavg}} {{include file="field_input.tpl" field=$maxloadavg}}
{{include file="field_input.tpl" field=$min_memory}} {{include file="field_input.tpl" field=$min_memory}}
{{include file="field_input.tpl" field=$worker_queues}} {{include file="field_input.tpl" field=$worker_queues}}
{{include file="field_checkbox.tpl" field=$worker_dont_fork}}
{{include file="field_checkbox.tpl" field=$worker_fastlane}} {{include file="field_checkbox.tpl" field=$worker_fastlane}}
{{include file="field_checkbox.tpl" field=$worker_frontend}}
<div class="submit"><input type="submit" name="page_site" value="{{$submit}}"/></div> <div class="submit"><input type="submit" name="page_site" value="{{$submit}}"/></div>

View File

@ -275,9 +275,7 @@
{{include file="field_input.tpl" field=$maxloadavg}} {{include file="field_input.tpl" field=$maxloadavg}}
{{include file="field_input.tpl" field=$min_memory}} {{include file="field_input.tpl" field=$min_memory}}
{{include file="field_input.tpl" field=$worker_queues}} {{include file="field_input.tpl" field=$worker_queues}}
{{include file="field_checkbox.tpl" field=$worker_dont_fork}}
{{include file="field_checkbox.tpl" field=$worker_fastlane}} {{include file="field_checkbox.tpl" field=$worker_fastlane}}
{{include file="field_checkbox.tpl" field=$worker_frontend}}
</div> </div>
<div class="panel-footer"> <div class="panel-footer">
<input type="submit" name="page_site" class="btn btn-primary" value="{{$submit}}"/> <input type="submit" name="page_site" class="btn btn-primary" value="{{$submit}}"/>