Performance improvements when storing items (via API)

This commit is contained in:
Michael 2016-11-12 13:17:28 +00:00
parent 8f6e9fa65e
commit e6148f4c1c
6 changed files with 102 additions and 49 deletions

View File

@ -1897,11 +1897,12 @@ function get_max_import_size() {
* @brief Wrap calls to proc_close(proc_open()) and call hook * @brief Wrap calls to proc_close(proc_open()) and call hook
* so plugins can take part in process :) * so plugins can take part in process :)
* *
* @param (string|integer) $cmd program to run or priority * @param (string|integer|array) $cmd program to run, priority or parameter array
* *
* next args are passed as $cmd command line * next args are passed as $cmd command line
* e.g.: proc_run("ls","-la","/tmp"); * e.g.: proc_run("ls","-la","/tmp");
* or: proc_run(PRIORITY_HIGH, "include/notifier.php", "drop", $drop_id); * or: proc_run(PRIORITY_HIGH, "include/notifier.php", "drop", $drop_id);
* or: proc_run(array('priority' => PRIORITY_HIGH, 'dont_fork' => true), "include/create_shadowentry.php", $post_id);
* *
* @note $cmd and string args are surrounded with "" * @note $cmd and string args are surrounded with ""
* *
@ -1912,24 +1913,31 @@ function proc_run($cmd){
$a = get_app(); $a = get_app();
$args = func_get_args(); $proc_args = func_get_args();
$newargs = array(); $args = array();
if (!count($args)) if (!count($proc_args)) {
return; return;
}
// Preserve the first parameter
// It could contain a command, the priority or an parameter array
// If we use the parameter array we have to protect it from the following function
$run_parameter = array_shift($proc_args);
// expand any arrays // expand any arrays
foreach ($proc_args as $arg) {
foreach($args as $arg) { if (is_array($arg)) {
if(is_array($arg)) { foreach ($arg as $n) {
foreach($arg as $n) { $args[] = $n;
$newargs[] = $n; }
} else {
$args[] = $arg;
} }
} else
$newargs[] = $arg;
} }
$args = $newargs; // Now we add the run parameters back to the array
array_unshift($args, $run_parameter);
$arr = array('args' => $args, 'run_cmd' => true); $arr = array('args' => $args, 'run_cmd' => true);
@ -1937,16 +1945,24 @@ function proc_run($cmd){
if (!$arr['run_cmd'] OR !count($args)) if (!$arr['run_cmd'] OR !count($args))
return; return;
if (!get_config("system", "worker") OR if (!get_config("system", "worker") OR (is_string($run_parameter) AND ($run_parameter != 'php'))) {
(($args[0] != 'php') AND !is_int($args[0]))) {
$a->proc_run($args); $a->proc_run($args);
return; return;
} }
if (is_int($args[0]))
$priority = $args[0];
else
$priority = PRIORITY_MEDIUM; $priority = PRIORITY_MEDIUM;
$dont_fork = get_config("system", "worker_dont_fork");
if (is_int($run_parameter)) {
$priority = $run_parameter;
} elseif (is_array($run_parameter)) {
if (isset($run_parameter['priority'])) {
$priority = $run_parameter['priority'];
}
if (isset($run_parameter['dont_fork'])) {
$dont_fork = $run_parameter['dont_fork'];
}
}
$argv = $args; $argv = $args;
array_shift($argv); array_shift($argv);
@ -1963,8 +1979,9 @@ function proc_run($cmd){
intval($priority)); intval($priority));
// Should we quit and wait for the poller to be called as a cronjob? // Should we quit and wait for the poller to be called as a cronjob?
if (get_config("system", "worker_dont_fork")) if ($dont_fork) {
return; return;
}
// Checking number of workers // Checking number of workers
$workers = q("SELECT COUNT(*) AS `workers` FROM `workerqueue` WHERE `executed` != '0000-00-00 00:00:00'"); $workers = q("SELECT COUNT(*) AS `workers` FROM `workerqueue` WHERE `executed` != '0000-00-00 00:00:00'");

View File

@ -707,11 +707,6 @@ class Photo {
); );
} }
// Update the cached values
if ($album != 'Contact Photos') {
photo_albums($uid, true);
}
return $r; return $r;
} }
} }
@ -872,7 +867,7 @@ function get_photo_info($url) {
$data = Cache::get($url); $data = Cache::get($url);
if (is_null($data) OR !$data) { if (is_null($data) OR !$data OR !is_array($data)) {
$img_str = fetch_url($url, true, $redirects, 4); $img_str = fetch_url($url, true, $redirects, 4);
$filesize = strlen($img_str); $filesize = strlen($img_str);

View File

@ -281,16 +281,15 @@
logger("API call duration: ".round($duration, 2)."\t".$a->query_string, LOGGER_DEBUG); logger("API call duration: ".round($duration, 2)."\t".$a->query_string, LOGGER_DEBUG);
if (get_config("system", "profiler")) { if (get_config("system", "profiler")) {
logger(sprintf("Database: %s/%s, Network: %s, Rendering: %s, Session: %s, I/O: %s, Other: %s, Total: %s", $duration = microtime(true)-$a->performance["start"];
logger(parse_url($a->query_string, PHP_URL_PATH).": ".sprintf("Database: %s/%s, Network: %s, I/O: %s, Other: %s, Total: %s",
round($a->performance["database"] - $a->performance["database_write"], 3), round($a->performance["database"] - $a->performance["database_write"], 3),
round($a->performance["database_write"], 3), round($a->performance["database_write"], 3),
round($a->performance["network"], 2), round($a->performance["network"], 2),
round($a->performance["rendering"], 2),
round($a->performance["parser"], 2),
round($a->performance["file"], 2), round($a->performance["file"], 2),
round($duration - $a->performance["database"] round($duration - ($a->performance["database"] + $a->performance["network"]
- $a->performance["network"] - $a->performance["rendering"] + $a->performance["file"]), 2),
- $a->performance["parser"] - $a->performance["file"], 2),
round($duration, 2)), round($duration, 2)),
LOGGER_DEBUG); LOGGER_DEBUG);

View File

@ -0,0 +1,40 @@
<?php
/**
* @file include/create_shadowentry.php
* @brief This script creates posts with UID = 0 for a given public post.
*
* This script is started from mod/item.php to save some time when doing a post.
*/
require_once("boot.php");
require_once("include/threads.php");
function create_shadowentry_run(&$argv, &$argc) {
global $a, $db;
if (is_null($a))
$a = new App;
if (is_null($db)) {
@include(".htconfig.php");
require_once("include/dba.php");
$db = new dba($db_host, $db_user, $db_pass, $db_data);
unset($db_host, $db_user, $db_pass, $db_data);
}
load_config('config');
load_config('system');
if ($argc != 2) {
return;
}
$message_id = intval($argv[1]);
add_shadow_entry($message_id);
}
if (array_search(__file__,get_included_files())===0){
create_shadowentry_run($_SERVER["argv"],$_SERVER["argc"]);
killme();
}
?>

View File

@ -123,8 +123,19 @@ function add_shadow_thread($itemid) {
function add_shadow_entry($itemid) { function add_shadow_entry($itemid) {
$items = q("SELECT * FROM `item` WHERE `id` = %d", intval($itemid)); $items = q("SELECT * FROM `item` WHERE `id` = %d", intval($itemid));
if (!dbm::is_result($items)) {
return;
}
$item = $items[0]; $item = $items[0];
// Is it a toplevel post?
if ($item['id'] == $item['parent']) {
add_shadow_thread($itemid);
return;
}
// Is this a shadow entry? // Is this a shadow entry?
if ($item['uid'] == 0) if ($item['uid'] == 0)
return; return;

View File

@ -1020,22 +1020,13 @@ function item_post(&$a) {
create_tags_from_item($post_id); create_tags_from_item($post_id);
create_files_from_item($post_id); create_files_from_item($post_id);
// Insert an item entry for UID=0 for global entries // Insert an item entry for UID=0 for global entries.
if ($post_id != $parent) { // We now do it in the background to save some time.
add_shadow_thread($post_id); // This is important in interactive environments like the frontend or the API.
} else { // We don't fork a new process since this is done anyway with the following command
add_shadow_entry($post_id); proc_run(array('priority' => PRIORITY_HIGH, 'dont_fork' => true), "include/create_shadowentry.php", $post_id);
}
// This is a real juggling act on shared hosting services which kill your processes
// e.g. dreamhost. We used to start delivery to our native delivery agents in the background
// and then run our plugin delivery from the foreground. We're now doing plugin delivery first,
// because as soon as you start loading up a bunch of remote delivey processes, *this* page is
// likely to get killed off. If you end up looking at an /item URL and a blank page,
// it's very likely the delivery got killed before all your friends could be notified.
// Currently the only realistic fixes are to use a reliable server - which precludes shared hosting,
// or cut back on plugins which do remote deliveries.
// Call the background process that is delivering the item to the receivers
proc_run(PRIORITY_HIGH, "include/notifier.php", $notify_type, $post_id); proc_run(PRIORITY_HIGH, "include/notifier.php", $notify_type, $post_id);
logger('post_complete'); logger('post_complete');