keith/friendica
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #3461 from annando/1705-some-more-dba

Browse source 
DBA: poller / SQL errors / dbclean
This commit is contained in:
Hypolite Petovan 2017-05-13 00:22:01 -04:00 committed by GitHub
commit db0c3620c0
10 changed files with 72 additions and 102 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
boot.php
View file

@ -1077,14 +1077,10 @@ function proc_run($cmd) {
array_shift($argv);
$parameters = json_encode($argv);
$found = q("SELECT `id` FROM `workerqueue` WHERE `parameter` = '%s'", dbesc($parameters));
$found = dba::select('workerqueue', array('id'), array('parameter' => $parameters), array('limit' => 1));
if (!dbm::is_result($found)) {
q("INSERT INTO `workerqueue` (`parameter`, `created`, `priority`)
VALUES ('%s', '%s', %d)",
dbesc($parameters),
dbesc(datetime_convert()),
intval($priority));
dba::insert('workerqueue', array('parameter' => $parameters, 'created' => datetime_convert(), 'priority' => $priority));
}
// Should we quit and wait for the poller to be called as a cronjob?

24
include/Contact.php
View file

@ -23,29 +23,7 @@ function user_remove($uid) {
$r[0]['nickname']
);
/// @todo Should be done in a background job since this likely will run into a time out
// don't delete yet, will be done later when contacts have deleted my stuff
// q("DELETE FROM `contact` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `gcign` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `group` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `group_member` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `intro` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `event` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `item` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `item_id` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `mail` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `mailacct` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `manage` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `notify` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `photo` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `attach` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `profile` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `profile_check` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `pconfig` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `search` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `spam` WHERE `uid` = %d", intval($uid));
// don't delete yet, will be done later when contacts have deleted my stuff
// q("DELETE FROM `user` WHERE `uid` = %d", intval($uid));
// The user and related data will be deleted in "cron_expire_and_remove_users" (cronjobs.php)
q("UPDATE `user` SET `account_removed` = 1, `account_expires_on` = UTC_TIMESTAMP() WHERE `uid` = %d", intval($uid));
proc_run(PRIORITY_HIGH, "include/notifier.php", "removeme", $uid);

5
include/cronjobs.php
View file

@ -110,12 +110,11 @@ function cron_expire_and_remove_users() {
AND `account_expires_on` > '%s'
AND `account_expires_on` < UTC_TIMESTAMP()", dbesc(NULL_DATE));
// delete user and contact records for recently removed accounts
// delete user records for recently removed accounts
$r = q("SELECT * FROM `user` WHERE `account_removed` AND `account_expires_on` < UTC_TIMESTAMP() - INTERVAL 3 DAY");
if (dbm::is_result($r)) {
foreach ($r as $user) {
q("DELETE FROM `contact` WHERE `uid` = %d", intval($user['uid']));
q("DELETE FROM `user` WHERE `uid` = %d", intval($user['uid']));
dba::delete('user', array('uid' => $user['uid']));
}
}
}

43
include/dbclean.php
View file

@ -18,13 +18,11 @@ function dbclean_run(&$argv, &$argc) {
}
if ($stage == 0) {
proc_run(PRIORITY_LOW, 'include/dbclean.php', 1);
proc_run(PRIORITY_LOW, 'include/dbclean.php', 2);
proc_run(PRIORITY_LOW, 'include/dbclean.php', 3);
proc_run(PRIORITY_LOW, 'include/dbclean.php', 4);
proc_run(PRIORITY_LOW, 'include/dbclean.php', 5);
proc_run(PRIORITY_LOW, 'include/dbclean.php', 6);
proc_run(PRIORITY_LOW, 'include/dbclean.php', 7);
for ($i = 1; $i <= 7; $i++) {
if (!Config::get('system', 'finished-dbclean-'.$i)) {
proc_run(PRIORITY_LOW, 'include/dbclean.php', $i);
}
}
} else {
remove_orphans($stage);
}
@ -54,9 +52,15 @@ function remove_orphans($stage = 0) {
}
} else {
logger("No global item orphans found");
}
dba::close($r);
logger("Done deleting ".$count." old global item entries from item table without user copy");
// We will eventually set this value when we found a good way to delete these items in another way.
// if ($count < $limit) {
// Config::set('system', 'finished-dbclean-1', true);
// }
} elseif ($stage == 2) {
logger("Deleting items without parents");
$r = dba::p("SELECT `id` FROM `item` WHERE NOT EXISTS (SELECT `id` FROM `item` AS `i` WHERE `item`.`parent` = `i`.`id`) LIMIT ".intval($limit));
@ -71,6 +75,10 @@ function remove_orphans($stage = 0) {
}
dba::close($r);
logger("Done deleting ".$count." items without parents");
if ($count < $limit) {
Config::set('system', 'finished-dbclean-2', true);
}
} elseif ($stage == 3) {
logger("Deleting orphaned data from thread table");
$r = dba::p("SELECT `iid` FROM `thread` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `thread`.`iid`) LIMIT ".intval($limit));
@ -83,9 +91,12 @@ function remove_orphans($stage = 0) {
} else {
logger("No thread orphans found");
}
dba::close($r);
logger("Done deleting ".$count." orphaned data from thread table");
if ($count < $limit) {
Config::set('system', 'finished-dbclean-3', true);
}
} elseif ($stage == 4) {
logger("Deleting orphaned data from notify table");
$r = dba::p("SELECT `iid` FROM `notify` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `notify`.`iid`) LIMIT ".intval($limit));
@ -100,6 +111,10 @@ function remove_orphans($stage = 0) {
}
dba::close($r);
logger("Done deleting ".$count." orphaned data from notify table");
if ($count < $limit) {
Config::set('system', 'finished-dbclean-4', true);
}
} elseif ($stage == 5) {
logger("Deleting orphaned data from notify-threads table");
$r = dba::p("SELECT `id` FROM `notify-threads` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `notify-threads`.`master-parent-item`) LIMIT ".intval($limit));
@ -114,6 +129,10 @@ function remove_orphans($stage = 0) {
}
dba::close($r);
logger("Done deleting ".$count." orphaned data from notify-threads table");
if ($count < $limit) {
Config::set('system', 'finished-dbclean-5', true);
}
} elseif ($stage == 6) {
logger("Deleting orphaned data from sign table");
$r = dba::p("SELECT `iid` FROM `sign` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `sign`.`iid`) LIMIT ".intval($limit));
@ -128,6 +147,10 @@ function remove_orphans($stage = 0) {
}
dba::close($r);
logger("Done deleting ".$count." orphaned data from sign table");
if ($count < $limit) {
Config::set('system', 'finished-dbclean-6', true);
}
} elseif ($stage == 7) {
logger("Deleting orphaned data from term table");
$r = dba::p("SELECT `oid` FROM `term` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `term`.`oid`) LIMIT ".intval($limit));
@ -142,6 +165,10 @@ function remove_orphans($stage = 0) {
}
dba::close($r);
logger("Done deleting ".$count." orphaned data from term table");
if ($count < $limit) {
Config::set('system', 'finished-dbclean-7', true);
}
}
// Call it again if not all entries were purged

27
include/items.php
View file

@ -501,6 +501,20 @@ function item_store($arr, $force_parent = false, $notify = false, $dontcache = f
}
}
if ($notify) {
$guid_prefix = "";
} elseif ((trim($arr['guid']) == "") AND (trim($arr['plink']) != "")) {
$arr['guid'] = uri_to_guid($arr['plink']);
} elseif ((trim($arr['guid']) == "") AND (trim($arr['uri']) != "")) {
$arr['guid'] = uri_to_guid($arr['uri']);
} else {
$parsed = parse_url($arr["author-link"]);
$guid_prefix = hash("crc32", $parsed["host"]);
}
$arr['guid'] = ((x($arr, 'guid')) ? notags(trim($arr['guid'])) : get_guid(32, $guid_prefix));
$arr['uri'] = ((x($arr, 'uri')) ? notags(trim($arr['uri'])) : item_new_uri($a->get_hostname(), $uid, $arr['guid']));
// Store conversation data
$arr = store_conversation($arr);
@ -585,20 +599,7 @@ function item_store($arr, $force_parent = false, $notify = false, $dontcache = f
item_add_language_opt($arr);
if ($notify) {
$guid_prefix = "";
} elseif ((trim($arr['guid']) == "") AND (trim($arr['plink']) != "")) {
$arr['guid'] = uri_to_guid($arr['plink']);
} elseif ((trim($arr['guid']) == "") AND (trim($arr['uri']) != "")) {
$arr['guid'] = uri_to_guid($arr['uri']);
} else {
$parsed = parse_url($arr["author-link"]);
$guid_prefix = hash("crc32", $parsed["host"]);
}
$arr['wall'] = ((x($arr, 'wall')) ? intval($arr['wall']) : 0);
$arr['guid'] = ((x($arr, 'guid')) ? notags(trim($arr['guid'])) : get_guid(32, $guid_prefix));
$arr['uri'] = ((x($arr, 'uri')) ? notags(trim($arr['uri'])) : item_new_uri($a->get_hostname(), $uid, $arr['guid']));
$arr['extid'] = ((x($arr, 'extid')) ? notags(trim($arr['extid'])) : '');
$arr['author-name'] = ((x($arr, 'author-name')) ? trim($arr['author-name']) : '');
$arr['author-link'] = ((x($arr, 'author-link')) ? notags(trim($arr['author-link'])) : '');

25
include/poller.php
View file

@ -121,12 +121,8 @@ function poller_execute($queue) {
return false;
}
$upd = q("UPDATE `workerqueue` SET `executed` = '%s', `pid` = %d WHERE `id` = %d AND `pid` = 0",
dbesc(datetime_convert()),
intval($mypid),
intval($queue["id"]));
if (!$upd) {
if (!dba::update('workerqueue', array('executed' => datetime_convert(), 'pid' => $mypid),
array('id' => $queue["id"], 'pid' => 0))) {
logger("Couldn't update queue entry ".$queue["id"]." - skip this execution", LOGGER_DEBUG);
dba::commit();
return true;
@ -156,7 +152,7 @@ function poller_execute($queue) {
if (!validate_include($include)) {
logger("Include file ".$argv[0]." is not valid!");
q("DELETE FROM `workerqueue` WHERE `id` = %d", intval($queue["id"]));
dba::delete('workerqueue', array('id' => $queue["id"]));
return true;
}
@ -168,7 +164,7 @@ function poller_execute($queue) {
poller_exec_function($queue, $funcname, $argv);
q("DELETE FROM `workerqueue` WHERE `id` = %d", intval($queue["id"]));
dba::delete('workerqueue', array('id' => $queue["id"]));
} else {
logger("Function ".$funcname." does not exist");
}
@ -376,8 +372,8 @@ function poller_kill_stale_workers() {
foreach ($r AS $pid) {
if (!posix_kill($pid["pid"], 0)) {
q("UPDATE `workerqueue` SET `executed` = '%s', `pid` = 0 WHERE `pid` = %d",
dbesc(NULL_DATE), intval($pid["pid"]));
dba::update('workerqueue', array('executed' => NULL_DATE, 'pid' => 0),
array('pid' => $pid["pid"]));
} else {
// Kill long running processes
@ -401,12 +397,9 @@ function poller_kill_stale_workers() {
// We killed the stale process.
// To avoid a blocking situation we reschedule the process at the beginning of the queue.
// Additionally we are lowering the priority.
q("UPDATE `workerqueue` SET `executed` = '%s', `created` = '%s',
`priority` = %d, `pid` = 0 WHERE `pid` = %d",
dbesc(NULL_DATE),
dbesc(datetime_convert()),
intval(PRIORITY_NEGLIGIBLE),
intval($pid["pid"]));
dba::update('workerqueue',
array('executed' => NULL_DATE, 'created' => datetime_convert(), 'priority' => PRIORITY_NEGLIGIBLE, 'pid' => 0),
array('pid' => $pid["pid"]));
} else {
logger("Worker process ".$pid["pid"]." (".implode(" ", $argv).") now runs for ".round($duration)." of ".$max_duration." allowed minutes. That's okay.", LOGGER_DEBUG);
}

4
include/threads.php
View file

@ -245,9 +245,7 @@ function delete_thread($itemid, $itemuri = "") {
intval($item["uid"])
);
if (!dbm::is_result($r)) {
$r = q("DELETE FROM `item` WHERE `uri` = '%s' AND `uid` = 0",
dbesc($itemuri)
);
dba::delete('item', array('uri' => $itemuri, 'uid' => 0));
logger("delete_thread: Deleted shadow for item ".$itemuri." - ".print_r($result, true), LOGGER_DEBUG);
}
}

12
include/uimport.php
View file

@ -63,16 +63,6 @@ function db_import_assoc($table, $arr) {
return q($query);
}
function import_cleanup($newuid) {
q("DELETE FROM `user` WHERE uid = %d", $newuid);
q("DELETE FROM `contact` WHERE uid = %d", $newuid);
q("DELETE FROM `profile` WHERE uid = %d", $newuid);
q("DELETE FROM `photo` WHERE uid = %d", $newuid);
q("DELETE FROM `group` WHERE uid = %d", $newuid);
q("DELETE FROM `group_member` WHERE uid = %d", $newuid);
q("DELETE FROM `pconfig` WHERE uid = %d", $newuid);
}
/**
* @brief Import account file exported from mod/uexport
*
@ -174,7 +164,7 @@ function import_account(App $a, $file) {
if ($r === false) {
logger("uimport:insert profile " . $profile['profile-name'] . " : ERROR : " . last_error(), LOGGER_NORMAL);
info(t("User profile creation error"));
import_cleanup($newuid);
dba::delete('user', array('uid' => $newuid));
return;
}
}

7
include/user.php
View file

@ -226,9 +226,7 @@ function create_user($arr) {
);
if ((dbm::is_result($r)) && (count($r) > 1) && $newuid) {
$result['message'] .= t('Nickname is already registered. Please choose another.') . EOL;
q("DELETE FROM `user` WHERE `uid` = %d",
intval($newuid)
);
dba::delete('user', array('uid' => $newuid));
return $result;
}
@ -248,8 +246,7 @@ function create_user($arr) {
if ($r === false) {
$result['message'] .= t('An error occurred creating your default profile. Please try again.') . EOL;
// Start fresh next time.
$r = q("DELETE FROM `user` WHERE `uid` = %d",
intval($newuid));
dba::delete('user', array('uid' => $newuid));
return $result;
}

19
mod/regmod.php
View file

@ -74,27 +74,18 @@ function user_deny($hash) {
dbesc($hash)
);
if(! dbm::is_result($register))
if (!dbm::is_result($register)) {
return false;
}
$user = q("SELECT * FROM `user` WHERE `uid` = %d LIMIT 1",
intval($register[0]['uid'])
);
$r = q("DELETE FROM `user` WHERE `uid` = %d",
intval($register[0]['uid'])
);
$r = q("DELETE FROM `contact` WHERE `uid` = %d",
intval($register[0]['uid'])
);
$r = q("DELETE FROM `profile` WHERE `uid` = %d",
intval($register[0]['uid'])
);
dba::delete('user', array('uid' => $register[0]['uid']));
dba::delete('register', array('hash' => $register[0]['hash']));
$r = q("DELETE FROM `register` WHERE `hash` = '%s'",
dbesc($register[0]['hash'])
);
notice( sprintf(t('Registration revoked for %s'), $user[0]['username']) . EOL);
notice(sprintf(t('Registration revoked for %s'), $user[0]['username']) . EOL);
return true;
}