From 6ef24890bb0795ae346140ec489ed868b30160f6 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 11 May 2017 20:13:45 +0000 Subject: [PATCH 1/6] dbclean is now using the new database functions / new functions for commit, rollback and transaction --- include/dba.php | 71 ++++++++++++++++++++++++++++++++++++++------- include/dbclean.php | 43 ++++++++++++++++++--------- include/items.php | 15 +++++----- include/poller.php | 12 ++++---- mod/item.php | 7 ++--- src/App.php | 4 +-- 6 files changed, 108 insertions(+), 44 deletions(-) diff --git a/include/dba.php b/include/dba.php index d0ebda191..39eb1f460 100644 --- a/include/dba.php +++ b/include/dba.php @@ -22,6 +22,7 @@ class dba { public $connected = false; public $error = false; private $_server_info = ''; + private $in_transaction = false; private static $dbo; private static $relation = array(); @@ -602,7 +603,8 @@ class dba { } if (self::$dbo->errorno != 0) { - logger('DB Error '.self::$dbo->errorno.': '.self::$dbo->error."\n".self::replace_parameters($sql, $args)); + logger('DB Error '.self::$dbo->errorno.': '.self::$dbo->error."\n". + $a->callstack(8))."\n".self::replace_parameters($sql, $args); } $a->save_timestamp($stamp1, 'database'); @@ -779,6 +781,48 @@ class dba { return self::e($sql, $param); } + /** + * @brief Starts a transaction + * + * @return boolean Was the command executed successfully? + */ + static public function transaction() { + if (!self::e('COMMIT')) { + return false; + } + if (!self::e('START TRANSACTION')) { + return false; + } + self::$in_transaction = true; + return true; + } + + /** + * @brief Does a commit + * + * @return boolean Was the command executed successfully? + */ + static public function commit() { + if (!self::e('COMMIT')) { + return false; + } + self::$in_transaction = false; + return true; + } + + /** + * @brief Does a rollback + * + * @return boolean Was the command executed successfully? + */ + static public function rollback() { + if (!self::e('ROLLBACK')) { + return false; + } + self::$in_transaction = false; + return true; + } + /** * @brief Build the array with the table relations * @@ -805,12 +849,12 @@ class dba { * * @param string $table Table name * @param array $param parameter array - * @param boolean $in_commit Internal use: Only do a commit after the last delete + * @param boolean $in_process Internal use: Only do a commit after the last delete * @param array $callstack Internal use: prevent endless loops * - * @return boolean|array was the delete successfull? When $in_commit is set: deletion data + * @return boolean|array was the delete successfull? When $in_process is set: deletion data */ - static public function delete($table, $param, $in_commit = false, &$callstack = array()) { + static public function delete($table, $param, $in_process = false, &$callstack = array()) { $commands = array(); @@ -872,10 +916,11 @@ class dba { } } - if (!$in_commit) { + if (!$in_process) { // Now we finalize the process - self::p("COMMIT"); - self::p("START TRANSACTION"); + if (!self::$in_transaction) { + self::transaction(); + } $compacted = array(); $counter = array(); @@ -887,7 +932,9 @@ class dba { logger(dba::replace_parameters($sql, $command['param']), LOGGER_DATA); if (!self::e($sql, $command['param'])) { - self::p("ROLLBACK"); + if (!self::$in_transaction) { + self::rollback(); + } return false; } } else { @@ -915,13 +962,17 @@ class dba { logger(dba::replace_parameters($sql, $field_values), LOGGER_DATA); if (!self::e($sql, $field_values)) { - self::p("ROLLBACK"); + if (!self::$in_transaction) { + self::rollback(); + } return false; } } } } - self::p("COMMIT"); + if (!self::$in_transaction) { + self::commit(); + } return true; } diff --git a/include/dbclean.php b/include/dbclean.php index f31bfef8a..29842834c 100644 --- a/include/dbclean.php +++ b/include/dbclean.php @@ -50,11 +50,13 @@ function remove_orphans($stage = 0) { if ($count > 0) { logger("found global item orphans: ".$count); while ($orphan = dba::fetch($r)) { - q("DELETE FROM `item` WHERE `id` = %d", intval($orphan["id"])); + dba::delete('item', array('id' => $orphan["id"])); } + } else { + logger("No global item orphans found"); } dba::close($r); - logger("Done deleting old global item entries from item table without user copy"); + logger("Done deleting ".$count." old global item entries from item table without user copy"); } if (($stage == 2) OR ($stage == 0)) { @@ -64,11 +66,13 @@ function remove_orphans($stage = 0) { if ($count > 0) { logger("found item orphans without parents: ".$count); while ($orphan = dba::fetch($r)) { - q("DELETE FROM `item` WHERE `id` = %d", intval($orphan["id"])); + dba::delete('item', array('id' => $orphan["id"])); } + } else { + logger("No item orphans without parents found"); } dba::close($r); - logger("Done deleting items without parents"); + logger("Done deleting ".$count." items without parents"); } if (($stage == 3) OR ($stage == 0)) { @@ -78,11 +82,14 @@ function remove_orphans($stage = 0) { if ($count > 0) { logger("found thread orphans: ".$count); while ($orphan = dba::fetch($r)) { - q("DELETE FROM `thread` WHERE `iid` = %d", intval($orphan["iid"])); + dba::delete('thread', array('iid' => $orphan["iid"])); } + } else { + logger("No thread orphans found"); } + dba::close($r); - logger("Done deleting orphaned data from thread table"); + logger("Done deleting ".$count." orphaned data from thread table"); } if (($stage == 4) OR ($stage == 0)) { @@ -92,11 +99,13 @@ function remove_orphans($stage = 0) { if ($count > 0) { logger("found notify orphans: ".$count); while ($orphan = dba::fetch($r)) { - q("DELETE FROM `notify` WHERE `iid` = %d", intval($orphan["iid"])); + dba::delete('notify', array('iid' => $orphan["iid"])); } + } else { + logger("No notify orphans found"); } dba::close($r); - logger("Done deleting orphaned data from notify table"); + logger("Done deleting ".$count." orphaned data from notify table"); } if (($stage == 5) OR ($stage == 0)) { @@ -106,11 +115,13 @@ function remove_orphans($stage = 0) { if ($count > 0) { logger("found notify-threads orphans: ".$count); while ($orphan = dba::fetch($r)) { - q("DELETE FROM `notify-threads` WHERE `id` = %d", intval($orphan["id"])); + dba::delete('notify-threads', array('id' => $orphan["id"])); } + } else { + logger("No notify-threads orphans found"); } dba::close($r); - logger("Done deleting orphaned data from notify-threads table"); + logger("Done deleting ".$count." orphaned data from notify-threads table"); } @@ -121,11 +132,13 @@ function remove_orphans($stage = 0) { if ($count > 0) { logger("found sign orphans: ".$count); while ($orphan = dba::fetch($r)) { - q("DELETE FROM `sign` WHERE `iid` = %d", intval($orphan["iid"])); + dba::delete('sign', array('iid' => $orphan["iid"])); } + } else { + logger("No sign orphans found"); } dba::close($r); - logger("Done deleting orphaned data from sign table"); + logger("Done deleting ".$count." orphaned data from sign table"); } @@ -136,11 +149,13 @@ function remove_orphans($stage = 0) { if ($count > 0) { logger("found term orphans: ".$count); while ($orphan = dba::fetch($r)) { - q("DELETE FROM `term` WHERE `oid` = %d", intval($orphan["oid"])); + dba::delete('term', array('oid' => $orphan["oid"])); } + } else { + logger("No term orphans found"); } dba::close($r); - logger("Done deleting orphaned data from term table"); + logger("Done deleting ".$count." orphaned data from term table"); } // Call it again if not all entries were purged diff --git a/include/items.php b/include/items.php index 3c5e88a38..3b790b6a1 100644 --- a/include/items.php +++ b/include/items.php @@ -951,8 +951,7 @@ function item_store($arr, $force_parent = false, $notify = false, $dontcache = f logger('item_store: ' . print_r($arr,true), LOGGER_DATA); - q("COMMIT"); - q("START TRANSACTION;"); + dba::transaction(); $r = dbq("INSERT INTO `item` (`" . implode("`, `", array_keys($arr)) @@ -974,7 +973,7 @@ function item_store($arr, $force_parent = false, $notify = false, $dontcache = f } } else { // This can happen - for example - if there are locking timeouts. - q("ROLLBACK"); + dba::rollback(); // Store the data into a spool file so that we can try again later. @@ -999,7 +998,7 @@ function item_store($arr, $force_parent = false, $notify = false, $dontcache = f if ($current_post == 0) { // This is one of these error messages that never should occur. logger("couldn't find created item - we better quit now."); - q("ROLLBACK"); + dba::rollback(); return 0; } @@ -1014,7 +1013,7 @@ function item_store($arr, $force_parent = false, $notify = false, $dontcache = f if (!dbm::is_result($r)) { // This shouldn't happen, since COUNT always works when the database connection is there. logger("We couldn't count the stored entries. Very strange ..."); - q("ROLLBACK"); + dba::rollback(); return 0; } @@ -1024,12 +1023,12 @@ function item_store($arr, $force_parent = false, $notify = false, $dontcache = f // Yes, we could do a rollback here - but we are having many users with MyISAM. q("DELETE FROM `item` WHERE `id` = %d", intval($current_post)); - q("COMMIT"); + dba::commit(); return 0; } elseif ($r[0]["entries"] == 0) { // This really should never happen since we quit earlier if there were problems. logger("Something is terribly wrong. We haven't found our created entry."); - q("ROLLBACK"); + dba::rollback(); return 0; } @@ -1109,7 +1108,7 @@ function item_store($arr, $force_parent = false, $notify = false, $dontcache = f update_thread($parent_id); } - q("COMMIT"); + dba::commit(); /* * Due to deadlock issues with the "term" table we are doing these steps after the commit. diff --git a/include/poller.php b/include/poller.php index 08e71fd48..eb97d4853 100644 --- a/include/poller.php +++ b/include/poller.php @@ -128,7 +128,7 @@ function poller_execute($queue) { if (!$upd) { logger("Couldn't update queue entry ".$queue["id"]." - skip this execution", LOGGER_DEBUG); - q("COMMIT"); + dba::commit(); return true; } @@ -136,18 +136,18 @@ function poller_execute($queue) { $id = q("SELECT `pid`, `executed` FROM `workerqueue` WHERE `id` = %d", intval($queue["id"])); if (!$id) { logger("Queue item ".$queue["id"]." vanished - skip this execution", LOGGER_DEBUG); - q("COMMIT"); + dba::commit(); return true; } elseif ((strtotime($id[0]["executed"]) <= 0) OR ($id[0]["pid"] == 0)) { logger("Entry for queue item ".$queue["id"]." wasn't stored - skip this execution", LOGGER_DEBUG); - q("COMMIT"); + dba::commit(); return true; } elseif ($id[0]["pid"] != $mypid) { logger("Queue item ".$queue["id"]." is to be executed by process ".$id[0]["pid"]." and not by me (".$mypid.") - skip this execution", LOGGER_DEBUG); - q("COMMIT"); + dba::commit(); return true; } - q("COMMIT"); + dba::commit(); $argv = json_decode($queue["parameter"]); @@ -558,7 +558,7 @@ function poller_passing_slow(&$highest_priority) { */ function poller_worker_process() { - q("START TRANSACTION;"); + dba::transaction(); // Check if we should pass some low priority process $highest_priority = 0; diff --git a/mod/item.php b/mod/item.php index f2d7babcd..f696a96a7 100644 --- a/mod/item.php +++ b/mod/item.php @@ -809,8 +809,7 @@ function item_post(App $a) { $post_id = 0; } - q("COMMIT"); - q("START TRANSACTION;"); + dba::transaction(); $r = q("INSERT INTO `item` (`guid`, `extid`, `uid`,`type`,`wall`,`gravity`, `network`, `contact-id`, `owner-name`,`owner-link`,`owner-avatar`, `owner-id`, @@ -900,7 +899,7 @@ function item_post(App $a) { } if ($post_id == 0) { - q("COMMIT"); + dba::commit(); logger('mod_item: unable to retrieve post that was just stored.'); notice(t('System error. Post not saved.') . EOL); goaway($return_path); @@ -1026,7 +1025,7 @@ function item_post(App $a) { update_thread($parent, true); } - q("COMMIT"); + dba::commit(); create_tags_from_item($post_id); create_files_from_item($post_id); diff --git a/src/App.php b/src/App.php index 15538e88d..c52b52820 100644 --- a/src/App.php +++ b/src/App.php @@ -722,8 +722,8 @@ class App { * * @return string */ - function callstack() { - $trace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS, 6); + function callstack($depth = 4) { + $trace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS, $depth + 2); // We remove the first two items from the list since they contain data that we don't need. array_shift($trace); From c52f7657ab5354c4a5f0ffcb23d401e195ad0b6b Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 11 May 2017 20:19:43 +0000 Subject: [PATCH 2/6] Now it should work --- include/dba.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/dba.php b/include/dba.php index 39eb1f460..b9d7f71e5 100644 --- a/include/dba.php +++ b/include/dba.php @@ -22,7 +22,7 @@ class dba { public $connected = false; public $error = false; private $_server_info = ''; - private $in_transaction = false; + private static $in_transaction = false; private static $dbo; private static $relation = array(); From c00a5223d669d462951cb784095c4235cbb48faf Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 12 May 2017 04:33:52 +0000 Subject: [PATCH 3/6] Removed code from the pre worker era. --- include/dbclean.php | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/include/dbclean.php b/include/dbclean.php index 29842834c..3b3ea7039 100644 --- a/include/dbclean.php +++ b/include/dbclean.php @@ -41,7 +41,7 @@ function remove_orphans($stage = 0) { // We split the deletion in many small tasks $limit = 1000; - if (($stage == 1) OR ($stage == 0)) { + if ($stage == 1) { logger("Deleting old global item entries from item table without user copy"); $r = dba::p("SELECT `id` FROM `item` WHERE `uid` = 0 AND NOT EXISTS (SELECT `guid` FROM `item` AS `i` WHERE `item`.`guid` = `i`.`guid` AND `i`.`uid` != 0) @@ -57,9 +57,7 @@ function remove_orphans($stage = 0) { } dba::close($r); logger("Done deleting ".$count." old global item entries from item table without user copy"); - } - - if (($stage == 2) OR ($stage == 0)) { + } elseif ($stage == 2) { logger("Deleting items without parents"); $r = dba::p("SELECT `id` FROM `item` WHERE NOT EXISTS (SELECT `id` FROM `item` AS `i` WHERE `item`.`parent` = `i`.`id`) LIMIT ".intval($limit)); $count = dba::num_rows($r); @@ -73,9 +71,7 @@ function remove_orphans($stage = 0) { } dba::close($r); logger("Done deleting ".$count." items without parents"); - } - - if (($stage == 3) OR ($stage == 0)) { + } elseif ($stage == 3) { logger("Deleting orphaned data from thread table"); $r = dba::p("SELECT `iid` FROM `thread` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `thread`.`iid`) LIMIT ".intval($limit)); $count = dba::num_rows($r); @@ -90,9 +86,7 @@ function remove_orphans($stage = 0) { dba::close($r); logger("Done deleting ".$count." orphaned data from thread table"); - } - - if (($stage == 4) OR ($stage == 0)) { + } elseif ($stage == 4) { logger("Deleting orphaned data from notify table"); $r = dba::p("SELECT `iid` FROM `notify` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `notify`.`iid`) LIMIT ".intval($limit)); $count = dba::num_rows($r); @@ -106,9 +100,7 @@ function remove_orphans($stage = 0) { } dba::close($r); logger("Done deleting ".$count." orphaned data from notify table"); - } - - if (($stage == 5) OR ($stage == 0)) { + } elseif ($stage == 5) { logger("Deleting orphaned data from notify-threads table"); $r = dba::p("SELECT `id` FROM `notify-threads` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `notify-threads`.`master-parent-item`) LIMIT ".intval($limit)); $count = dba::num_rows($r); @@ -122,10 +114,7 @@ function remove_orphans($stage = 0) { } dba::close($r); logger("Done deleting ".$count." orphaned data from notify-threads table"); - } - - - if (($stage == 6) OR ($stage == 0)) { + } elseif ($stage == 6) { logger("Deleting orphaned data from sign table"); $r = dba::p("SELECT `iid` FROM `sign` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `sign`.`iid`) LIMIT ".intval($limit)); $count = dba::num_rows($r); @@ -139,10 +128,7 @@ function remove_orphans($stage = 0) { } dba::close($r); logger("Done deleting ".$count." orphaned data from sign table"); - } - - - if (($stage == 7) OR ($stage == 0)) { + } elseif ($stage == 7) { logger("Deleting orphaned data from term table"); $r = dba::p("SELECT `oid` FROM `term` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `term`.`oid`) LIMIT ".intval($limit)); $count = dba::num_rows($r); From 11e524a5555b316e1787463acf5b318e09a847c5 Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 12 May 2017 06:17:48 +0000 Subject: [PATCH 4/6] This item removal is much more cleaner --- include/expire.php | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/include/expire.php b/include/expire.php index da121157d..73bffb20d 100644 --- a/include/expire.php +++ b/include/expire.php @@ -10,8 +10,13 @@ function expire_run(&$argv, &$argc){ require_once('include/Contact.php'); // physically remove anything that has been deleted for more than two months - - $r = q("DELETE FROM `item` WHERE `deleted` = 1 AND `changed` < UTC_TIMESTAMP() - INTERVAL 60 DAY"); + $r = dba::p("SELECT `id` FROM `item` WHERE `deleted` AND `changed` < UTC_TIMESTAMP() - INTERVAL 60 DAY"); + if (dbm::is_result($r)) { + while ($row = dba::fetch($r)) { + dba::delete('item', array('id' => $row['id'])); + } + dba::close($r); + } // make this optional as it could have a performance impact on large sites From 58b2b1dbef0b5f9b914d861e32ee12ad8c52bdf4 Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 12 May 2017 06:30:45 +0000 Subject: [PATCH 5/6] Some more cleaner delete --- include/items.php | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/include/items.php b/include/items.php index 3b790b6a1..0033decea 100644 --- a/include/items.php +++ b/include/items.php @@ -1377,10 +1377,7 @@ function tag_deliver($uid, $item_id) { // mmh.. no mention.. community page or private group... no wall.. no origin.. top-post (not a comment) // delete it! logger("tag_deliver: no-mention top-level post to communuty or private group. delete."); - q("DELETE FROM item WHERE id = %d and uid = %d", - intval($item_id), - intval($uid) - ); + dba::delete('item', array('id' => $item_id)); return true; } return; @@ -2236,23 +2233,6 @@ function drop_item($id, $interactive = true) { // ignore the result } - - // clean up item_id and sign meta-data tables - - /* - /// @TODO Old code - caused very long queries and warning entries in the mysql logfiles: - - $r = q("DELETE FROM item_id where iid in (select id from item where parent = %d and uid = %d)", - intval($item['id']), - intval($item['uid']) - ); - - $r = q("DELETE FROM sign where iid in (select id from item where parent = %d and uid = %d)", - intval($item['id']), - intval($item['uid']) - ); - */ - // The new code splits the queries since the mysql optimizer really has bad problems with subqueries // Creating list of parents From acd9f453d75026f4e79a0824fbd655548f4d84d2 Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 12 May 2017 06:55:04 +0000 Subject: [PATCH 6/6] And another one --- include/items.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/items.php b/include/items.php index 0033decea..820c79e13 100644 --- a/include/items.php +++ b/include/items.php @@ -1022,7 +1022,7 @@ function item_store($arr, $force_parent = false, $notify = false, $dontcache = f logger('Duplicated post occurred. uri = ' . $arr['uri'] . ' uid = ' . $arr['uid']); // Yes, we could do a rollback here - but we are having many users with MyISAM. - q("DELETE FROM `item` WHERE `id` = %d", intval($current_post)); + dba::delete('item', array('id' => $current_post)); dba::commit(); return 0; } elseif ($r[0]["entries"] == 0) {