keith/friendica
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #13118 from nupplaphil/feat/db_refac

Browse source 
DB encapsulations
This commit is contained in:
Hypolite Petovan 2023-05-14 08:46:39 -04:00 committed by GitHub
commit 89d57a3484
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 238 additions and 85 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
src/Core/Update.php
View file

@ -129,20 +129,6 @@ class Update
DI::lock()->release('dbupdate', true);
}
if (!DBStructure::existsTable('config')) {
DBA::e(<<<EOF
CREATE TABLE IF NOT EXISTS `config` (
`id` int unsigned NOT NULL auto_increment COMMENT '',
`cat` varbinary(50) NOT NULL DEFAULT '' COMMENT 'The category of the entry',
`k` varbinary(50) NOT NULL DEFAULT '' COMMENT 'The key of the entry',
`v` mediumtext COMMENT '',
PRIMARY KEY(`id`),
UNIQUE INDEX `cat_k` (`cat`,`k`)
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='main configuration storage';
EOF
);
}
$build = DI::config()->get('system', 'build');
if (empty($build)) {

6
src/Core/Worker/Cron.php
View file

@ -151,8 +151,8 @@ class Cron
// We are acquiring the two locks from the worker to avoid locking problems
if (DI::lock()->acquire(Worker::LOCK_PROCESS, 10)) {
if (DI::lock()->acquire(Worker::LOCK_WORKER, 10)) {
DBA::e("OPTIMIZE TABLE `workerqueue`");
DBA::e("OPTIMIZE TABLE `process`");
DBA::optimizeTable('workerqueue');
DBA::optimizeTable('process');
DI::lock()->release(Worker::LOCK_WORKER);
}
DI::lock()->release(Worker::LOCK_PROCESS);
@ -197,7 +197,7 @@ class Cron
// Optimizing this table only last seconds
if (DI::config()->get('system', 'optimize_tables')) {
Logger::info('Optimize start');
DBA::e("OPTIMIZE TABLE `post-delivery`");
DBA::optimizeTable('post-delivery');
Logger::info('Optimize end');
}
}

21
src/Database/DBA.php
View file

@ -821,6 +821,27 @@ class DBA
return DI::dba()->processlist();
}
/**
* Optimizes tables
*
* @param string $table a given table
*
* @return bool True, if successfully optimized, otherwise false
* @throws \Exception
*/
public static function optimizeTable(string $table): bool
{
return DI::dba()->optimizeTable($table);
}
/**
* Kill sleeping database processes
*/
public static function deleteSleepingProcesses()
{
DI::dba()->deleteSleepingProcesses();
}
/**
* Fetch a database variable
*

15
src/Database/DBStructure.php
View file

@ -57,6 +57,18 @@ class DBStructure
echo DI::l10n()->t('The database version had been set to %s.', $version);
}
/**
* Drops a specific table
*
* @param string $table the table name
*
* @return bool true if possible, otherwise false
*/
public static function dropTable(string $table): bool
{
return DBA::isResult(DBA::e('DROP TABLE ' . DBA::quoteIdentifier($table) . ';'));
}
/**
* Drop unused tables
*
@ -94,8 +106,7 @@ class DBStructure
$sql = 'DROP TABLE ' . DBA::quoteIdentifier($table) . ';';
echo $sql . "\n";
$result = DBA::e($sql);
if (!DBA::isResult($result)) {
if (!static::dropTable($table)) {
self::printUpdateError($sql);
}
} else {

43
src/Database/Database.php
View file

@ -1357,6 +1357,15 @@ class Database
}
$fields = $this->castFields($table, $fields);
$direct_fields = [];
foreach ($fields as $key => $value) {
if (is_numeric($key)) {
$direct_fields[] = $value;
unset($fields[$key]);
}
}
$table_string = DBA::buildTableString([$table]);
@ -1369,7 +1378,8 @@ class Database
}
$sql = "UPDATE " . $ignore . $table_string . " SET "
. implode(" = ?, ", array_map([DBA::class, 'quoteIdentifier'], array_keys($fields))) . " = ?"
. ((count($fields) > 0) ? implode(" = ?, ", array_map([DBA::class, 'quoteIdentifier'], array_keys($fields))) . " = ?" : "")
. ((count($direct_fields) > 0) ? ((count($fields) > 0) ? " , " : "") . implode(" , ", $direct_fields) : "")
. $condition_string;
// Combines the updated fields parameter values with the condition parameter values
@ -1758,6 +1768,37 @@ class Database
return (['list' => $statelist, 'amount' => $processes]);
}
/**
* Optimizes tables
*
* @param string $table a given table
*
* @return bool True, if successfully optimized, otherwise false
* @throws \Exception
*/
public function optimizeTable(string $table): bool
{
return $this->e("OPTIMIZE TABLE " . DBA::buildTableString([$table])) !== false;
}
/**
* Kill sleeping database processes
*
* @return void
*/
public function deleteSleepingProcesses()
{
$processes = $this->p("SHOW FULL PROCESSLIST");
while ($process = $this->fetch($processes)) {
if (($process['Command'] != 'Sleep') || ($process['Time'] < 300) || ($process['db'] != $this->databaseName())) {
continue;
}
$this->e("KILL ?", $process['Id']);
}
$this->close($processes);
}
/**
* Fetch a database variable
*

18
src/Database/DatabaseException.php
View file

@ -38,22 +38,24 @@ class DatabaseException extends Exception
*
* @link https://php.net/manual/en/exception.construct.php
*
* @param string $message The Database error message.
* @param int $code The Database error code.
* @param string $query The Database error query.
* @param Throwable $previous [optional] The previous throwable used for the exception chaining.
* @param string $message The Database error message.
* @param int $code The Database error code.
* @param string $query The Database error query.
* @param Throwable|null $previous [optional] The previous throwable used for the exception chaining.
*/
public function __construct(string $message, int $code, string $query, Throwable $previous = null)
{
parent::__construct($message, $code, $previous);
parent::__construct(sprintf('"%s" at "%s"', $message, $query) , $code, $previous);
$this->query = $query;
}
/**
* {@inheritDoc}
* Returns the query, which caused the exception
*
* @return string
*/
public function __toString()
public function getQuery(): string
{
return sprintf('Database error %d "%s" at "%s"', $this->message, $this->code, $this->query);
return $this->query;
}
}

20
src/Federation/Repository/DeliveryQueueItem.php
View file

@ -88,7 +88,10 @@ final class DeliveryQueueItem extends \Friendica\BaseRepository
public function remove(Entity\DeliveryQueueItem $deliveryQueueItem): bool
{
return $this->db->delete(self::$table_name, ['uri-id' => $deliveryQueueItem->postUriId, 'gsid' => $deliveryQueueItem->targetServerId]);
return $this->db->delete(self::$table_name, [
'uri-id' => $deliveryQueueItem->postUriId,
'gsid' => $deliveryQueueItem->targetServerId
]);
}
public function removeFailedByServerId(int $gsid, int $failedThreshold): bool
@ -98,16 +101,17 @@ final class DeliveryQueueItem extends \Friendica\BaseRepository
public function incrementFailed(Entity\DeliveryQueueItem $deliveryQueueItem): bool
{
return $this->db->e("
UPDATE " . DBA::buildTableString([self::$table_name]) . "
SET `failed` = `failed` + 1
WHERE `uri-id` = ? AND `gsid` = ?",
$deliveryQueueItem->postUriId, $deliveryQueueItem->targetServerId
);
return $this->db->update(self::$table_name, [
"`failed` = `failed` + 1"
], [
"`uri-id` = ? AND `gsid` = ?",
$deliveryQueueItem->postUriId,
$deliveryQueueItem->targetServerId
]);
}
public function optimizeStorage(): bool
{
return $this->db->e("OPTIMIZE TABLE " . DBA::buildTableString([self::$table_name]));
return $this->db->optimizeTable(self::$table_name);
}
}

2
src/Model/Post/Delivery.php
View file

@ -78,7 +78,7 @@ class Delivery
*/
public static function incrementFailed(int $uri_id, string $inbox)
{
return DBA::e('UPDATE `post-delivery` SET `failed` = `failed` + 1 WHERE `uri-id` = ? AND `inbox-id` = ?', $uri_id, ItemURI::getIdByURI($inbox));
return DBA::update('post-delivery', ["`failed` = `failed` + 1"], ['uri-id' => $uri_id, 'inbox-id' => ItemURI::getIdByURI($inbox)]);
}
public static function selectForInbox(string $inbox)

18
src/Model/Post/DeliveryData.php
View file

@ -82,27 +82,27 @@ class DeliveryData
*/
public static function incrementQueueDone(int $uri_id, int $protocol = 0)
{
$sql = '';
$increments = ["`queue_done` = `queue_done` + 1"];
switch ($protocol) {
case self::ACTIVITYPUB:
$sql = ", `activitypub` = `activitypub` + 1";
$increments[] = ["`activitypub` = `activitypub` + 1"];
break;
case self::DFRN:
$sql = ", `dfrn` = `dfrn` + 1";
$increments[] = ["`dfrn` = `dfrn` + 1"];
break;
case self::LEGACY_DFRN:
$sql = ", `legacy_dfrn` = `legacy_dfrn` + 1";
$increments[] = ["`legacy_dfrn` = `legacy_dfrn` + 1"];
break;
case self::DIASPORA:
$sql = ", `diaspora` = `diaspora` + 1";
$increments[] = ["`diaspora` = `diaspora` + 1"];
break;
case self::OSTATUS:
$sql = ", `ostatus` = `ostatus` + 1";
$increments[] = ["`ostatus` = `ostatus` + 1"];
break;
}
return DBA::e('UPDATE `post-delivery-data` SET `queue_done` = `queue_done` + 1' . $sql . ' WHERE `uri-id` = ?', $uri_id);
return DBA::update('post-delivery-data', $increments, ['uri-id' => $uri_id]);
}
/**
@ -116,7 +116,7 @@ class DeliveryData
*/
public static function incrementQueueFailed(int $uri_id)
{
return DBA::e('UPDATE `post-delivery-data` SET `queue_failed` = `queue_failed` + 1 WHERE `uri-id` = ?', $uri_id);
return DBA::update('post-delivery-data', ["`queue_failed` = `queue_failed` + 1"], ['uri-id' => $uri_id]);
}
/**
@ -129,7 +129,7 @@ class DeliveryData
*/
public static function incrementQueueCount(int $uri_id, int $increment = 1)
{
return DBA::e('UPDATE `post-delivery-data` SET `queue_count` = `queue_count` + ? WHERE `uri-id` = ?', $increment, $uri_id);
return DBA::update('post-delivery-data', ["`queue_count` = `queue_count` + $increment"], ['uri-id' => $uri_id]);
}
/**

2
src/Protocol/ActivityPub/Queue.php
View file

@ -312,7 +312,7 @@ class Queue
// Optimizing this table only last seconds
if (DI::config()->get('system', 'optimize_tables')) {
Logger::info('Optimize start');
DBA::e("OPTIMIZE TABLE `inbox-entry`");
DBA::optimizeTable('inbox-entry');
Logger::info('Optimize end');
}
}

11
src/Worker/Cron.php
View file

@ -163,15 +163,6 @@ class Cron
{
Logger::info('Looking for sleeping processes');
$processes = DBA::p("SHOW FULL PROCESSLIST");
while ($process = DBA::fetch($processes)) {
if (($process['Command'] != 'Sleep') || ($process['Time'] < 300) || ($process['db'] != DBA::databaseName())) {
continue;
}
DBA::e("KILL ?", $process['Id']);
Logger::notice('Killed sleeping process', ['id' => $process['Id']]);
}
DBA::close($processes);
DBA::deleteSleepingProcesses();
}
}

56
src/Worker/OptimizeTables.php
View file

@ -40,36 +40,36 @@ class OptimizeTables
Logger::info('Optimize start');
DBA::e("OPTIMIZE TABLE `cache`");
DBA::e("OPTIMIZE TABLE `locks`");
DBA::e("OPTIMIZE TABLE `oembed`");
DBA::e("OPTIMIZE TABLE `parsed_url`");
DBA::e("OPTIMIZE TABLE `session`");
DBA::optimizeTable('cache');
DBA::optimizeTable('locks');
DBA::optimizeTable('oembed');
DBA::optimizeTable('parsed_url');
DBA::optimizeTable('session');
if (DI::config()->get('system', 'optimize_all_tables')) {
DBA::e("OPTIMIZE TABLE `apcontact`");
DBA::e("OPTIMIZE TABLE `contact`");
DBA::e("OPTIMIZE TABLE `contact-relation`");
DBA::e("OPTIMIZE TABLE `conversation`");
DBA::e("OPTIMIZE TABLE `diaspora-contact`");
DBA::e("OPTIMIZE TABLE `diaspora-interaction`");
DBA::e("OPTIMIZE TABLE `fcontact`");
DBA::e("OPTIMIZE TABLE `gserver`");
DBA::e("OPTIMIZE TABLE `gserver-tag`");
DBA::e("OPTIMIZE TABLE `inbox-status`");
DBA::e("OPTIMIZE TABLE `item-uri`");
DBA::e("OPTIMIZE TABLE `notification`");
DBA::e("OPTIMIZE TABLE `notify`");
DBA::e("OPTIMIZE TABLE `photo`");
DBA::e("OPTIMIZE TABLE `post`");
DBA::e("OPTIMIZE TABLE `post-content`");
DBA::e("OPTIMIZE TABLE `post-delivery-data`");
DBA::e("OPTIMIZE TABLE `post-link`");
DBA::e("OPTIMIZE TABLE `post-thread`");
DBA::e("OPTIMIZE TABLE `post-thread-user`");
DBA::e("OPTIMIZE TABLE `post-user`");
DBA::e("OPTIMIZE TABLE `storage`");
DBA::e("OPTIMIZE TABLE `tag`");
DBA::optimizeTable('apcontact');
DBA::optimizeTable('contact');
DBA::optimizeTable('contact-relation');
DBA::optimizeTable('conversation');
DBA::optimizeTable('diaspora-contact');
DBA::optimizeTable('diaspora-interaction');
DBA::optimizeTable('fcontact');
DBA::optimizeTable('gserver');
DBA::optimizeTable('gserver-tag');
DBA::optimizeTable('inbox-status');
DBA::optimizeTable('item-uri');
DBA::optimizeTable('notification');
DBA::optimizeTable('notify');
DBA::optimizeTable('photo');
DBA::optimizeTable('post');
DBA::optimizeTable('post-content');
DBA::optimizeTable('post-delivery-data');
DBA::optimizeTable('post-link');
DBA::optimizeTable('post-thread');
DBA::optimizeTable('post-thread-user');
DBA::optimizeTable('post-user');
DBA::optimizeTable('storage');
DBA::optimizeTable('tag');
}
Logger::info('Optimize end');

9
tests/datasets/api.fixture.php
View file

@ -35,6 +35,15 @@ return [
'workerqueue',
'mail',
'post-delivery-data',
'gserver' => [
[
'url' => 'https://friendica.local',
'nurl' => 'http://friendica.local',
'register_policy' => 0,
'registered-users' => 0,
'network' => 'unkn',
],
],
// Base test config to avoid notice messages
'user' => [
[

88
tests/src/Database/DatabaseTest.php Normal file
View file

@ -0,0 +1,88 @@
<?php
/**
* @copyright Copyright (C) 2010-2023, the Friendica project
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
namespace Friendica\Test\src\Database;
use Friendica\Core\Config\Util\ConfigFileManager;
use Friendica\Core\Config\ValueObject\Cache;
use Friendica\Test\FixtureTest;
use Friendica\Test\Util\CreateDatabaseTrait;
class DatabaseTest extends FixtureTest
{
use CreateDatabaseTrait;
protected function setUp(): void
{
$this->setUpVfsDir();
parent::setUp();
$this->configCache = new Cache();
$this->configFileManager = new ConfigFileManager($this->root->url(), $this->root->url() . '/config/', $this->root->url() . '/static/');
}
/**
* Test, if directly updating a field is possible
*/
public function testUpdateIncrease()
{
$db = $this->getDbInstance();
self::assertTrue($db->insert('config', ['cat' => 'test', 'k' => 'inc', 'v' => 0]));
self::assertTrue($db->update('config', ["`v` = `v` + 1"], ['cat' => 'test', 'k' => 'inc']));
self::assertEquals(1, $db->selectFirst('config', ['v'], ['cat' => 'test', 'k' => 'inc'])['v']);
}
/**
* Test if combining directly field updates with normal updates is working
*/
public function testUpdateWithField()
{
$db = $this->getDbInstance();
self::assertEquals('https://friendica.local', $db->selectFirst('gserver', ['url'], ['nurl' => 'http://friendica.local'])['url']);
self::assertTrue($db->update('gserver', ['active-week-users' => 0], ['nurl' => 'http://friendica.local']));
self::assertTrue($db->update('gserver', [
'site_name' => 'test', "`registered-users` = `registered-users` + 1",
'info' => 'another test',
"`active-week-users` = `active-week-users` + 2"
], [
'nurl' => 'http://friendica.local'
]));
self::assertEquals(1, $db->selectFirst('gserver', ['registered-users'], ['nurl' => 'http://friendica.local'])['registered-users']);
self::assertEquals(2, $db->selectFirst('gserver', ['active-week-users'], ['nurl' => 'http://friendica.local'])['active-week-users']);
self::assertTrue($db->update('gserver', [
'site_name' => 'test', "`registered-users` = `registered-users` + 1",
'info' => 'another test'
], [
'nurl' => 'http://friendica.local'
]));
self::assertEquals(2, $db->selectFirst('gserver', ['registered-users'], ['nurl' => 'http://friendica.local'])['registered-users']);
self::assertTrue($db->update('gserver', [
'site_name' => 'test', "`registered-users` = `registered-users` - 1",
'info' => 'another test'
], [
'nurl' => 'http://friendica.local'
]));
self::assertEquals(1, $db->selectFirst('gserver', ['registered-users'], ['nurl' => 'http://friendica.local'])['registered-users']);
}
}