From c0b5339691d63fcbae6caa7a032923d2826f9df0 Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 28 Apr 2017 16:35:53 +0000 Subject: [PATCH 1/6] There is now a function to store conversations --- include/items.php | 103 ++++++++++++++++++++++++-------------------- include/ostatus.php | 2 + 2 files changed, 59 insertions(+), 46 deletions(-) diff --git a/include/items.php b/include/items.php index 285a5745a..6456fa2cb 100644 --- a/include/items.php +++ b/include/items.php @@ -410,7 +410,60 @@ function uri_to_guid($uri, $host = "") { return $guid_prefix.$host_hash; } -/// @TODO Maybe $arr must be called-by-reference? This function modifies it +function store_conversation($arr) { + if (in_array($arr['network'], array(NETWORK_DFRN, NETWORK_DIASPORA, NETWORK_OSTATUS))) { + $conversation = array('item-uri' => $arr['uri'], 'received' => dbm::date()); + + if (isset($arr['parent-uri']) AND ($arr['parent-uri'] != $arr['uri'])) { + $conversation['reply-to-uri'] = $arr['parent-uri']; + } + if (isset($arr['thr-parent']) AND ($arr['thr-parent'] != $arr['uri'])) { + $conversation['reply-to-uri'] = $arr['thr-parent']; + } + + if (isset($arr['conversation-uri'])) { + $conversation['conversation-uri'] = $arr['conversation-uri']; + } + + if (isset($arr['conversation-href'])) { + $conversation['conversation-href'] = $arr['conversation-href']; + } + + if (isset($arr['protocol'])) { + $conversation['protocol'] = $arr['protocol']; + } + + if (isset($arr['source'])) { + $conversation['source'] = $arr['source']; + } + + $conv = dba::fetch_first("SELECT `protocol` FROM `conversation` WHERE `item-uri` = ?", $conversation['item-uri']); + if (dbm::is_result($conv)) { + if (($conv['protocol'] < $conversation['protocol']) AND ($conv['protocol'] != 0)) { + unset($conversation['protocol']); + unset($conversation['source']); + } + // Replace the conversation entry when the new one is better + //if ((($conv['protocol'] == 0) OR ($conv['protocol'] >= $conversation['protocol'])) AND ($conversation['protocol'] > 0)) { + if (!dba::update('conversation', $conversation, array('item-uri' => $conversation['item-uri']))) { + logger('Conversation: update for '.$conversation['item-uri'].' from '.$conv['protocol'].' to '.$conversation['protocol'].' failed', LOGGER_DEBUG); + } + //} + } else { + if (!dba::insert('conversation', $conversation)) { + logger('Conversation: insert for '.$conversation['item-uri'].' (protocol '.$conversation['protocol'].') failed', LOGGER_DEBUG); + } + } + } + + unset($arr['conversation-uri']); + unset($arr['conversation-href']); + unset($arr['protocol']); + unset($arr['source']); + + return $arr; +} + /// @TODO add type-hint array function item_store($arr, $force_parent = false, $notify = false, $dontcache = false) { @@ -436,6 +489,9 @@ function item_store($arr, $force_parent = false, $notify = false, $dontcache = f } } + // Store conversation data + $arr = store_conversation($arr); + /* * If a Diaspora signature structure was passed in, pull it out of the * item array and set it aside for later storage. @@ -691,51 +747,6 @@ function item_store($arr, $force_parent = false, $notify = false, $dontcache = f $arr['thr-parent'] = $arr['parent-uri']; - if (in_array($arr['network'], array(NETWORK_DFRN, NETWORK_DIASPORA, NETWORK_OSTATUS))) { - $conversation = array('item-uri' => $arr['uri'], 'received' => dbm::date()); - - if (isset($arr['thr-parent'])) { - if ($arr['thr-parent'] != $arr['uri']) { - $conversation['reply-to-uri'] = $arr['thr-parent']; - } - } - - if (isset($arr['conversation-uri'])) { - $conversation['conversation-uri'] = $arr['conversation-uri']; - } - - if (isset($arr['conversation-href'])) { - $conversation['conversation-href'] = $arr['conversation-href']; - } - - if (isset($arr['protocol'])) { - $conversation['protocol'] = $arr['protocol']; - } - - if (isset($arr['source'])) { - $conversation['source'] = $arr['source']; - } - - $conv = dba::fetch_first("SELECT `protocol` FROM `conversation` WHERE `item-uri` = ?", $conversation['item-uri']); - if (dbm::is_result($conv)) { - // Replace the conversation entry when the new one is better - if ((($conv['protocol'] == 0) OR ($conv['protocol'] > $conversation['protocol'])) AND ($conversation['protocol'] > 0)) { - if (!dba::update('conversation', $conversation, array('item-uri' => $conversation['item-uri']))) { - logger('Conversation: update for '.$conversation['item-uri'].' from '.$conv['protocol'].' to '.$conversation['protocol'].' failed', LOGGER_DEBUG); - } - } - } else { - if (!dba::insert('conversation', $conversation)) { - logger('Conversation: insert for '.$conversation['item-uri'].' (protocol '.$conversation['protocol'].') failed', LOGGER_DEBUG); - } - } - } - - unset($arr['conversation-uri']); - unset($arr['conversation-href']); - unset($arr['protocol']); - unset($arr['source']); - if ($arr['parent-uri'] === $arr['uri']) { $parent_id = 0; $parent_deleted = 0; diff --git a/include/ostatus.php b/include/ostatus.php index 1800a9cc8..40bcdcdcc 100644 --- a/include/ostatus.php +++ b/include/ostatus.php @@ -916,6 +916,8 @@ class ostatus { ($item["verb"] == ACTIVITY_LIKE) OR ($conversation_url == "")) { $item_stored = item_store($item, $all_threads); return $item_stored; + } elseif (count($item) > 0) { + $item = store_conversation($item); } // Get the parent From edbbdf97618a2588479dca642c03bf927e947abc Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 29 Apr 2017 05:44:13 +0000 Subject: [PATCH 2/6] Store conversation data for self created posts as well --- include/items.php | 7 +++++++ mod/item.php | 14 ++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/include/items.php b/include/items.php index 6456fa2cb..9f948485a 100644 --- a/include/items.php +++ b/include/items.php @@ -410,6 +410,12 @@ function uri_to_guid($uri, $host = "") { return $guid_prefix.$host_hash; } +/** + * @brief Store the conversation data + * + * @param array $arr Item array with conversation data + * @return array Item array with removed conversation data + */ function store_conversation($arr) { if (in_array($arr['network'], array(NETWORK_DFRN, NETWORK_DIASPORA, NETWORK_OSTATUS))) { $conversation = array('item-uri' => $arr['uri'], 'received' => dbm::date()); @@ -476,6 +482,7 @@ function item_store($arr, $force_parent = false, $notify = false, $dontcache = f $arr['origin'] = 1; $arr['last-child'] = 1; $arr['network'] = NETWORK_DFRN; + $arr['protocol'] = PROTOCOL_DFRN; // We have to avoid duplicates. So we create the GUID in form of a hash of the plink or uri. // In difference to the call to "uri_to_guid" several lines below we add the hash of our own host. diff --git a/mod/item.php b/mod/item.php index 8bf922d97..d8be579ff 100644 --- a/mod/item.php +++ b/mod/item.php @@ -723,6 +723,18 @@ function item_post(App $a) { $datarray['last-child'] = 1; $datarray['visible'] = 1; + $datarray['protocol'] = PROTOCOL_DFRN; + + $r = dba::fetch_first("SELECT `conversation-uri`, `conversation-href` FROM `conversation` WHERE `item-uri` = ?", $datarray['parent-uri']); + if (dbm::is_result($r)) { + if ($r['conversation-uri'] != '') { + $datarray['conversation-uri'] = $r['conversation-uri']; + } + if ($r['conversation-href'] != '') { + $datarray['conversation-href'] = $r['conversation-href']; + } + } + if ($orig_post) { $datarray['edit'] = true; } @@ -762,6 +774,8 @@ function item_post(App $a) { // Fill the cache field put_item_in_cache($datarray); + $datarray = store_conversation($datarray); + if ($orig_post) { $r = q("UPDATE `item` SET `title` = '%s', `body` = '%s', `tag` = '%s', `attach` = '%s', `file` = '%s', `rendered-html` = '%s', `rendered-hash` = '%s', `edited` = '%s', `changed` = '%s' WHERE `id` = %d AND `uid` = %d", dbesc($datarray['title']), From 0e4cd900d5306c0107530f75554c23226edc16a6 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 29 Apr 2017 07:50:09 +0000 Subject: [PATCH 3/6] Improved dba:update function. --- include/dba.php | 66 +++++++++++++++++++++++++++++++++++++++++++++-- include/items.php | 16 +++++------- 2 files changed, 71 insertions(+), 11 deletions(-) diff --git a/include/dba.php b/include/dba.php index 3f4d81482..f695902af 100644 --- a/include/dba.php +++ b/include/dba.php @@ -756,15 +756,77 @@ class dba { /** * @brief Updates rows * + * Updates rows in the database. When $old_fields is set to an array, + * the system will only do an update if the fields in that array changed. + * + * Attention: + * Only the values in $old_fields are compared. + * This is an intentional behaviour. + * + * Example: + * We include the timestamp field in $fields but not in $old_fields. + * Then the row will only get the new timestamp when the other fields had changed. + * + * When $old_fields is set to a boolean value the system will do this compare itself. + * When $old_fields is set to "true" the system will do an insert if the row doesn't exists. + * + * Attention: + * Only set $old_fields to a boolean value when you are sure that you will update a single row. + * When you set $old_fields to "true" then $fields must contain all relevant fields! + * * @param string $table Table name * @param array $fields contains the fields that are updated * @param array $condition condition array with the key values + * @param array|boolean $old_fields array with the old field values that are about to be replaced * * @return boolean was the update successfull? */ - static public function update($table, $fields, $condition) { + static public function update($table, $fields, $condition, $old_fields = array()) { - $sql = "UPDATE `".self::$dbo->escape($table)."` SET `". + /** @todo We may use MySQL specific functions here: + * INSERT INTO `config` (`cat`, `k`, `v`) VALUES ('%s', '%s', '%s') ON DUPLICATE KEY UPDATE `v` = '%s'" + * But I think that it doesn't make sense here. + */ + + $table = self::$dbo->escape($table); + + if (is_bool($old_fields)) { + $sql = "SELECT * FROM `".$table."` WHERE `". + implode("` = ? AND `", array_keys($condition))."` = ? LIMIT 1"; + + $params = array(); + foreach ($condition AS $value) { + $params[] = $value; + } + + $do_insert = $old_fields; + + $old_fields = self::fetch_first($sql, $params); + if (is_bool($old_fields)) { + if ($do_insert) { + return self::insert($table, $fields); + } + $old_fields = array(); + } + } + + $do_update = (count($old_fields) == 0); + + foreach ($old_fields AS $fieldname => $content) { + if (isset($fields[$fieldname])) { + if ($fields[$fieldname] == $content) { + unset($fields[$fieldname]); + } else { + $do_update = true; + } + } + } + + if (!$do_update OR (count($fields) == 0)) { + return true; + } + + $sql = "UPDATE `".$table."` SET `". implode("` = ?, `", array_keys($fields))."` = ? WHERE `". implode("` = ? AND `", array_keys($condition))."` = ?"; diff --git a/include/items.php b/include/items.php index 9f948485a..4d2a8ae52 100644 --- a/include/items.php +++ b/include/items.php @@ -443,18 +443,16 @@ function store_conversation($arr) { $conversation['source'] = $arr['source']; } - $conv = dba::fetch_first("SELECT `protocol` FROM `conversation` WHERE `item-uri` = ?", $conversation['item-uri']); - if (dbm::is_result($conv)) { - if (($conv['protocol'] < $conversation['protocol']) AND ($conv['protocol'] != 0)) { + $old_conv = dba::fetch_first("SELECT `item-uri`, `reply-to-uri`, `conversation-uri`, `conversation-href`, `protocol`, `source` + FROM `conversation` WHERE `item-uri` = ?", $conversation['item-uri']); + if (dbm::is_result($old_conv)) { + if (($old_conv['protocol'] < $conversation['protocol']) AND ($old_conv['protocol'] != 0)) { unset($conversation['protocol']); unset($conversation['source']); } - // Replace the conversation entry when the new one is better - //if ((($conv['protocol'] == 0) OR ($conv['protocol'] >= $conversation['protocol'])) AND ($conversation['protocol'] > 0)) { - if (!dba::update('conversation', $conversation, array('item-uri' => $conversation['item-uri']))) { - logger('Conversation: update for '.$conversation['item-uri'].' from '.$conv['protocol'].' to '.$conversation['protocol'].' failed', LOGGER_DEBUG); - } - //} + if (!dba::update('conversation', $conversation, array('item-uri' => $conversation['item-uri']), $old_conv)) { + logger('Conversation: update for '.$conversation['item-uri'].' from '.$conv['protocol'].' to '.$conversation['protocol'].' failed', LOGGER_DEBUG); + } } else { if (!dba::insert('conversation', $conversation)) { logger('Conversation: insert for '.$conversation['item-uri'].' (protocol '.$conversation['protocol'].') failed', LOGGER_DEBUG); From fa14096ea8dc82ee050168b5465552b52786c929 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 29 Apr 2017 08:11:54 +0000 Subject: [PATCH 4/6] Added documentation --- doc/database.md | 1 + doc/database/db_conversation.md | 14 ++++++++++++++ 2 files changed, 15 insertions(+) create mode 100644 doc/database/db_conversation.md diff --git a/doc/database.md b/doc/database.md index 4c6150520..27ff886ab 100644 --- a/doc/database.md +++ b/doc/database.md @@ -14,6 +14,7 @@ Database Tables | [config](help/database/db_config) | main configuration storage | | [contact](help/database/db_contact) | contact table | | [conv](help/database/db_conv) | private messages | +| [conversation](help/database/db_conversation) | Raw data and structure information for messages | | [event](help/database/db_event) | Events | | [fcontact](help/database/db_fcontact) | friend suggestion stuff | | [ffinder](help/database/db_ffinder) | friend suggestion stuff | diff --git a/doc/database/db_conversation.md b/doc/database/db_conversation.md new file mode 100644 index 000000000..8b923ec04 --- /dev/null +++ b/doc/database/db_conversation.md @@ -0,0 +1,14 @@ +Table conversation +================== + +| Field | Description | Type | Null | Key | Default | Extra | +|-------------------| ------------- |---------------------|------|-----|---------------------|----------------| +| item-uri | | varbinary(255) | NO | PRI | NULL | | +| reply-to-uri | | varbinary(255) | NO | | | | +| conversation-uri | | varbinary(255) | NO | | | | +| conversation-href | | varbinary(255) | NO | | | | +| protocol | | tinyint(1) unsigned | NO | | 0 | | +| source | | mediumtext | NO | | | | +| received | | datetime | NO | | 0001-01-01 | | + +Return to [database documentation](help/database) From 10ea7592827e0292797b2e7dd2e0676b6665ac5f Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 29 Apr 2017 08:30:43 +0000 Subject: [PATCH 5/6] Don't update the conversation only because the source changed --- include/items.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/include/items.php b/include/items.php index 4d2a8ae52..5eac12d89 100644 --- a/include/items.php +++ b/include/items.php @@ -446,6 +446,12 @@ function store_conversation($arr) { $old_conv = dba::fetch_first("SELECT `item-uri`, `reply-to-uri`, `conversation-uri`, `conversation-href`, `protocol`, `source` FROM `conversation` WHERE `item-uri` = ?", $conversation['item-uri']); if (dbm::is_result($old_conv)) { + // Don't update when only the source has changed. + // Only do this when there had been no source before. + if ($old_conv['source'] != '') { + unset($old_conv['source']); + } + // Update structure data all the time but the source only when its from a better protocol. if (($old_conv['protocol'] < $conversation['protocol']) AND ($old_conv['protocol'] != 0)) { unset($conversation['protocol']); unset($conversation['source']); From 55c39a80fadeef2cbd53509a150b6002292bb8b3 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 29 Apr 2017 12:19:49 +0000 Subject: [PATCH 6/6] Description added --- doc/database/db_conversation.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/database/db_conversation.md b/doc/database/db_conversation.md index 8b923ec04..32d030cb1 100644 --- a/doc/database/db_conversation.md +++ b/doc/database/db_conversation.md @@ -1,14 +1,14 @@ Table conversation ================== -| Field | Description | Type | Null | Key | Default | Extra | -|-------------------| ------------- |---------------------|------|-----|---------------------|----------------| -| item-uri | | varbinary(255) | NO | PRI | NULL | | -| reply-to-uri | | varbinary(255) | NO | | | | -| conversation-uri | | varbinary(255) | NO | | | | -| conversation-href | | varbinary(255) | NO | | | | -| protocol | | tinyint(1) unsigned | NO | | 0 | | -| source | | mediumtext | NO | | | | -| received | | datetime | NO | | 0001-01-01 | | +| Field | Description | Type | Null | Key | Default | Extra | +|-------------------| ---------------------------------- |---------------------|------|-----|---------------------|----------------| +| item-uri | URI of the item | varbinary(255) | NO | PRI | NULL | | +| reply-to-uri | URI to which this item is a reply | varbinary(255) | NO | | | | +| conversation-uri | GNU Social conversation URI | varbinary(255) | NO | | | | +| conversation-href | GNU Social conversation link | varbinary(255) | NO | | | | +| protocol | The protocol of the item | tinyint(1) unsigned | NO | | 0 | | +| source | Original source | mediumtext | NO | | | | +| received | Receiving date | datetime | NO | | 0001-01-01 | | Return to [database documentation](help/database)