From df880f7fd2b6f3cef62e1b7968e420554cdea218 Mon Sep 17 00:00:00 2001 From: Michael Vogel <icarus@dabo.de> Date: Sat, 7 Mar 2015 21:07:26 +0100 Subject: [PATCH 01/12] Adding performance measuring to the photo functions. --- include/Photo.php | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/include/Photo.php b/include/Photo.php index 2f7c99025..ccb6af29e 100644 --- a/include/Photo.php +++ b/include/Photo.php @@ -516,7 +516,12 @@ class Photo { return FALSE; $string = $this->imageString(); + + $a = get_app(); + + $stamp1 = microtime(true); file_put_contents($path, $string); + $a->save_timestamp($stamp1, "file"); } public function imageString() { @@ -767,7 +772,12 @@ function get_photo_info($url) { $filesize = strlen($img_str); $tempfile = tempnam(get_temppath(), "cache"); + + $a = get_app(); + $stamp1 = microtime(true); file_put_contents($tempfile, $img_str); + $a->save_timestamp($stamp1, "file"); + $data = getimagesize($tempfile); unlink($tempfile); @@ -851,7 +861,10 @@ function store_photo($a, $uid, $imagedata = "", $url = "") { return(array()); } elseif (strlen($imagedata) == 0) { logger("Uploading picture from ".$url, LOGGER_DEBUG); + + $stamp1 = microtime(true); $imagedata = @file_get_contents($url); + $a->save_timestamp($stamp1, "file"); } $maximagesize = get_config('system','maximagesize'); @@ -875,7 +888,11 @@ function store_photo($a, $uid, $imagedata = "", $url = "") { */ $tempfile = tempnam(get_temppath(), "cache"); + + $stamp1 = microtime(true); file_put_contents($tempfile, $imagedata); + $a->save_timestamp($stamp1, "file"); + $data = getimagesize($tempfile); if (!isset($data["mime"])) { From 30548dfeed6c6c790b400e7e4b32f754a42f6c7b Mon Sep 17 00:00:00 2001 From: Michael Vogel <icarus@dabo.de> Date: Sat, 7 Mar 2015 21:08:22 +0100 Subject: [PATCH 02/12] Improving performance measuring for the bbcode parser. --- include/bbcode.php | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/include/bbcode.php b/include/bbcode.php index fa4fa72c7..7e9dcf3ba 100644 --- a/include/bbcode.php +++ b/include/bbcode.php @@ -852,9 +852,13 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true, $simplehtml = fal } while ($oldtext != $Text); } + $a->save_timestamp($stamp1, "parser"); + // Handle attached links or videos $Text = bb_attachment($Text, ($simplehtml != 4) AND ($simplehtml != 0), $tryoembed); + $stamp1 = microtime(true); + $Text = str_replace(array("\r","\n"), array('<br />','<br />'), $Text); if($preserve_nl) @@ -884,10 +888,14 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true, $simplehtml = fal if ($simplehtml == 5) $Text = preg_replace("/[^#@]\[url\=([$URLSearchString]*)\](.*?)\[\/url\]/ism", '[url]$1[/url]', $Text); + $a->save_timestamp($stamp1, "parser"); + // Perform URL Search if ($tryoembed) $Text = preg_replace_callback("/\[bookmark\=([^\]]*)\](.*?)\[\/bookmark\]/ism",'tryoembed',$Text); + $stamp1 = microtime(true); + if ($simplehtml == 5) $Text = preg_replace("/\[bookmark\=([^\]]*)\](.*?)\[\/bookmark\]/ism",'[url]$1[/url]',$Text); else @@ -896,6 +904,8 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true, $simplehtml = fal // Handle Diaspora posts $Text = preg_replace_callback("&\[url=/posts/([^\[\]]*)\](.*)\[\/url\]&Usi", 'bb_DiasporaLinks', $Text); + $a->save_timestamp($stamp1, "parser"); + // if the HTML is used to generate plain text, then don't do this search, but replace all URL of that kind to text if (!$forplaintext) $Text = preg_replace("/([^\]\='".'"'."]|^)(https?\:\/\/[a-zA-Z0-9\:\/\-\?\&\;\.\=\_\~\#\%\$\!\+\,]+)/ism", '$1<a href="$2" target="_blank">$2</a>', $Text); @@ -907,6 +917,8 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true, $simplehtml = fal if ($tryoembed) $Text = preg_replace_callback("/\[url\]([$URLSearchString]*)\[\/url\]/ism",'tryoembed',$Text); + $stamp1 = microtime(true); + $Text = preg_replace("/\[url\]([$URLSearchString]*)\[\/url\]/ism", '<a href="$1" target="_blank">$1</a>', $Text); $Text = preg_replace("/\[url\=([$URLSearchString]*)\](.*?)\[\/url\]/ism", '<a href="$1" target="_blank">$2</a>', $Text); //$Text = preg_replace("/\[url\=([$URLSearchString]*)\]([$URLSearchString]*)\[\/url\]/ism", '<a href="$1" target="_blank">$2</a>', $Text); @@ -1078,8 +1090,13 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true, $simplehtml = fal $Text = preg_replace("/\[video\](.*?\.(ogg|ogv|oga|ogm|webm|mp4))\[\/video\]/ism", '<video src="$1" controls="controls" width="' . $a->videowidth . '" height="' . $a->videoheight . '"><a href="$1">$1</a></video>', $Text); $Text = preg_replace("/\[audio\](.*?\.(ogg|ogv|oga|ogm|webm|mp4|mp3))\[\/audio\]/ism", '<audio src="$1" controls="controls"><a href="$1">$1</a></audio>', $Text); + $a->save_timestamp($stamp1, "parser"); + $Text = preg_replace_callback("/\[video\](.*?)\[\/video\]/ism", 'tryoembed', $Text); $Text = preg_replace_callback("/\[audio\](.*?)\[\/audio\]/ism", 'tryoembed', $Text); + + $stamp1 = microtime(true); + } else { $Text = preg_replace("/\[video\](.*?)\[\/video\]/", '<a href="$1" target="_blank">$1</a>', $Text); @@ -1097,9 +1114,13 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true, $simplehtml = fal // Youtube extensions if ($tryoembed) { + $a->save_timestamp($stamp1, "parser"); + $Text = preg_replace_callback("/\[youtube\](https?:\/\/www.youtube.com\/watch\?v\=.*?)\[\/youtube\]/ism", 'tryoembed', $Text); $Text = preg_replace_callback("/\[youtube\](www.youtube.com\/watch\?v\=.*?)\[\/youtube\]/ism", 'tryoembed', $Text); $Text = preg_replace_callback("/\[youtube\](https?:\/\/youtu.be\/.*?)\[\/youtube\]/ism",'tryoembed',$Text); + + $stamp1 = microtime(true); } $Text = preg_replace("/\[youtube\]https?:\/\/www.youtube.com\/watch\?v\=(.*?)\[\/youtube\]/ism",'[youtube]$1[/youtube]',$Text); @@ -1113,8 +1134,12 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true, $simplehtml = fal '<a href="https://www.youtube.com/watch?v=$1" target="_blank">https://www.youtube.com/watch?v=$1</a>', $Text); if ($tryoembed) { - $Text = preg_replace_callback("/\[vimeo\](https?:\/\/player.vimeo.com\/video\/[0-9]+).*?\[\/vimeo\]/ism",'tryoembed',$Text); - $Text = preg_replace_callback("/\[vimeo\](https?:\/\/vimeo.com\/[0-9]+).*?\[\/vimeo\]/ism",'tryoembed',$Text); + $a->save_timestamp($stamp1, "parser"); + + $Text = preg_replace_callback("/\[vimeo\](https?:\/\/player.vimeo.com\/video\/[0-9]+).*?\[\/vimeo\]/ism",'tryoembed',$Text); + $Text = preg_replace_callback("/\[vimeo\](https?:\/\/vimeo.com\/[0-9]+).*?\[\/vimeo\]/ism",'tryoembed',$Text); + + $stamp1 = microtime(true); } $Text = preg_replace("/\[vimeo\]https?:\/\/player.vimeo.com\/video\/([0-9]+)(.*?)\[\/vimeo\]/ism",'[vimeo]$1[/vimeo]',$Text); @@ -1128,10 +1153,13 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true, $simplehtml = fal // $Text = preg_replace("/\[youtube\](.*?)\[\/youtube\]/", '<object width="425" height="350" type="application/x-shockwave-flash" data="http://www.youtube.com/v/$1" ><param name="movie" value="http://www.youtube.com/v/$1"></param><!--[if IE]><embed src="http://www.youtube.com/v/$1" type="application/x-shockwave-flash" width="425" height="350" /><![endif]--></object>', $Text); + $a->save_timestamp($stamp1, "parser"); // oembed tag $Text = oembed_bbcode2html($Text); + $stamp1 = microtime(true); + // Avoid triple linefeeds through oembed $Text = str_replace("<br style='clear:left'></span><br /><br />", "<br style='clear:left'></span><br />", $Text); From dc380d780019f8c26f23ad5e4bb1df2601974b2a Mon Sep 17 00:00:00 2001 From: Michael Vogel <icarus@dabo.de> Date: Sat, 7 Mar 2015 21:09:32 +0100 Subject: [PATCH 03/12] Adding some more data to Diaspora reshares --- include/diaspora.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/diaspora.php b/include/diaspora.php index 2c19cd4c4..1202a3a55 100755 --- a/include/diaspora.php +++ b/include/diaspora.php @@ -1167,6 +1167,8 @@ function diaspora_reshare($importer,$xml,$msg) { $prefix = "[share author='".str_replace(array("'", "[", "]"), array("'", "[", "]"),$person['name']). "' profile='".$person['url']. "' avatar='".((x($person,'thumb')) ? $person['thumb'] : $person['photo']). + "' guid='".$orig_guid. + "' posted='".$orig_created. "' link='".str_replace(array("'", "[", "]"), array("'", "[", "]"),$orig_url)."']"; $datarray['author-name'] = $contact['name']; $datarray['author-link'] = $contact['url']; From e5fc28bb314f5498248a763f2372592b7f99d3cc Mon Sep 17 00:00:00 2001 From: Michael Vogel <icarus@dabo.de> Date: Sat, 7 Mar 2015 21:10:02 +0100 Subject: [PATCH 04/12] Removed some unnecessary code --- include/html2bbcode.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/include/html2bbcode.php b/include/html2bbcode.php index 1021bcdf1..d2699460e 100644 --- a/include/html2bbcode.php +++ b/include/html2bbcode.php @@ -88,9 +88,6 @@ function deletenode(&$doc, $node) function html2bbcode($message) { - //$file = tempnam("/tmp/", "html"); - //file_put_contents($file, $message); - $message = str_replace("\r", "", $message); $message = str_replace(array( From 67d39770ed69b385a927a269e4d406d0d252020e Mon Sep 17 00:00:00 2001 From: Michael Vogel <icarus@dabo.de> Date: Sat, 7 Mar 2015 21:12:18 +0100 Subject: [PATCH 05/12] Improved the margins of the headers --- view/global.css | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/view/global.css b/view/global.css index 604a1e449..c61ded6ae 100644 --- a/view/global.css +++ b/view/global.css @@ -129,7 +129,7 @@ blockquote.shared_content { /* headers */ h1, h2, h3, h4, h5, h6 { - margin: 0px 0px 5px 0px; + margin: 5px 0px 5px 0px; font-weight: normal; line-height: normal; text-rendering: optimizelegibility; @@ -158,3 +158,7 @@ h5 { h6 { font-size: 14.95px; } + +span.oembed, h4 { + margin: 0px 0px 0px 0px; +} From 7c4a1a059dd7ecb5fa4fb0e7a9dc7be3f2470f2e Mon Sep 17 00:00:00 2001 From: Michael Vogel <icarus@dabo.de> Date: Sat, 7 Mar 2015 21:24:39 +0100 Subject: [PATCH 06/12] New fields for the term table, improved query for the tag search. Changed the cache handling for rendered bbcode. --- boot.php | 2 +- include/dbstructure.php | 7 +- include/items.php | 6 +- include/network.php | 7 - include/ostatus_conversation.php | 7 +- include/tags.php | 23 +++- include/tagupdate.php | 22 +++ include/text.php | 50 +++---- mod/item.php | 228 ++++++++++++++++--------------- mod/search.php | 89 +++++------- update.php | 10 +- 11 files changed, 233 insertions(+), 218 deletions(-) create mode 100644 include/tagupdate.php diff --git a/boot.php b/boot.php index fbf4a90d2..ec9b70eb7 100644 --- a/boot.php +++ b/boot.php @@ -18,7 +18,7 @@ define ( 'FRIENDICA_PLATFORM', 'Friendica'); define ( 'FRIENDICA_CODENAME', 'Ginger'); define ( 'FRIENDICA_VERSION', '3.3.3' ); define ( 'DFRN_PROTOCOL_VERSION', '2.23' ); -define ( 'DB_UPDATE_VERSION', 1180 ); +define ( 'DB_UPDATE_VERSION', 1181 ); define ( 'EOL', "<br />\r\n" ); define ( 'ATOM_TIME', 'Y-m-d\TH:i:s\Z' ); diff --git a/include/dbstructure.php b/include/dbstructure.php index 0ee28e0a6..ff24be5de 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -776,6 +776,8 @@ function db_definition() { "last-child" => array("type" => "tinyint(1) unsigned", "not null" => "1", "default" => "1"), "mention" => array("type" => "tinyint(1)", "not null" => "1", "default" => "0"), "network" => array("type" => "varchar(32)", "not null" => "1", "default" => ""), + "rendered-hash" => array("type" => "varchar(32)", "not null" => "1", "default" => ""), + "rendered-html" => array("type" => "mediumtext", "not null" => "1"), ), "indexes" => array( "PRIMARY" => array("id"), @@ -1188,6 +1190,9 @@ function db_definition() { "type" => array("type" => "tinyint(3) unsigned", "not null" => "1", "default" => "0"), "term" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "url" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), + "guid" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), + "created" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), + "received" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), "aid" => array("type" => "int(10) unsigned", "not null" => "1", "default" => "0"), "uid" => array("type" => "int(10) unsigned", "not null" => "1", "default" => "0"), ), @@ -1196,7 +1201,7 @@ function db_definition() { "oid_otype_type_term" => array("oid","otype","type","term"), "uid_term_tid" => array("uid","term","tid"), "type_term" => array("type","term"), - "uid_otype_type_term_tid" => array("uid","otype","type","term","tid"), + "uid_otype_type_term_created" => array("uid","otype","type","term","created"), "otype_type_term_tid" => array("otype","type","term","tid"), ) ); diff --git a/include/items.php b/include/items.php index b698a0cd3..8f9eb5aa6 100644 --- a/include/items.php +++ b/include/items.php @@ -1346,6 +1346,9 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa return 0; } + // Fill the cache field + put_item_in_cache($arr); + call_hooks('post_remote',$arr); if(x($arr,'cancel')) { @@ -1478,9 +1481,6 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa // in it. if (!$deleted AND !$dontcache) { - // Store the fresh generated item into the cache - put_item_in_cache($arr); - $r = q('SELECT * FROM `item` WHERE id = %d', intval($current_post)); if (count($r) == 1) { call_hooks('post_remote_end', $r[0]); diff --git a/include/network.php b/include/network.php index 911d9784a..ce21f3fbb 100644 --- a/include/network.php +++ b/include/network.php @@ -870,13 +870,6 @@ function scale_external_images($srctext, $include_link = true, $scale_replace = if(! $i) return $srctext; - $cachefile = get_cachefile(hash("md5", $scaled)); - if ($cachefile != '') { - $stamp1 = microtime(true); - file_put_contents($cachefile, $i); - $a->save_timestamp($stamp1, "file"); - } - // guess mimetype from headers or filename $type = guess_image_type($mtch[1],true); diff --git a/include/ostatus_conversation.php b/include/ostatus_conversation.php index 403f95f4b..667f7dde4 100644 --- a/include/ostatus_conversation.php +++ b/include/ostatus_conversation.php @@ -52,7 +52,7 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio $a->last_ostatus_conversation_url = $conversation_url; - $messages = q("SELECT `uid`, `parent`, `created` FROM `item` WHERE `id` = %d LIMIT 1", intval($itemid)); + $messages = q("SELECT `uid`, `parent`, `created`, `received`, `guid` FROM `item` WHERE `id` = %d LIMIT 1", intval($itemid)); if (!$messages) return; $message = $messages[0]; @@ -62,8 +62,9 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio intval($message["uid"]), intval($itemid), intval(TERM_OBJ_POST), intval(TERM_CONVERSATION)); if (!$conversation) { - $r = q("INSERT INTO `term` (`uid`, `oid`, `otype`, `type`, `term`, `url`) VALUES (%d, %d, %d, %d, '%s', '%s')", - intval($message["uid"]), intval($itemid), intval(TERM_OBJ_POST), intval(TERM_CONVERSATION), dbesc($message["created"]), dbesc($conversation_url)); + $r = q("INSERT INTO `term` (`uid`, `oid`, `otype`, `type`, `term`, `url`, `created`, `received`, `guid`) VALUES (%d, %d, %d, %d, '%s', '%s', '%s', '%s', '%s')", + intval($message["uid"]), intval($itemid), intval(TERM_OBJ_POST), intval(TERM_CONVERSATION), + dbesc($message["created"]), dbesc($conversation_url), dbesc($message["created"]), dbesc($message["received"]), dbesc($message["guid"])); logger('complete_conversation: Storing conversation url '.$conversation_url.' for id '.$itemid); } diff --git a/include/tags.php b/include/tags.php index 489ca47d2..72cac1d63 100644 --- a/include/tags.php +++ b/include/tags.php @@ -9,7 +9,7 @@ function create_tags_from_item($itemid) { $searchpath = $a->get_baseurl()."/search?tag="; - $messages = q("SELECT `guid`, `uid`, `id`, `edited`, `deleted`, `title`, `body`, `tag`, `parent` FROM `item` WHERE `id` = %d LIMIT 1", intval($itemid)); + $messages = q("SELECT `guid`, `uid`, `id`, `edited`, `deleted`, `created`, `received`, `title`, `body`, `tag`, `parent` FROM `item` WHERE `id` = %d LIMIT 1", intval($itemid)); if (!$messages) return; @@ -69,8 +69,10 @@ function create_tags_from_item($itemid) { $term = $tag; } - $r = q("INSERT INTO `term` (`uid`, `oid`, `otype`, `type`, `term`, `url`) VALUES (%d, %d, %d, %d, '%s', '%s')", - intval($message["uid"]), intval($itemid), intval(TERM_OBJ_POST), intval($type), dbesc($term), dbesc($link)); + $r = q("INSERT INTO `term` (`uid`, `oid`, `otype`, `type`, `term`, `url`, `guid`, `created`, `received`) + VALUES (%d, %d, %d, %d, '%s', '%s', '%s', '%s', '%s')", + intval($message["uid"]), intval($itemid), intval(TERM_OBJ_POST), intval($type), + dbesc($term), dbesc($link), dbesc($message["guid"]), dbesc($message["created"]), dbesc($message["received"])); // Search for mentions if ((substr($tag, 0, 1) == '@') AND (strpos($link, $profile_base_friendica) OR strpos($link, $profile_base_diaspora))) { @@ -96,10 +98,17 @@ function create_tags_from_itemuri($itemuri, $uid) { } function update_items() { - //$messages = q("SELECT `id` FROM `item` where tag !='' ORDER BY `created` DESC limit 10"); - $messages = q("SELECT `id` FROM `item` where tag !=''"); + global $db; - foreach ($messages as $message) - create_tags_from_item($message["id"]); + $messages = $db->q("SELECT `oid`,`item`.`guid`, `item`.`created`, `item`.`received` FROM `term` INNER JOIN `item` ON `item`.`id`=`term`.`oid` WHERE `term`.`otype` = 1 AND `term`.`guid` = ''", true); + + logger("fetched messages: ".count($messages)); + while ($message = $db->qfetch()) { + q("UPDATE `term` SET `guid` = '%s', `created` = '%s', `received` = '%s' WHERE `otype` = %d AND `oid` = %d", + dbesc($message["guid"]), dbesc($message["created"]), dbesc($message["received"]), + intval(TERM_OBJ_POST), intval($message["oid"])); + } + + $db->qclose(); } ?> diff --git a/include/tagupdate.php b/include/tagupdate.php new file mode 100644 index 000000000..b12e80977 --- /dev/null +++ b/include/tagupdate.php @@ -0,0 +1,22 @@ +<?php +require_once("boot.php"); +require_once("include/tags.php"); + +global $a, $db; + +if(is_null($a)) + $a = new App; + +if(is_null($db)) { + @include(".htconfig.php"); + require_once("include/dba.php"); + $db = new dba($db_host, $db_user, $db_pass, $db_data); + unset($db_host, $db_user, $db_pass, $db_data); +} + +load_config('config'); +load_config('system'); + +update_items(); +killme(); +?> diff --git a/include/text.php b/include/text.php index be74756cd..4aa529d31 100644 --- a/include/text.php +++ b/include/text.php @@ -1296,16 +1296,26 @@ function redir_private_images($a, &$item) { }} -function put_item_in_cache($item) { - $cachefile = get_cachefile(urlencode($item["guid"])."-".hash("md5", $item['body'])); +function put_item_in_cache(&$item, $update = false) { + + if (($item["rendered-hash"] != hash("md5", $item["body"])) OR ($item["rendered-hash"] == "") OR + ($item["rendered-html"] == "") OR get_config("system", "ignore_cache")) { + + // The function "redir_private_images" changes the body. + // I'm not sure if we should store it permanently, so we save the old value. + $body = $item["body"]; - if (($cachefile != '') AND !file_exists($cachefile)) { - $s = prepare_text($item['body']); $a = get_app(); - $stamp1 = microtime(true); - file_put_contents($cachefile, $s); - $a->save_timestamp($stamp1, "file"); - logger('put item '.$item["guid"].' into cachefile '.$cachefile); + redir_private_images($a, $item); + + $item["rendered-html"] = prepare_text($item["body"]); + $item["rendered-hash"] = hash("md5", $item["body"]); + $item["body"] = $body; + + if ($update AND ($item["id"] != 0)) { + q("UPDATE `item` SET `rendered-html` = '%s', `rendered-hash` = '%s' WHERE `id` = %d", + dbesc($item["rendered-html"]), dbesc($item["rendered-hash"]), intval($item["id"])); + } } } @@ -1359,28 +1369,8 @@ function prepare_body(&$item,$attach = false, $preview = false) { $item['hashtags'] = $hashtags; $item['mentions'] = $mentions; - - $cachefile = get_cachefile(urlencode($item["guid"])."-".hash("md5", $item['body'])); - - if (($cachefile != '')) { - if (file_exists($cachefile)) { - $stamp1 = microtime(true); - $s = file_get_contents($cachefile); - $a->save_timestamp($stamp1, "file"); - } else { - redir_private_images($a, $item); - $s = prepare_text($item['body']); - - $stamp1 = microtime(true); - file_put_contents($cachefile, $s); - $a->save_timestamp($stamp1, "file"); - - logger('prepare_body: put item '.$item["id"].' into cachefile '.$cachefile); - } - } else { - redir_private_images($a, $item); - $s = prepare_text($item['body']); - } + put_item_in_cache($item, true); + $s = $item["rendered-html"]; require_once("mod/proxy.php"); $s = proxy_parse_html($s); diff --git a/mod/item.php b/mod/item.php index c5fc4ff2c..d9f2c3945 100644 --- a/mod/item.php +++ b/mod/item.php @@ -746,14 +746,18 @@ function item_post(&$a) { killme(); } + // Fill the cache field + put_item_in_cache($datarray); if($orig_post) { - $r = q("UPDATE `item` SET `title` = '%s', `body` = '%s', `tag` = '%s', `attach` = '%s', `file` = '%s', `edited` = '%s', `changed` = '%s' WHERE `id` = %d AND `uid` = %d", + $r = q("UPDATE `item` SET `title` = '%s', `body` = '%s', `tag` = '%s', `attach` = '%s', `file` = '%s', `rendered-html` = '%s', `rendered-hash` = '%s', `edited` = '%s', `changed` = '%s' WHERE `id` = %d AND `uid` = %d", dbesc($datarray['title']), dbesc($datarray['body']), dbesc($datarray['tag']), dbesc($datarray['attach']), dbesc($datarray['file']), + dbesc($datarray['rendered-html']), + dbesc($datarray['rendered-hash']), dbesc(datetime_convert()), dbesc(datetime_convert()), intval($post_id), @@ -778,10 +782,10 @@ function item_post(&$a) { $post_id = 0; - $r = q("INSERT INTO `item` (`guid`, `extid`, `uid`,`type`,`wall`,`gravity`, `network`, `contact-id`,`owner-name`,`owner-link`,`owner-avatar`, - `author-name`, `author-link`, `author-avatar`, `created`, `edited`, `commented`, `received`, `changed`, `uri`, `thr-parent`, `title`, `body`, `app`, `location`, `coord`, - `tag`, `inform`, `verb`, `object-type`, `postopts`, `allow_cid`, `allow_gid`, `deny_cid`, `deny_gid`, `private`, `pubmail`, `attach`, `bookmark`,`origin`, `moderated`, `file` ) - VALUES( '%s', '%s', %d, '%s', %d, %d, '%s', %d, '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d, %d, '%s', %d, %d, %d, '%s' )", + $r = q("INSERT INTO `item` (`guid`, `extid`, `uid`,`type`,`wall`,`gravity`, `network`, `contact-id`,`owner-name`,`owner-link`,`owner-avatar`, `author-name`, `author-link`, `author-avatar`, + `created`, `edited`, `commented`, `received`, `changed`, `uri`, `thr-parent`, `title`, `body`, `app`, `location`, `coord`, `tag`, `inform`, `verb`, `object-type`, `postopts`, + `allow_cid`, `allow_gid`, `deny_cid`, `deny_gid`, `private`, `pubmail`, `attach`, `bookmark`,`origin`, `moderated`, `file`, `rendered-html`, `rendered-hash`) + VALUES( '%s', '%s', %d, '%s', %d, %d, '%s', %d, '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d, %d, '%s', %d, %d, %d, '%s', '%s', '%s')", dbesc($datarray['guid']), dbesc($datarray['extid']), intval($datarray['uid']), @@ -823,121 +827,122 @@ function item_post(&$a) { intval($datarray['bookmark']), intval($datarray['origin']), intval($datarray['moderated']), - dbesc($datarray['file']) + dbesc($datarray['file']), + dbesc($datarray['rendered-html']), + dbesc($datarray['rendered-hash']) ); $r = q("SELECT `id` FROM `item` WHERE `uri` = '%s' LIMIT 1", dbesc($datarray['uri'])); - if(count($r)) { - $post_id = $r[0]['id']; - logger('mod_item: saved item ' . $post_id); - - // update filetags in pconfig - file_tag_update_pconfig($uid,$categories_old,$categories_new,'category'); - - // Store the fresh generated item into the cache - put_item_in_cache($datarray); - - if($parent) { - - // This item is the last leaf and gets the comment box, clear any ancestors - $r = q("UPDATE `item` SET `last-child` = 0, `changed` = '%s' WHERE `parent` = %d ", - dbesc(datetime_convert()), - intval($parent) - ); - update_thread($parent, true); - - // Inherit ACLs from the parent item. - - $r = q("UPDATE `item` SET `allow_cid` = '%s', `allow_gid` = '%s', `deny_cid` = '%s', `deny_gid` = '%s', `private` = %d - WHERE `id` = %d", - dbesc($parent_item['allow_cid']), - dbesc($parent_item['allow_gid']), - dbesc($parent_item['deny_cid']), - dbesc($parent_item['deny_gid']), - intval($parent_item['private']), - intval($post_id) - ); - - if($contact_record != $author) { - notification(array( - 'type' => NOTIFY_COMMENT, - 'notify_flags' => $user['notify-flags'], - 'language' => $user['language'], - 'to_name' => $user['username'], - 'to_email' => $user['email'], - 'uid' => $user['uid'], - 'item' => $datarray, - 'link' => $a->get_baseurl().'/display/'.urlencode($datarray['guid']), - 'source_name' => $datarray['author-name'], - 'source_link' => $datarray['author-link'], - 'source_photo' => $datarray['author-avatar'], - 'verb' => ACTIVITY_POST, - 'otype' => 'item', - 'parent' => $parent, - 'parent_uri' => $parent_item['uri'] - )); - - } - - - // Store the comment signature information in case we need to relay to Diaspora - store_diaspora_comment_sig($datarray, $author, ($self ? $a->user['prvkey'] : false), $parent_item, $post_id); - - } else { - $parent = $post_id; - - if($contact_record != $author) { - notification(array( - 'type' => NOTIFY_WALL, - 'notify_flags' => $user['notify-flags'], - 'language' => $user['language'], - 'to_name' => $user['username'], - 'to_email' => $user['email'], - 'uid' => $user['uid'], - 'item' => $datarray, - 'link' => $a->get_baseurl().'/display/'.urlencode($datarray['guid']), - 'source_name' => $datarray['author-name'], - 'source_link' => $datarray['author-link'], - 'source_photo' => $datarray['author-avatar'], - 'verb' => ACTIVITY_POST, - 'otype' => 'item' - )); - } - } - - // fallback so that parent always gets set to non-zero. - - if(! $parent) - $parent = $post_id; - - $r = q("UPDATE `item` SET `parent` = %d, `parent-uri` = '%s', `plink` = '%s', `changed` = '%s', `last-child` = 1, `visible` = 1 - WHERE `id` = %d", - intval($parent), - dbesc(($parent == $post_id) ? $uri : $parent_item['uri']), - dbesc($a->get_baseurl().'/display/'.urlencode($datarray['guid'])), - dbesc(datetime_convert()), - intval($post_id) - ); - - // photo comments turn the corresponding item visible to the profile wall - // This way we don't see every picture in your new photo album posted to your wall at once. - // They will show up as people comment on them. - - if(! $parent_item['visible']) { - $r = q("UPDATE `item` SET `visible` = 1 WHERE `id` = %d", - intval($parent_item['id']) - ); - update_thread($parent_item['id']); - } - } - else { + if(!count($r)) { logger('mod_item: unable to retrieve post that was just stored.'); notice( t('System error. Post not saved.') . EOL); goaway($a->get_baseurl() . "/" . $return_path ); // NOTREACHED } + $post_id = $r[0]['id']; + logger('mod_item: saved item ' . $post_id); + + $datarray["id"] = $post_id; + $datarray["plink"] = $a->get_baseurl().'/display/'.urlencode($datarray["guid"]); + + // update filetags in pconfig + file_tag_update_pconfig($uid,$categories_old,$categories_new,'category'); + + if($parent) { + + // This item is the last leaf and gets the comment box, clear any ancestors + $r = q("UPDATE `item` SET `last-child` = 0, `changed` = '%s' WHERE `parent` = %d ", + dbesc(datetime_convert()), + intval($parent) + ); + update_thread($parent, true); + + // Inherit ACLs from the parent item. + + $r = q("UPDATE `item` SET `allow_cid` = '%s', `allow_gid` = '%s', `deny_cid` = '%s', `deny_gid` = '%s', `private` = %d + WHERE `id` = %d", + dbesc($parent_item['allow_cid']), + dbesc($parent_item['allow_gid']), + dbesc($parent_item['deny_cid']), + dbesc($parent_item['deny_gid']), + intval($parent_item['private']), + intval($post_id) + ); + + if($contact_record != $author) { + notification(array( + 'type' => NOTIFY_COMMENT, + 'notify_flags' => $user['notify-flags'], + 'language' => $user['language'], + 'to_name' => $user['username'], + 'to_email' => $user['email'], + 'uid' => $user['uid'], + 'item' => $datarray, + 'link' => $a->get_baseurl().'/display/'.urlencode($datarray['guid']), + 'source_name' => $datarray['author-name'], + 'source_link' => $datarray['author-link'], + 'source_photo' => $datarray['author-avatar'], + 'verb' => ACTIVITY_POST, + 'otype' => 'item', + 'parent' => $parent, + 'parent_uri' => $parent_item['uri'] + )); + + } + + + // Store the comment signature information in case we need to relay to Diaspora + store_diaspora_comment_sig($datarray, $author, ($self ? $a->user['prvkey'] : false), $parent_item, $post_id); + + } else { + $parent = $post_id; + + if($contact_record != $author) { + notification(array( + 'type' => NOTIFY_WALL, + 'notify_flags' => $user['notify-flags'], + 'language' => $user['language'], + 'to_name' => $user['username'], + 'to_email' => $user['email'], + 'uid' => $user['uid'], + 'item' => $datarray, + 'link' => $a->get_baseurl().'/display/'.urlencode($datarray['guid']), + 'source_name' => $datarray['author-name'], + 'source_link' => $datarray['author-link'], + 'source_photo' => $datarray['author-avatar'], + 'verb' => ACTIVITY_POST, + 'otype' => 'item' + )); + } + } + + // fallback so that parent always gets set to non-zero. + + if(! $parent) + $parent = $post_id; + + $r = q("UPDATE `item` SET `parent` = %d, `parent-uri` = '%s', `plink` = '%s', `changed` = '%s', `last-child` = 1, `visible` = 1 + WHERE `id` = %d", + intval($parent), + dbesc(($parent == $post_id) ? $uri : $parent_item['uri']), + dbesc($a->get_baseurl().'/display/'.urlencode($datarray['guid'])), + dbesc(datetime_convert()), + intval($post_id) + ); + + // photo comments turn the corresponding item visible to the profile wall + // This way we don't see every picture in your new photo album posted to your wall at once. + // They will show up as people comment on them. + + if(! $parent_item['visible']) { + $r = q("UPDATE `item` SET `visible` = 1 WHERE `id` = %d", + intval($parent_item['id']) + ); + update_thread($parent_item['id']); + } + // update the commented timestamp on the parent q("UPDATE `item` set `commented` = '%s', `changed` = '%s' WHERE `id` = %d", @@ -948,9 +953,6 @@ function item_post(&$a) { if ($post_id != $parent) update_thread($parent); - $datarray['id'] = $post_id; - $datarray['plink'] = $a->get_baseurl().'/display/'.urlencode($datarray['guid']); - call_hooks('post_local_end', $datarray); if(strlen($emailcc) && $profile_uid == local_user()) { diff --git a/mod/search.php b/mod/search.php index 338b377e8..ee9e48bc6 100644 --- a/mod/search.php +++ b/mod/search.php @@ -127,67 +127,49 @@ function search_content(&$a) { if (get_config('system','only_tag_search')) $tag = true; - if($tag) { - $sql_extra = ""; - - $sql_table = sprintf("`item` INNER JOIN (SELECT `oid` FROM `term` WHERE `term` = '%s' AND `otype` = %d AND `type` = %d AND `uid` IN (%d, 0)) AS `term` ON `item`.`id` = `term`.`oid` ", - dbesc(protect_sprintf($search)), intval(TERM_OBJ_POST), intval(TERM_HASHTAG), intval(local_user())); - - $sql_order = "`item`.`id`"; - } else { - if (get_config('system','use_fulltext_engine')) { - $sql_extra = sprintf(" AND MATCH (`item`.`body`, `item`.`title`) AGAINST ('%s' in boolean mode) ", dbesc(protect_sprintf($search))); - } else { - $sql_extra = sprintf(" AND `item`.`body` REGEXP '%s' ", dbesc(protect_sprintf(preg_quote($search)))); - } - $sql_table = "`item`"; - $sql_order = "`item`.`id`"; - //$sql_order = "`item`.`received`"; - } - // Here is the way permissions work in the search module... // Only public posts can be shown // OR your own posts if you are a logged in member // No items will be shown if the member has a blocked profile wall. - if(get_config('system', 'old_pager')) { - $r = q("SELECT distinct(`item`.`uri`) as `total` - FROM $sql_table INNER JOIN `contact` ON `contact`.`id` = `item`.`contact-id` - AND `contact`.`blocked` = 0 AND `contact`.`pending` = 0 - WHERE `item`.`visible` = 1 AND `item`.`deleted` = 0 and `item`.`moderated` = 0 - AND ((`item`.`allow_cid` = '' AND `item`.`allow_gid` = '' AND `item`.`deny_cid` = '' AND `item`.`deny_gid` = '' AND `item`.`private` = 0 AND `item`.`uid` = 0) - OR (`item`.`uid` = %d)) - $sql_extra ", - intval(local_user()) - ); + if($tag) { + logger("Start tag search for '".$search."'"); - if(count($r)) - $a->set_pager_total(count($r)); + $r = q("SELECT `item`.`uri`, `item`.*, `item`.`id` AS `item_id`, + `contact`.`name`, `contact`.`photo`, `contact`.`url`, `contact`.`alias`, `contact`.`rel`, + `contact`.`network`, `contact`.`thumb`, `contact`.`self`, `contact`.`writable`, + `contact`.`id` AS `cid`, `contact`.`uid` AS `contact-uid` + FROM `term` + INNER JOIN `item` ON `item`.`id`=`term`.`oid` + INNER JOIN `contact` ON `contact`.`id` = `item`.`contact-id` AND NOT `contact`.`blocked` AND NOT `contact`.`pending` + WHERE `item`.`visible` AND NOT `item`.`deleted` AND NOT `item`.`moderated` + AND `term`.`uid` IN (%d,0) AND `term`.`otype` = %d AND `term`.`type` = %d AND `term`.`term` = '%s' AND `term`.`guid` != '' + GROUP BY `term`.`guid` ORDER BY term.created DESC LIMIT %d , %d ", + intval(local_user()), intval(TERM_OBJ_POST), intval(TERM_HASHTAG), dbesc(protect_sprintf($search)), + intval($a->pager['start']), intval($a->pager['itemspage'])); + } else { + logger("Start fulltext search for '".$search."'"); - if(! count($r)) { - info( t('No results.') . EOL); - return $o; - } + if (get_config('system','use_fulltext_engine')) { + $sql_extra = sprintf(" AND MATCH (`item`.`body`, `item`.`title`) AGAINST ('%s' in boolean mode) ", dbesc(protect_sprintf($search))); + } else { + $sql_extra = sprintf(" AND `item`.`body` REGEXP '%s' ", dbesc(protect_sprintf(preg_quote($search)))); + } + + $r = q("SELECT `item`.`uri`, `item`.*, `item`.`id` AS `item_id`, + `contact`.`name`, `contact`.`photo`, `contact`.`url`, `contact`.`alias`, `contact`.`rel`, + `contact`.`network`, `contact`.`thumb`, `contact`.`self`, `contact`.`writable`, + `contact`.`id` AS `cid`, `contact`.`uid` AS `contact-uid` + FROM `item` + INNER JOIN `contact` ON `contact`.`id` = `item`.`contact-id` AND NOT `contact`.`blocked` AND NOT `contact`.`pending` + WHERE `item`.`visible` AND NOT `item`.`deleted` AND NOT `item`.`moderated` + AND (`item`.`uid` = 0 OR (`item`.`uid` = %s AND (`item`.`private` OR NOT `item`.`network` IN ('%s', '%s', '%s')))) + $sql_extra + GROUP BY `item`.`uri` ORDER BY `item`.`id` DESC LIMIT %d , %d ", + intval(local_user()), dbesc(NETWORK_DFRN), dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DIASPORA), + intval($a->pager['start']), intval($a->pager['itemspage'])); } - $r = q("SELECT `item`.`uri`, `item`.*, `item`.`id` AS `item_id`, - `contact`.`name`, `contact`.`photo`, `contact`.`url`, `contact`.`alias`, `contact`.`rel`, - `contact`.`network`, `contact`.`thumb`, `contact`.`self`, `contact`.`writable`, - `contact`.`id` AS `cid`, `contact`.`uid` AS `contact-uid` - FROM $sql_table INNER JOIN `contact` ON `contact`.`id` = `item`.`contact-id` - AND `contact`.`blocked` = 0 AND `contact`.`pending` = 0 - WHERE `item`.`visible` = 1 AND `item`.`deleted` = 0 and `item`.`moderated` = 0 - AND ((`item`.`allow_cid` = '' AND `item`.`allow_gid` = '' AND `item`.`deny_cid` = '' AND `item`.`deny_gid` = '' AND `item`.`private` = 0 AND `item`.`uid`=0) - OR `item`.`uid` = %d) - $sql_extra - GROUP BY `item`.`uri` - ORDER BY $sql_order DESC LIMIT %d , %d ", - intval(local_user()), - intval($a->pager['start']), - intval($a->pager['itemspage']) - - ); - if(! count($r)) { info( t('No results.') . EOL); return $o; @@ -199,13 +181,16 @@ function search_content(&$a) { else $o .= '<h2>Search results for: ' . $search . '</h2>'; + logger("Start Conversation"); $o .= conversation($a,$r,'search',false); + logger("Start Pager"); if(!get_config('system', 'old_pager')) { $o .= alt_pager($a,count($r)); } else { $o .= paginate($a); } + logger("Done"); return $o; } diff --git a/update.php b/update.php index 954993a70..93c3c4bef 100644 --- a/update.php +++ b/update.php @@ -1,6 +1,6 @@ <?php -define( 'UPDATE_VERSION' , 1180 ); +define( 'UPDATE_VERSION' , 1181 ); /** * @@ -1640,3 +1640,11 @@ function update_1178() { return UPDATE_SUCCESS; } + +function update_1180() { + + // Fill the new fields in the term table. + proc_run('php',"include/tagupdate.php"); + + return UPDATE_SUCCESS; +} From 1cf3cb493f0ed8e77599e4fece57b26da5b98ae1 Mon Sep 17 00:00:00 2001 From: Michael Vogel <icarus@dabo.de> Date: Sat, 7 Mar 2015 21:39:28 +0100 Subject: [PATCH 07/12] removed unneeded code. --- mod/search.php | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/mod/search.php b/mod/search.php index ee9e48bc6..50d8e60ce 100644 --- a/mod/search.php +++ b/mod/search.php @@ -184,12 +184,8 @@ function search_content(&$a) { logger("Start Conversation"); $o .= conversation($a,$r,'search',false); - logger("Start Pager"); - if(!get_config('system', 'old_pager')) { - $o .= alt_pager($a,count($r)); - } else { - $o .= paginate($a); - } + $o .= alt_pager($a,count($r)); + logger("Done"); return $o; From dbe8275ae5de7610ce21c90088b94a1e18d02921 Mon Sep 17 00:00:00 2001 From: Michael Vogel <icarus@dabo.de> Date: Sat, 7 Mar 2015 23:14:26 +0100 Subject: [PATCH 08/12] Added some more timestamp measuring --- include/bbcode.php | 8 ++ include/network.php | 294 +++++++++++++++++++++++--------------------- include/oembed.php | 1 - mod/parse_url.php | 96 ++++++++------- 4 files changed, 211 insertions(+), 188 deletions(-) diff --git a/include/bbcode.php b/include/bbcode.php index 7e9dcf3ba..1f6a31553 100644 --- a/include/bbcode.php +++ b/include/bbcode.php @@ -681,6 +681,8 @@ function bb_RemovePictureLinks($match) { if(is_null($text)){ $a = get_app(); + $stamp1 = microtime(true); + $ch = @curl_init($match[1]); @curl_setopt($ch, CURLOPT_NOBODY, true); @curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); @@ -688,6 +690,8 @@ function bb_RemovePictureLinks($match) { @curl_exec($ch); $curl_info = @curl_getinfo($ch); + $a->save_timestamp($stamp1, "network"); + if (substr($curl_info["content_type"], 0, 6) == "image/") $text = "[url=".$match[1]."]".$match[1]."[/url]"; else { @@ -731,6 +735,8 @@ function bb_CleanPictureLinksSub($match) { if(is_null($text)){ $a = get_app(); + $stamp1 = microtime(true); + $ch = @curl_init($match[1]); @curl_setopt($ch, CURLOPT_NOBODY, true); @curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); @@ -738,6 +744,8 @@ function bb_CleanPictureLinksSub($match) { @curl_exec($ch); $curl_info = @curl_getinfo($ch); + $a->save_timestamp($stamp1, "network"); + // if its a link to a picture then embed this picture if (substr($curl_info["content_type"], 0, 6) == "image/") $text = "[img]".$match[1]."[/img]"; diff --git a/include/network.php b/include/network.php index ce21f3fbb..a07507da1 100644 --- a/include/network.php +++ b/include/network.php @@ -261,43 +261,43 @@ function http_status_exit($val) { if(! function_exists('convert_xml_element_to_array')) { function convert_xml_element_to_array($xml_element, &$recursion_depth=0) { - // If we're getting too deep, bail out - if ($recursion_depth > 512) { - return(null); - } + // If we're getting too deep, bail out + if ($recursion_depth > 512) { + return(null); + } - if (!is_string($xml_element) && - !is_array($xml_element) && - (get_class($xml_element) == 'SimpleXMLElement')) { - $xml_element_copy = $xml_element; - $xml_element = get_object_vars($xml_element); - } + if (!is_string($xml_element) && + !is_array($xml_element) && + (get_class($xml_element) == 'SimpleXMLElement')) { + $xml_element_copy = $xml_element; + $xml_element = get_object_vars($xml_element); + } - if (is_array($xml_element)) { - $result_array = array(); - if (count($xml_element) <= 0) { - return (trim(strval($xml_element_copy))); - } + if (is_array($xml_element)) { + $result_array = array(); + if (count($xml_element) <= 0) { + return (trim(strval($xml_element_copy))); + } - foreach($xml_element as $key=>$value) { + foreach($xml_element as $key=>$value) { - $recursion_depth++; - $result_array[strtolower($key)] = - convert_xml_element_to_array($value, $recursion_depth); - $recursion_depth--; - } - if ($recursion_depth == 0) { - $temp_array = $result_array; - $result_array = array( - strtolower($xml_element_copy->getName()) => $temp_array, - ); - } + $recursion_depth++; + $result_array[strtolower($key)] = + convert_xml_element_to_array($value, $recursion_depth); + $recursion_depth--; + } + if ($recursion_depth == 0) { + $temp_array = $result_array; + $result_array = array( + strtolower($xml_element_copy->getName()) => $temp_array, + ); + } - return ($result_array); + return ($result_array); - } else { - return (trim(strval($xml_element))); - } + } else { + return (trim(strval($xml_element))); + } }} // Given an email style address, perform webfinger lookup and @@ -964,8 +964,8 @@ function xml2array($contents, $namespaces = true, $get_attributes=1, $priority = if(!$contents) return array(); if(!function_exists('xml_parser_create')) { - logger('xml2array: parser function missing'); - return array(); + logger('xml2array: parser function missing'); + return array(); } @@ -1008,29 +1008,29 @@ function xml2array($contents, $namespaces = true, $get_attributes=1, $priority = // Go through the tags. $repeated_tag_index = array(); // Multiple tags with same name will be turned into an array foreach($xml_values as $data) { - unset($attributes,$value); // Remove existing values, or there will be trouble + unset($attributes,$value); // Remove existing values, or there will be trouble - // This command will extract these variables into the foreach scope - // tag(string), type(string), level(int), attributes(array). - extract($data); // We could use the array by itself, but this cooler. + // This command will extract these variables into the foreach scope + // tag(string), type(string), level(int), attributes(array). + extract($data); // We could use the array by itself, but this cooler. - $result = array(); - $attributes_data = array(); + $result = array(); + $attributes_data = array(); - if(isset($value)) { - if($priority == 'tag') $result = $value; - else $result['value'] = $value; // Put the value in a assoc array if we are in the 'Attribute' mode - } + if(isset($value)) { + if($priority == 'tag') $result = $value; + else $result['value'] = $value; // Put the value in a assoc array if we are in the 'Attribute' mode + } - //Set the attributes too. - if(isset($attributes) and $get_attributes) { - foreach($attributes as $attr => $val) { - if($priority == 'tag') $attributes_data[$attr] = $val; - else $result['@attributes'][$attr] = $val; // Set all the attributes in a array called 'attr' - } - } + //Set the attributes too. + if(isset($attributes) and $get_attributes) { + foreach($attributes as $attr => $val) { + if($priority == 'tag') $attributes_data[$attr] = $val; + else $result['@attributes'][$attr] = $val; // Set all the attributes in a array called 'attr' + } + } - // See tag status and do the needed. + // See tag status and do the needed. if($namespaces && strpos($tag,':')) { $namespc = substr($tag,0,strrpos($tag,':')); $tag = strtolower(substr($tag,strlen($namespc)+1)); @@ -1039,72 +1039,72 @@ function xml2array($contents, $namespaces = true, $get_attributes=1, $priority = $tag = strtolower($tag); if($type == "open") { // The starting of the tag '<tag>' - $parent[$level-1] = &$current; - if(!is_array($current) or (!in_array($tag, array_keys($current)))) { // Insert New tag - $current[$tag] = $result; - if($attributes_data) $current[$tag. '_attr'] = $attributes_data; - $repeated_tag_index[$tag.'_'.$level] = 1; + $parent[$level-1] = &$current; + if(!is_array($current) or (!in_array($tag, array_keys($current)))) { // Insert New tag + $current[$tag] = $result; + if($attributes_data) $current[$tag. '_attr'] = $attributes_data; + $repeated_tag_index[$tag.'_'.$level] = 1; - $current = &$current[$tag]; + $current = &$current[$tag]; - } else { // There was another element with the same tag name + } else { // There was another element with the same tag name - if(isset($current[$tag][0])) { // If there is a 0th element it is already an array - $current[$tag][$repeated_tag_index[$tag.'_'.$level]] = $result; - $repeated_tag_index[$tag.'_'.$level]++; - } else { // This section will make the value an array if multiple tags with the same name appear together - $current[$tag] = array($current[$tag],$result); // This will combine the existing item and the new item together to make an array - $repeated_tag_index[$tag.'_'.$level] = 2; + if(isset($current[$tag][0])) { // If there is a 0th element it is already an array + $current[$tag][$repeated_tag_index[$tag.'_'.$level]] = $result; + $repeated_tag_index[$tag.'_'.$level]++; + } else { // This section will make the value an array if multiple tags with the same name appear together + $current[$tag] = array($current[$tag],$result); // This will combine the existing item and the new item together to make an array + $repeated_tag_index[$tag.'_'.$level] = 2; - if(isset($current[$tag.'_attr'])) { // The attribute of the last(0th) tag must be moved as well - $current[$tag]['0_attr'] = $current[$tag.'_attr']; - unset($current[$tag.'_attr']); - } + if(isset($current[$tag.'_attr'])) { // The attribute of the last(0th) tag must be moved as well + $current[$tag]['0_attr'] = $current[$tag.'_attr']; + unset($current[$tag.'_attr']); + } - } - $last_item_index = $repeated_tag_index[$tag.'_'.$level]-1; - $current = &$current[$tag][$last_item_index]; - } + } + $last_item_index = $repeated_tag_index[$tag.'_'.$level]-1; + $current = &$current[$tag][$last_item_index]; + } - } elseif($type == "complete") { // Tags that ends in 1 line '<tag />' - //See if the key is already taken. - if(!isset($current[$tag])) { //New Key - $current[$tag] = $result; - $repeated_tag_index[$tag.'_'.$level] = 1; - if($priority == 'tag' and $attributes_data) $current[$tag. '_attr'] = $attributes_data; + } elseif($type == "complete") { // Tags that ends in 1 line '<tag />' + //See if the key is already taken. + if(!isset($current[$tag])) { //New Key + $current[$tag] = $result; + $repeated_tag_index[$tag.'_'.$level] = 1; + if($priority == 'tag' and $attributes_data) $current[$tag. '_attr'] = $attributes_data; - } else { // If taken, put all things inside a list(array) - if(isset($current[$tag][0]) and is_array($current[$tag])) { // If it is already an array... + } else { // If taken, put all things inside a list(array) + if(isset($current[$tag][0]) and is_array($current[$tag])) { // If it is already an array... - // ...push the new element into that array. - $current[$tag][$repeated_tag_index[$tag.'_'.$level]] = $result; + // ...push the new element into that array. + $current[$tag][$repeated_tag_index[$tag.'_'.$level]] = $result; - if($priority == 'tag' and $get_attributes and $attributes_data) { - $current[$tag][$repeated_tag_index[$tag.'_'.$level] . '_attr'] = $attributes_data; - } - $repeated_tag_index[$tag.'_'.$level]++; + if($priority == 'tag' and $get_attributes and $attributes_data) { + $current[$tag][$repeated_tag_index[$tag.'_'.$level] . '_attr'] = $attributes_data; + } + $repeated_tag_index[$tag.'_'.$level]++; - } else { // If it is not an array... - $current[$tag] = array($current[$tag],$result); //...Make it an array using using the existing value and the new value - $repeated_tag_index[$tag.'_'.$level] = 1; - if($priority == 'tag' and $get_attributes) { - if(isset($current[$tag.'_attr'])) { // The attribute of the last(0th) tag must be moved as well + } else { // If it is not an array... + $current[$tag] = array($current[$tag],$result); //...Make it an array using using the existing value and the new value + $repeated_tag_index[$tag.'_'.$level] = 1; + if($priority == 'tag' and $get_attributes) { + if(isset($current[$tag.'_attr'])) { // The attribute of the last(0th) tag must be moved as well - $current[$tag]['0_attr'] = $current[$tag.'_attr']; - unset($current[$tag.'_attr']); - } + $current[$tag]['0_attr'] = $current[$tag.'_attr']; + unset($current[$tag.'_attr']); + } - if($attributes_data) { - $current[$tag][$repeated_tag_index[$tag.'_'.$level] . '_attr'] = $attributes_data; - } - } - $repeated_tag_index[$tag.'_'.$level]++; // 0 and 1 indexes are already taken - } - } + if($attributes_data) { + $current[$tag][$repeated_tag_index[$tag.'_'.$level] . '_attr'] = $attributes_data; + } + } + $repeated_tag_index[$tag.'_'.$level]++; // 0 and 1 indexes are already taken + } + } - } elseif($type == 'close') { // End of tag '</tag>' - $current = &$parent[$level-1]; - } + } elseif($type == 'close') { // End of tag '</tag>' + $current = &$parent[$level-1]; + } } return($xml_array); @@ -1146,32 +1146,36 @@ function original_url($url, $depth=1, $fetchbody = false) { $url = substr($url, 0, -1); } - if ($depth > 10) - return($url); + if ($depth > 10) + return($url); - $url = trim($url, "'"); + $url = trim($url, "'"); - $siteinfo = array(); - $ch = curl_init(); - curl_setopt($ch, CURLOPT_URL, $url); + $stamp1 = microtime(true); + + $siteinfo = array(); + $ch = curl_init(); + curl_setopt($ch, CURLOPT_URL, $url); curl_setopt($ch, CURLOPT_HEADER, 1); curl_setopt($ch, CURLOPT_NOBODY, 1); - curl_setopt($ch, CURLOPT_TIMEOUT, 10); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_TIMEOUT, 10); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); curl_setopt($ch, CURLOPT_USERAGENT, $a->get_useragent()); - $header = curl_exec($ch); - $curl_info = @curl_getinfo($ch); - $http_code = $curl_info['http_code']; - curl_close($ch); + $header = curl_exec($ch); + $curl_info = @curl_getinfo($ch); + $http_code = $curl_info['http_code']; + curl_close($ch); - if ((($curl_info['http_code'] == "301") OR ($curl_info['http_code'] == "302")) - AND (($curl_info['redirect_url'] != "") OR ($curl_info['location'] != ""))) { - if ($curl_info['redirect_url'] != "") - return(original_url($curl_info['redirect_url'], ++$depth, $fetchbody)); - else - return(original_url($curl_info['location'], ++$depth, $fetchbody)); - } + $a->save_timestamp($stamp1, "network"); + + if ((($curl_info['http_code'] == "301") OR ($curl_info['http_code'] == "302")) + AND (($curl_info['redirect_url'] != "") OR ($curl_info['location'] != ""))) { + if ($curl_info['redirect_url'] != "") + return(original_url($curl_info['redirect_url'], ++$depth, $fetchbody)); + else + return(original_url($curl_info['location'], ++$depth, $fetchbody)); + } // Check for redirects in the meta elements of the body if there are no redirects in the header. if (!$fetchbody) @@ -1185,6 +1189,8 @@ function original_url($url, $depth=1, $fetchbody = false) { if (($curl_info["content_type"] != "") AND !strstr(strtolower($curl_info["content_type"]),"html")) return($url); + $stamp1 = microtime(true); + $ch = curl_init(); curl_setopt($ch, CURLOPT_URL, $url); curl_setopt($ch, CURLOPT_HEADER, 0); @@ -1196,33 +1202,35 @@ function original_url($url, $depth=1, $fetchbody = false) { $body = curl_exec($ch); curl_close($ch); - if (trim($body) == "") + $a->save_timestamp($stamp1, "network"); + + if (trim($body) == "") return($url); // Check for redirect in meta elements - $doc = new DOMDocument(); - @$doc->loadHTML($body); + $doc = new DOMDocument(); + @$doc->loadHTML($body); - $xpath = new DomXPath($doc); + $xpath = new DomXPath($doc); - $list = $xpath->query("//meta[@content]"); - foreach ($list as $node) { - $attr = array(); - if ($node->attributes->length) - foreach ($node->attributes as $attribute) - $attr[$attribute->name] = $attribute->value; + $list = $xpath->query("//meta[@content]"); + foreach ($list as $node) { + $attr = array(); + if ($node->attributes->length) + foreach ($node->attributes as $attribute) + $attr[$attribute->name] = $attribute->value; - if (@$attr["http-equiv"] == 'refresh') { - $path = $attr["content"]; - $pathinfo = explode(";", $path); - $content = ""; - foreach ($pathinfo AS $value) - if (substr(strtolower($value), 0, 4) == "url=") - return(original_url(substr($value, 4), ++$depth)); - } - } + if (@$attr["http-equiv"] == 'refresh') { + $path = $attr["content"]; + $pathinfo = explode(";", $path); + $content = ""; + foreach ($pathinfo AS $value) + if (substr(strtolower($value), 0, 4) == "url=") + return(original_url(substr($value, 4), ++$depth)); + } + } - return($url); + return($url); } if (!function_exists('short_link')) { diff --git a/include/oembed.php b/include/oembed.php index 19bdc474f..69583167c 100755 --- a/include/oembed.php +++ b/include/oembed.php @@ -11,7 +11,6 @@ function oembed_replacecb($matches){ function oembed_fetch_url($embedurl, $no_rich_type = false){ - $embedurl = trim($embedurl, "'"); $embedurl = trim($embedurl, '"'); diff --git a/mod/parse_url.php b/mod/parse_url.php index 9f7b31be3..cd2263dbe 100644 --- a/mod/parse_url.php +++ b/mod/parse_url.php @@ -24,30 +24,30 @@ if(!function_exists('deletenode')) { } function completeurl($url, $scheme) { - $urlarr = parse_url($url); + $urlarr = parse_url($url); - if (isset($urlarr["scheme"])) - return($url); + if (isset($urlarr["scheme"])) + return($url); - $schemearr = parse_url($scheme); + $schemearr = parse_url($scheme); - $complete = $schemearr["scheme"]."://".$schemearr["host"]; + $complete = $schemearr["scheme"]."://".$schemearr["host"]; - if (@$schemearr["port"] != "") - $complete .= ":".$schemearr["port"]; + if (@$schemearr["port"] != "") + $complete .= ":".$schemearr["port"]; if(strpos($urlarr['path'],'/') !== 0) $complete .= '/'; - $complete .= $urlarr["path"]; + $complete .= $urlarr["path"]; - if (@$urlarr["query"] != "") - $complete .= "?".$urlarr["query"]; + if (@$urlarr["query"] != "") + $complete .= "?".$urlarr["query"]; - if (@$urlarr["fragment"] != "") - $complete .= "#".$urlarr["fragment"]; + if (@$urlarr["fragment"] != "") + $complete .= "#".$urlarr["fragment"]; - return($complete); + return($complete); } function parseurl_getsiteinfo($url, $no_guessing = false, $do_oembed = true, $count = 1) { @@ -70,6 +70,8 @@ function parseurl_getsiteinfo($url, $no_guessing = false, $do_oembed = true, $co $siteinfo["url"] = $url; $siteinfo["type"] = "link"; + $stamp1 = microtime(true); + $ch = curl_init(); curl_setopt($ch, CURLOPT_URL, $url); curl_setopt($ch, CURLOPT_HEADER, 1); @@ -81,9 +83,11 @@ function parseurl_getsiteinfo($url, $no_guessing = false, $do_oembed = true, $co $header = curl_exec($ch); $curl_info = @curl_getinfo($ch); - $http_code = $curl_info['http_code']; + $http_code = $curl_info['http_code']; curl_close($ch); + $a->save_timestamp($stamp1, "network"); + if ((($curl_info['http_code'] == "301") OR ($curl_info['http_code'] == "302") OR ($curl_info['http_code'] == "303") OR ($curl_info['http_code'] == "307")) AND (($curl_info['redirect_url'] != "") OR ($curl_info['location'] != ""))) { if ($curl_info['redirect_url'] != "") @@ -110,6 +114,8 @@ function parseurl_getsiteinfo($url, $no_guessing = false, $do_oembed = true, $co if (($curl_info["content_type"] != "") AND !strstr(strtolower($curl_info["content_type"]),"html")) return($siteinfo); + $stamp1 = microtime(true); + // Now fetch the body as well $ch = curl_init(); curl_setopt($ch, CURLOPT_URL, $url); @@ -121,9 +127,11 @@ function parseurl_getsiteinfo($url, $no_guessing = false, $do_oembed = true, $co $header = curl_exec($ch); $curl_info = @curl_getinfo($ch); - $http_code = $curl_info['http_code']; + $http_code = $curl_info['http_code']; curl_close($ch); + $a->save_timestamp($stamp1, "network"); + // Fetch the first mentioned charset. Can be in body or header $charset = ""; if (preg_match('/charset=(.*?)['."'".'"\s\n]/', $header, $matches)) @@ -165,25 +173,25 @@ function parseurl_getsiteinfo($url, $no_guessing = false, $do_oembed = true, $co $xpath = new DomXPath($doc); $list = $xpath->query("//meta[@content]"); - foreach ($list as $node) { - $attr = array(); - if ($node->attributes->length) - foreach ($node->attributes as $attribute) - $attr[$attribute->name] = $attribute->value; + foreach ($list as $node) { + $attr = array(); + if ($node->attributes->length) + foreach ($node->attributes as $attribute) + $attr[$attribute->name] = $attribute->value; - if (@$attr["http-equiv"] == 'refresh') { - $path = $attr["content"]; - $pathinfo = explode(";", $path); - $content = ""; - foreach ($pathinfo AS $value) { - if (substr(strtolower($value), 0, 4) == "url=") - $content = substr($value, 4); - } - if ($content != "") { - $siteinfo = parseurl_getsiteinfo($content, $no_guessing, $do_oembed, ++$count); - return($siteinfo); - } - } + if (@$attr["http-equiv"] == 'refresh') { + $path = $attr["content"]; + $pathinfo = explode(";", $path); + $content = ""; + foreach ($pathinfo AS $value) { + if (substr(strtolower($value), 0, 4) == "url=") + $content = substr($value, 4); + } + if ($content != "") { + $siteinfo = parseurl_getsiteinfo($content, $no_guessing, $do_oembed, ++$count); + return($siteinfo); + } + } } //$list = $xpath->query("head/title"); @@ -196,8 +204,8 @@ function parseurl_getsiteinfo($url, $no_guessing = false, $do_oembed = true, $co foreach ($list as $node) { $attr = array(); if ($node->attributes->length) - foreach ($node->attributes as $attribute) - $attr[$attribute->name] = $attribute->value; + foreach ($node->attributes as $attribute) + $attr[$attribute->name] = $attribute->value; $attr["content"] = trim(html_entity_decode($attr["content"], ENT_QUOTES, "UTF-8")); @@ -256,8 +264,8 @@ function parseurl_getsiteinfo($url, $no_guessing = false, $do_oembed = true, $co foreach ($list as $node) { $attr = array(); if ($node->attributes->length) - foreach ($node->attributes as $attribute) - $attr[$attribute->name] = $attribute->value; + foreach ($node->attributes as $attribute) + $attr[$attribute->name] = $attribute->value; $attr["content"] = trim(html_entity_decode($attr["content"], ENT_QUOTES, "UTF-8")); @@ -285,12 +293,12 @@ function parseurl_getsiteinfo($url, $no_guessing = false, $do_oembed = true, $co } if ((@$siteinfo["image"] == "") AND !$no_guessing) { - $list = $xpath->query("//img[@src]"); - foreach ($list as $node) { - $attr = array(); - if ($node->attributes->length) - foreach ($node->attributes as $attribute) - $attr[$attribute->name] = $attribute->value; + $list = $xpath->query("//img[@src]"); + foreach ($list as $node) { + $attr = array(); + if ($node->attributes->length) + foreach ($node->attributes as $attribute) + $attr[$attribute->name] = $attribute->value; $src = completeurl($attr["src"], $url); $photodata = @getimagesize($src); @@ -309,7 +317,7 @@ function parseurl_getsiteinfo($url, $no_guessing = false, $do_oembed = true, $co "height"=>$photodata[1]); } - } + } } else { $src = completeurl($siteinfo["image"], $url); From b080eddf8dbd2c4856b8e898992c7ff823639d28 Mon Sep 17 00:00:00 2001 From: Michael Vogel <icarus@dabo.de> Date: Sun, 8 Mar 2015 03:24:54 +0100 Subject: [PATCH 09/12] Improved logging for the search. --- mod/search.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mod/search.php b/mod/search.php index 50d8e60ce..670df299e 100644 --- a/mod/search.php +++ b/mod/search.php @@ -133,7 +133,7 @@ function search_content(&$a) { // No items will be shown if the member has a blocked profile wall. if($tag) { - logger("Start tag search for '".$search."'"); + logger("Start tag search for '".$search."'", LOGGER_DEBUG); $r = q("SELECT `item`.`uri`, `item`.*, `item`.`id` AS `item_id`, `contact`.`name`, `contact`.`photo`, `contact`.`url`, `contact`.`alias`, `contact`.`rel`, @@ -148,7 +148,7 @@ function search_content(&$a) { intval(local_user()), intval(TERM_OBJ_POST), intval(TERM_HASHTAG), dbesc(protect_sprintf($search)), intval($a->pager['start']), intval($a->pager['itemspage'])); } else { - logger("Start fulltext search for '".$search."'"); + logger("Start fulltext search for '".$search."'", LOGGER_DEBUG); if (get_config('system','use_fulltext_engine')) { $sql_extra = sprintf(" AND MATCH (`item`.`body`, `item`.`title`) AGAINST ('%s' in boolean mode) ", dbesc(protect_sprintf($search))); @@ -181,12 +181,12 @@ function search_content(&$a) { else $o .= '<h2>Search results for: ' . $search . '</h2>'; - logger("Start Conversation"); + logger("Start Conversation for '".$search."'", LOGGER_DEBUG); $o .= conversation($a,$r,'search',false); $o .= alt_pager($a,count($r)); - logger("Done"); + logger("Done '".$search."'", LOGGER_DEBUG); return $o; } From e2965c9ec5b0ffeea6c21912b20294050069523f Mon Sep 17 00:00:00 2001 From: Michael Vogel <icarus@dabo.de> Date: Sun, 8 Mar 2015 03:25:32 +0100 Subject: [PATCH 10/12] Removed the timestamp for parser. --- include/bbcode.php | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/include/bbcode.php b/include/bbcode.php index 1f6a31553..3cf67af69 100644 --- a/include/bbcode.php +++ b/include/bbcode.php @@ -787,8 +787,6 @@ function bb_CleanPictureLinks($text) { function bbcode($Text,$preserve_nl = false, $tryoembed = true, $simplehtml = false, $forplaintext = false) { - $stamp1 = microtime(true); - $a = get_app(); // Hide all [noparse] contained bbtags by spacefying them @@ -860,13 +858,9 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true, $simplehtml = fal } while ($oldtext != $Text); } - $a->save_timestamp($stamp1, "parser"); - // Handle attached links or videos $Text = bb_attachment($Text, ($simplehtml != 4) AND ($simplehtml != 0), $tryoembed); - $stamp1 = microtime(true); - $Text = str_replace(array("\r","\n"), array('<br />','<br />'), $Text); if($preserve_nl) @@ -896,14 +890,10 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true, $simplehtml = fal if ($simplehtml == 5) $Text = preg_replace("/[^#@]\[url\=([$URLSearchString]*)\](.*?)\[\/url\]/ism", '[url]$1[/url]', $Text); - $a->save_timestamp($stamp1, "parser"); - // Perform URL Search if ($tryoembed) $Text = preg_replace_callback("/\[bookmark\=([^\]]*)\](.*?)\[\/bookmark\]/ism",'tryoembed',$Text); - $stamp1 = microtime(true); - if ($simplehtml == 5) $Text = preg_replace("/\[bookmark\=([^\]]*)\](.*?)\[\/bookmark\]/ism",'[url]$1[/url]',$Text); else @@ -912,8 +902,6 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true, $simplehtml = fal // Handle Diaspora posts $Text = preg_replace_callback("&\[url=/posts/([^\[\]]*)\](.*)\[\/url\]&Usi", 'bb_DiasporaLinks', $Text); - $a->save_timestamp($stamp1, "parser"); - // if the HTML is used to generate plain text, then don't do this search, but replace all URL of that kind to text if (!$forplaintext) $Text = preg_replace("/([^\]\='".'"'."]|^)(https?\:\/\/[a-zA-Z0-9\:\/\-\?\&\;\.\=\_\~\#\%\$\!\+\,]+)/ism", '$1<a href="$2" target="_blank">$2</a>', $Text); @@ -925,8 +913,6 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true, $simplehtml = fal if ($tryoembed) $Text = preg_replace_callback("/\[url\]([$URLSearchString]*)\[\/url\]/ism",'tryoembed',$Text); - $stamp1 = microtime(true); - $Text = preg_replace("/\[url\]([$URLSearchString]*)\[\/url\]/ism", '<a href="$1" target="_blank">$1</a>', $Text); $Text = preg_replace("/\[url\=([$URLSearchString]*)\](.*?)\[\/url\]/ism", '<a href="$1" target="_blank">$2</a>', $Text); //$Text = preg_replace("/\[url\=([$URLSearchString]*)\]([$URLSearchString]*)\[\/url\]/ism", '<a href="$1" target="_blank">$2</a>', $Text); @@ -1098,13 +1084,8 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true, $simplehtml = fal $Text = preg_replace("/\[video\](.*?\.(ogg|ogv|oga|ogm|webm|mp4))\[\/video\]/ism", '<video src="$1" controls="controls" width="' . $a->videowidth . '" height="' . $a->videoheight . '"><a href="$1">$1</a></video>', $Text); $Text = preg_replace("/\[audio\](.*?\.(ogg|ogv|oga|ogm|webm|mp4|mp3))\[\/audio\]/ism", '<audio src="$1" controls="controls"><a href="$1">$1</a></audio>', $Text); - $a->save_timestamp($stamp1, "parser"); - $Text = preg_replace_callback("/\[video\](.*?)\[\/video\]/ism", 'tryoembed', $Text); $Text = preg_replace_callback("/\[audio\](.*?)\[\/audio\]/ism", 'tryoembed', $Text); - - $stamp1 = microtime(true); - } else { $Text = preg_replace("/\[video\](.*?)\[\/video\]/", '<a href="$1" target="_blank">$1</a>', $Text); @@ -1122,13 +1103,9 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true, $simplehtml = fal // Youtube extensions if ($tryoembed) { - $a->save_timestamp($stamp1, "parser"); - $Text = preg_replace_callback("/\[youtube\](https?:\/\/www.youtube.com\/watch\?v\=.*?)\[\/youtube\]/ism", 'tryoembed', $Text); $Text = preg_replace_callback("/\[youtube\](www.youtube.com\/watch\?v\=.*?)\[\/youtube\]/ism", 'tryoembed', $Text); $Text = preg_replace_callback("/\[youtube\](https?:\/\/youtu.be\/.*?)\[\/youtube\]/ism",'tryoembed',$Text); - - $stamp1 = microtime(true); } $Text = preg_replace("/\[youtube\]https?:\/\/www.youtube.com\/watch\?v\=(.*?)\[\/youtube\]/ism",'[youtube]$1[/youtube]',$Text); @@ -1142,12 +1119,8 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true, $simplehtml = fal '<a href="https://www.youtube.com/watch?v=$1" target="_blank">https://www.youtube.com/watch?v=$1</a>', $Text); if ($tryoembed) { - $a->save_timestamp($stamp1, "parser"); - $Text = preg_replace_callback("/\[vimeo\](https?:\/\/player.vimeo.com\/video\/[0-9]+).*?\[\/vimeo\]/ism",'tryoembed',$Text); $Text = preg_replace_callback("/\[vimeo\](https?:\/\/vimeo.com\/[0-9]+).*?\[\/vimeo\]/ism",'tryoembed',$Text); - - $stamp1 = microtime(true); } $Text = preg_replace("/\[vimeo\]https?:\/\/player.vimeo.com\/video\/([0-9]+)(.*?)\[\/vimeo\]/ism",'[vimeo]$1[/vimeo]',$Text); @@ -1161,13 +1134,9 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true, $simplehtml = fal // $Text = preg_replace("/\[youtube\](.*?)\[\/youtube\]/", '<object width="425" height="350" type="application/x-shockwave-flash" data="http://www.youtube.com/v/$1" ><param name="movie" value="http://www.youtube.com/v/$1"></param><!--[if IE]><embed src="http://www.youtube.com/v/$1" type="application/x-shockwave-flash" width="425" height="350" /><![endif]--></object>', $Text); - $a->save_timestamp($stamp1, "parser"); - // oembed tag $Text = oembed_bbcode2html($Text); - $stamp1 = microtime(true); - // Avoid triple linefeeds through oembed $Text = str_replace("<br style='clear:left'></span><br /><br />", "<br style='clear:left'></span><br />", $Text); @@ -1236,8 +1205,6 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true, $simplehtml = fal call_hooks('bbcode',$Text); - $a->save_timestamp($stamp1, "parser"); - return trim($Text); } ?> From 2260415ca9be83e3389175d994d1d10c3b24c7e6 Mon Sep 17 00:00:00 2001 From: Michael Vogel <icarus@dabo.de> Date: Sun, 8 Mar 2015 03:27:14 +0100 Subject: [PATCH 11/12] "global" flag for terms that indicate if the term is present in the global posts. --- include/dbstructure.php | 2 ++ include/tags.php | 40 +++++++++++++++++++++++++++++++++------- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/include/dbstructure.php b/include/dbstructure.php index ff24be5de..630eb4fb2 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -1193,6 +1193,7 @@ function db_definition() { "guid" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "created" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), "received" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), + "global" => array("type" => "tinyint(1)", "not null" => "1", "default" => "0"), "aid" => array("type" => "int(10) unsigned", "not null" => "1", "default" => "0"), "uid" => array("type" => "int(10) unsigned", "not null" => "1", "default" => "0"), ), @@ -1203,6 +1204,7 @@ function db_definition() { "type_term" => array("type","term"), "uid_otype_type_term_created" => array("uid","otype","type","term","created"), "otype_type_term_tid" => array("otype","type","term","tid"), + "guid" => array("guid"), ) ); $database["thread"] = array( diff --git a/include/tags.php b/include/tags.php index 72cac1d63..e3e4edfa1 100644 --- a/include/tags.php +++ b/include/tags.php @@ -69,10 +69,22 @@ function create_tags_from_item($itemid) { $term = $tag; } - $r = q("INSERT INTO `term` (`uid`, `oid`, `otype`, `type`, `term`, `url`, `guid`, `created`, `received`) - VALUES (%d, %d, %d, %d, '%s', '%s', '%s', '%s', '%s')", - intval($message["uid"]), intval($itemid), intval(TERM_OBJ_POST), intval($type), - dbesc($term), dbesc($link), dbesc($message["guid"]), dbesc($message["created"]), dbesc($message["received"])); + if ($message["uid"] == 0) { + $global = true; + + q("UPDATE `term` SET `global` = 1 WHERE `otype` = %d AND `guid` = '%s'", + intval(TERM_OBJ_POST), dbesc($message["guid"])); + } else { + $isglobal = q("SELECT `global` FROM `term` WHERE `uid` = 0 AND `otype` = %d AND `guid` = '%s'", + intval(TERM_OBJ_POST), dbesc($message["guid"])); + + $global = (count($isglobal) > 0); + } + + $r = q("INSERT INTO `term` (`uid`, `oid`, `otype`, `type`, `term`, `url`, `guid`, `created`, `received`, `global`) + VALUES (%d, %d, %d, %d, '%s', '%s', '%s', '%s', '%s', %d)", + intval($message["uid"]), intval($itemid), intval(TERM_OBJ_POST), intval($type), dbesc($term), + dbesc($link), dbesc($message["guid"]), dbesc($message["created"]), dbesc($message["received"]), intval($global)); // Search for mentions if ((substr($tag, 0, 1) == '@') AND (strpos($link, $profile_base_friendica) OR strpos($link, $profile_base_diaspora))) { @@ -100,13 +112,27 @@ function create_tags_from_itemuri($itemuri, $uid) { function update_items() { global $db; - $messages = $db->q("SELECT `oid`,`item`.`guid`, `item`.`created`, `item`.`received` FROM `term` INNER JOIN `item` ON `item`.`id`=`term`.`oid` WHERE `term`.`otype` = 1 AND `term`.`guid` = ''", true); +// $messages = $db->q("SELECT `oid`,`item`.`guid`, `item`.`created`, `item`.`received` FROM `term` INNER JOIN `item` ON `item`.`id`=`term`.`oid` WHERE `term`.`otype` = 1 AND `term`.`guid` = ''", true); + $messages = $db->q("SELECT `oid`,`item`.`guid`, `item`.`created`, `item`.`received` FROM `term` INNER JOIN `item` ON `item`.`id`=`term`.`oid` WHERE `term`.`otype` = 1", true); logger("fetched messages: ".count($messages)); while ($message = $db->qfetch()) { - q("UPDATE `term` SET `guid` = '%s', `created` = '%s', `received` = '%s' WHERE `otype` = %d AND `oid` = %d", + + if ($message["uid"] == 0) { + $global = true; + + q("UPDATE `term` SET `global` = 1 WHERE `otype` = %d AND `guid` = '%s'", + intval(TERM_OBJ_POST), dbesc($message["guid"])); + } else { + $isglobal = q("SELECT `global` FROM `term` WHERE `uid` = 0 AND `otype` = %d AND `guid` = '%s'", + intval(TERM_OBJ_POST), dbesc($message["guid"])); + + $global = (count($isglobal) > 0); + } +echo $message["created"]." - ".$message["guid"]."\n"; + q("UPDATE `term` SET `guid` = '%s', `created` = '%s', `received` = '%s', `global` = %d WHERE `otype` = %d AND `oid` = %d", dbesc($message["guid"]), dbesc($message["created"]), dbesc($message["received"]), - intval(TERM_OBJ_POST), intval($message["oid"])); + intval($global), intval(TERM_OBJ_POST), intval($message["oid"])); } $db->qclose(); From ea06a1ec4584c298a1d71c139bed196a0e54dcec Mon Sep 17 00:00:00 2001 From: Michael Vogel <icarus@dabo.de> Date: Mon, 9 Mar 2015 00:45:53 +0100 Subject: [PATCH 12/12] There is now a "global" field in the item table that tells if this item is present as global copy as well. --- include/dbstructure.php | 1 + include/items.php | 11 +++++++++++ include/tags.php | 14 +++++++++++--- mod/search.php | 4 ++-- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/include/dbstructure.php b/include/dbstructure.php index 630eb4fb2..c078e4051 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -778,6 +778,7 @@ function db_definition() { "network" => array("type" => "varchar(32)", "not null" => "1", "default" => ""), "rendered-hash" => array("type" => "varchar(32)", "not null" => "1", "default" => ""), "rendered-html" => array("type" => "mediumtext", "not null" => "1"), + "global" => array("type" => "tinyint(1)", "not null" => "1", "default" => "0"), ), "indexes" => array( "PRIMARY" => array("id"), diff --git a/include/items.php b/include/items.php index 8f9eb5aa6..3ec476ad6 100644 --- a/include/items.php +++ b/include/items.php @@ -1346,6 +1346,17 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa return 0; } + // Is this item available in the global items (with uid=0)? + if ($arr["uid"] == 0) { + $arr["global"] = true; + + q("UPDATE `item` SET `global` = 1 WHERE `guid` = '%s'", dbesc($arr["guid"])); + } else { + $isglobal = q("SELECT `global` FROM `item` WHERE `uid` = 0 AND `guid` = '%s'", dbesc($arr["guid"])); + + $arr["global"] = (count($isglobal) > 0); + } + // Fill the cache field put_item_in_cache($arr); diff --git a/include/tags.php b/include/tags.php index e3e4edfa1..fc1b57db4 100644 --- a/include/tags.php +++ b/include/tags.php @@ -112,8 +112,7 @@ function create_tags_from_itemuri($itemuri, $uid) { function update_items() { global $db; -// $messages = $db->q("SELECT `oid`,`item`.`guid`, `item`.`created`, `item`.`received` FROM `term` INNER JOIN `item` ON `item`.`id`=`term`.`oid` WHERE `term`.`otype` = 1 AND `term`.`guid` = ''", true); - $messages = $db->q("SELECT `oid`,`item`.`guid`, `item`.`created`, `item`.`received` FROM `term` INNER JOIN `item` ON `item`.`id`=`term`.`oid` WHERE `term`.`otype` = 1", true); + $messages = $db->q("SELECT `oid`,`item`.`guid`, `item`.`created`, `item`.`received` FROM `term` INNER JOIN `item` ON `item`.`id`=`term`.`oid` WHERE `term`.`otype` = 1 AND `term`.`guid` = ''", true); logger("fetched messages: ".count($messages)); while ($message = $db->qfetch()) { @@ -129,12 +128,21 @@ function update_items() { $global = (count($isglobal) > 0); } -echo $message["created"]." - ".$message["guid"]."\n"; + q("UPDATE `term` SET `guid` = '%s', `created` = '%s', `received` = '%s', `global` = %d WHERE `otype` = %d AND `oid` = %d", dbesc($message["guid"]), dbesc($message["created"]), dbesc($message["received"]), intval($global), intval(TERM_OBJ_POST), intval($message["oid"])); } $db->qclose(); + + $messages = $db->q("SELECT `guid` FROM `item` WHERE `uid` = 0", true); + + logger("fetched messages: ".count($messages)); + while ($message = $db->qfetch()) { + q("UPDATE `item` SET `global` = 1 WHERE `guid` = '%s'", dbesc($message["guid"])); + } + + $db->qclose(); } ?> diff --git a/mod/search.php b/mod/search.php index 670df299e..b9bad6405 100644 --- a/mod/search.php +++ b/mod/search.php @@ -143,8 +143,8 @@ function search_content(&$a) { INNER JOIN `item` ON `item`.`id`=`term`.`oid` INNER JOIN `contact` ON `contact`.`id` = `item`.`contact-id` AND NOT `contact`.`blocked` AND NOT `contact`.`pending` WHERE `item`.`visible` AND NOT `item`.`deleted` AND NOT `item`.`moderated` - AND `term`.`uid` IN (%d,0) AND `term`.`otype` = %d AND `term`.`type` = %d AND `term`.`term` = '%s' AND `term`.`guid` != '' - GROUP BY `term`.`guid` ORDER BY term.created DESC LIMIT %d , %d ", + AND (`term`.`uid` = 0 OR (`term`.`uid` = %d AND NOT `term`.`global`)) AND `term`.`otype` = %d AND `term`.`type` = %d AND `term`.`term` = '%s' + ORDER BY term.created DESC LIMIT %d , %d ", intval(local_user()), intval(TERM_OBJ_POST), intval(TERM_HASHTAG), dbesc(protect_sprintf($search)), intval($a->pager['start']), intval($a->pager['itemspage'])); } else {