Merge branch 'release/3.4'

.gitignore

@@ -34,7 +34,11 @@ report/
 #ignore cache folders
 /privacy_image_cache/
 /photo/
+/proxy/
 nbproject
 
 #ignore vagrant dir
 .vagrant/
+
+#ignore local folder
+/local/

CHANGELOG

@@ -1,3 +1,65 @@
+Version 3.4
+
+	Optionally, "like" and "dislike" activities don't update thread timestamp (annando)
+	Updated markdown libraries (annando)
+	Updated jQuery (StefOfficiel)
+	Cache zrl verification requests to prevent DSoS (issue #1453) (annando)
+	"Verify SSL" options affects also VERIFYHOST (annando)
+	Better handling of hashtags (annando)
+	Updated translations (translation teams, tobias)
+	Access a contact directly from the contact-manager-page (FlxAlbroscheit)
+	Reworked GUID generation, remove db store (annando)
+	Improve search for tags and terms (annando)
+	Fix OAuth signature (thorsten23)
+	Fix utf8 characters in items (issue #1307) (hauke)
+	Ignore tag-likes char sequences in code blocks (issue #1041) (fabrixxm)
+	Fix sending email to CC recipients (issue #1437) (fabrixxm)
+	Fix signature check of likes from diaspora (issue #905) (mike, annando)
+	Fix pagination urls (issue #1341) (fabrixxm)
+	Add scheme if missing in "web link" dialog (issue #1362) (fabrixxm)
+	Don't detect Facebook and App.net RSS feeds as contacts (issue #1432) (annando)
+	Add cli command to generate database.sql from scheme description (issue #1370) (fabrixxm)
+	Fix warning trying to creating already existing itemcache dir (pztrn)
+	Send update to directory when account is removed (issue #1038) (annando)
+	Fix settings page's aside menu visibility (issue #1459) (fabrixxm)
+	Don't show past events in event reminder in profile page (issue #1306) (annando)
+	Add help text to explain the options for approving contacts (issue #1349) (silke)
+	API set as unseen only posts returned by the call (issue #1063) (annando)
+
+Version 3.3.3
+
+	More separation between php and html in photo album (issue #1258) (rabuzarus)
+	Enhanced community page shows public posts from public contacts of public profiles (annando)
+	Support for IndieAuth/Web-sign-in (hauke)
+	New hooks "emailer_send_prepare" and "emailer_send" (fabrixxm)
+	New hook "oembed_fetch_url" (annando)
+	Add un/ignore function to quattro theme (tobiasd)
+	Enhanced POCO data (annando)
+	Use HTML5 features to validate inputs in install wizard and in some settings fields (tobiasd)
+	Option to receive text-only notification emails (fabrixxm)
+	Better OStatus support (annando)
+	Share-it button support (annando)
+	More reliable reshare from Diaspora (annando)
+	Load more images via proxy (annando)
+	util/typo.php uses "php -l" insead of "eval()" to validate code (fabrixxm)
+	Use $_SERVER array in cli script instead of $argv/$argc (issue #1218) (annando)
+	Updated vagrant setup script (silke)
+	API: support to star/unstar items (fabrixxm)
+	API: attachments for better AndStatus support (annando)
+	Fix missing spaces in photo URLs (issue #920) (annando)
+	Fix avatar for "remote-self" items (annando)
+	Fix encodings issues with scrape functionality (annando)
+	Fix site info scraping when URL points to big file (annando)
+	Fix tools for translations (ddorian1)
+	Fix API login via LDAP (issue #1286) (fabrixxm)
+	Fix to link URL in tabs, pager (issues #1341, #1190) (ddorian1)
+	Fix poke activities translation (fabrixxm)
+	Fix html escaping in templates (fabrixxm)
+	Fix Friendica contacts shown as Diaspora contacts via Poco (annando)
+	Fix shared contacts wrong linking (issue #1388) (annando)
+	Fix email validation (ddorian1)
+	Better documentation for developers (silke)
+
 Version 3.3.2
 
 	Set default value for all not-null fields (fixes SQL warinigs) (annando)
@@ -33,17 +95,17 @@ Version 3.3
 	API
 	added support in the API to allow image uploads from Twidere
 	support for the diaspora app in Firefox
-	
+
-	Themes 
+	Themes
 	Stopped support of unmaintained themes. They will continue to work if enabled but are no longer displayed in the list of themes.
-	Merged all "zero" themes into a theme with variations. 
+	Merged all "zero" themes into a theme with variations.
 	new default avatar by Andi Stadler
-	
+
 	Usability
 	network page as default page after login
 	sections on users' settings page are now collapsable
 	automatic updating the network stream was improved
-	
+
 	Interaction
 	ignoring of threads
 	for selected contects one can now get notifications when they post something, useful e.g. for forums
@@ -51,8 +113,8 @@ Version 3.3
 	many improvement on all connectors, new app.net connector
 	the algorithm for shortening postings when posting to limited platforms was improved
 	improvements for the remote_self functionality for RSS/Atom feeds were done
-	
+
-	System stuff 
+	System stuff
 	no more apc support due problems with PHP 5.5
 	privacy image cache moved from an addon into the core
 	updated the following libraries: smarty 3.1.19, fullcalendar 1.6.4, jquery 1.11, jgrowl 1.3.0
@@ -65,8 +127,8 @@ Version 3.3
 	some bugs were fixed for the profile import function
 	BBCode handling and reformatting to e.g. markdown was improved
 	Internal PusH server for communication with OStatus contacts
-	
+
-	Addons 
+	Addons
 	translation now done at transifex as well
 	"newmemberwidget" adds widget with help links + welcome message to sidebar of network tab for new members
 	new statistics addon to take part in the Diaspora* survey
@@ -75,10 +137,10 @@ Version 3.3
 	new connector for the buffer service
 	improvements for the connectors with Twitter, StatusNet/GNU Social, pump.io, google+ and facebook
 	improvements to the cal and jappix-mini addons
-	
+
 	Change in the structure of the git repo
 	The "master" branch will now contain stable stuff and hotfixes.
-	The new "develop" branch will contain the latest changes. 
+	The new "develop" branch will contain the latest changes.
 
 Version 3.2
 

boot.php

@@ -15,10 +15,10 @@ require_once('update.php');
 require_once('include/dbstructure.php');
 
 define ( 'FRIENDICA_PLATFORM',     'Friendica');
-define ( 'FRIENDICA_CODENAME',     'Ginger');
+define ( 'FRIENDICA_CODENAME',     'Lily of the valley');
-define ( 'FRIENDICA_VERSION',      '3.3.2' );
+define ( 'FRIENDICA_VERSION',      '3.4.0' );
 define ( 'DFRN_PROTOCOL_VERSION',  '2.23'    );
-define ( 'DB_UPDATE_VERSION',      1175      );
+define ( 'DB_UPDATE_VERSION',      1182      );
 define ( 'EOL',                    "<br />\r\n"     );
 define ( 'ATOM_TIME',              'Y-m-d\TH:i:s\Z' );
 
@@ -127,6 +127,11 @@ define ( 'PAGE_FREELOVE',          3 );
 define ( 'PAGE_BLOG',              4 );
 define ( 'PAGE_PRVGROUP',          5 );
 
+// Type of the community page
+define ( 'CP_NO_COMMUNITY_PAGE',   -1 );
+define ( 'CP_USERS_ON_SERVER',     0 );
+define ( 'CP_GLOBAL_COMMUNITY',    1 );
+
 /**
  * Network and protocol family types
  */
@@ -435,7 +440,7 @@ if(! class_exists('App')) {
 
 		function __construct() {
 
-			global $default_timezone, $argv, $argc;
+			global $default_timezone;
 
 			$hostname = "";
 
@@ -504,9 +509,9 @@ if(! class_exists('App')) {
 			if ($hostname != "")
 				$this->hostname = $hostname;
 
-			if (is_array($argv) && $argc>1 && substr(end($argv), 0, 4)=="http" ) {
+			if (is_array($_SERVER["argv"]) && $_SERVER["argc"]>1 && substr(end($_SERVER["argv"]), 0, 4)=="http" ) {
-				$this->set_baseurl(array_pop($argv) );
+				$this->set_baseurl(array_pop($_SERVER["argv"]) );
-				$argc --;
+				$_SERVER["argc"] --;
 			}
 
 			#set_include_path("include/$this->hostname" . PATH_SEPARATOR . get_include_path());
@@ -1191,36 +1196,25 @@ if(! function_exists('check_plugins')) {
 	}
 }
 
-function get_guid($size=16) {
+function get_guid($size=16, $prefix = "") {
-	$exists = true; // assume by default that we don't have a unique guid
-	do {
-		$prefix = "";
-		while (strlen($prefix) < ($size - 13))
-			$prefix .= mt_rand();
 
-		$s = substr(uniqid($prefix), -$size);
+	if ($prefix == "") {
+		$a = get_app();
+		$prefix = hash("crc32", $a->get_hostname());
+	}
 
-		$r = q("select id from guid where guid = '%s' limit 1", dbesc($s));
+	while (strlen($prefix) < ($size - 13))
-		if(! count($r))
+		$prefix .= mt_rand();
-			$exists = false;
+
-	} while($exists);
+	if ($size >= 24) {
-	q("insert into guid (guid) values ('%s') ", dbesc($s));
+		$prefix = substr($prefix, 0, $size - 22);
-	return $s;
+		return(str_replace(".", "", uniqid($prefix, true)));
+	} else {
+		$prefix = substr($prefix, 0, $size - 13);
+		return(uniqid($prefix));
+	}
 }
 
-/*function get_guid($size=16) {
-	$exists = true; // assume by default that we don't have a unique guid
-	do {
-		$s = random_string($size);
-		$r = q("select id from guid where guid = '%s' limit 1", dbesc($s));
-		if(! count($r))
-			$exists = false;
-	} while($exists);
-	q("insert into guid ( guid ) values ( '%s' ) ", dbesc($s));
-	return $s;
-}*/
-
-
 // wrapper for adding a login box. If $register == true provide a registration
 // link. This will most always depend on the value of $a->config['register_policy'].
 // returns the complete html for inserting into the page
@@ -1405,11 +1399,11 @@ if(! function_exists('get_max_import_size')) {
 if(! function_exists('profile_load')) {
 	function profile_load(&$a, $nickname, $profile = 0, $profiledata = array()) {
 
-		$user = q("select uid from user where nickname = '%s' limit 1",
+		$user = q("SELECT `uid` FROM `user` WHERE `nickname` = '%s' LIMIT 1",
 			dbesc($nickname)
 		);
 
-		if(! ($user && count($user))) {
+		if(!$user && count($user) && !count($profiledata)) {
 			logger('profile error: ' . $a->query_string, LOGGER_DEBUG);
 			notice( t('Requested account is not available.') . EOL );
 			$a->error = 404;
@@ -1440,7 +1434,7 @@ if(! function_exists('profile_load')) {
 					intval($profile_int)
 			);
 		}
-		if((! $r) && (!  count($r))) {
+		if((!$r) && (!count($r))) {
 			$r = q("SELECT `profile`.`uid` AS `profile_uid`, `profile`.* , `contact`.`avatar-date` AS picdate, `user`.* FROM `profile`
 					INNER JOIN `contact` on `contact`.`uid` = `profile`.`uid` INNER JOIN `user` ON `profile`.`uid` = `user`.`uid`
 					WHERE `user`.`nickname` = '%s' AND `profile`.`is-default` = 1 and `contact`.`self` = 1 LIMIT 1",
@@ -1448,7 +1442,7 @@ if(! function_exists('profile_load')) {
 			);
 		}
 
-		if(($r === false) || (! count($r))) {
+		if(($r === false) || (!count($r)) && !count($profiledata)) {
 			logger('profile error: ' . $a->query_string, LOGGER_DEBUG);
 			notice( t('Requested profile is not available.') . EOL );
 			$a->error = 404;
@@ -1457,7 +1451,7 @@ if(! function_exists('profile_load')) {
 
 		// fetch user tags if this isn't the default profile
 
-		if(! $r[0]['is-default']) {
+		if(!$r[0]['is-default']) {
 			$x = q("select `pub_keywords` from `profile` where uid = %d and `is-default` = 1 limit 1",
 					intval($r[0]['profile_uid'])
 			);
@@ -1649,8 +1643,10 @@ if(! function_exists('profile_sidebar')) {
 
 		$homepage = ((x($profile,'homepage') == 1) ?  t('Homepage:') : False);
 
+		$about = ((x($profile,'about') == 1) ?  t('About:') : False);
+
 		if(($profile['hidewall'] || $block) && (! local_user()) && (! remote_user())) {
-			$location = $pdesc = $gender = $marital = $homepage = False;
+			$location = $pdesc = $gender = $marital = $homepage = $about = False;
 		}
 
 		$firstname = ((strpos($profile['name'],' '))
@@ -1671,8 +1667,24 @@ if(! function_exists('profile_sidebar')) {
 
 		if (!$block){
 			$contact_block = contact_block();
-		}
 
+			if(is_array($a->profile) AND !$a->profile['hide-friends']) {
+				$r = q("SELECT `gcontact`.`updated` FROM `contact` INNER JOIN `gcontact` WHERE `gcontact`.`nurl` = `contact`.`nurl` AND `self` AND `uid` = %d LIMIT 1",
+					intval($a->profile['uid']));
+				if(count($r))
+					$updated =  date("c", strtotime($r[0]['updated']));
+
+				$r = q("SELECT COUNT(*) AS `total` FROM `contact` WHERE `uid` = %d AND `self` = 0 AND `blocked` = 0 and `pending` = 0 AND `hidden` = 0 AND `archive` = 0
+						AND `network` IN ('%s', '%s', '%s', '')",
+					intval($profile['uid']),
+					dbesc(NETWORK_DFRN),
+					dbesc(NETWORK_DIASPORA),
+					dbesc(NETWORK_OSTATUS)
+				);
+				if(count($r))
+					$contacts = intval($r[0]['total']);
+			}
+		}
 
 		$p = array();
 		foreach($profile as $k => $v) {
@@ -1695,7 +1707,10 @@ if(! function_exists('profile_sidebar')) {
 			'$pdesc'	=> $pdesc,
 			'$marital'  => $marital,
 			'$homepage' => $homepage,
+			'$about' => $about,
 			'$network' =>  t('Network:'),
+			'$contacts' => $contacts,
+			'$updated' => $updated,
 			'$diaspora' => $diaspora,
 			'$contact_block' => $contact_block,
 		));
@@ -1815,10 +1830,10 @@ if(! function_exists('get_events')) {
 		$bd_short = t('F d');
 
 		$r = q("SELECT `event`.* FROM `event`
-				WHERE `event`.`uid` = %d AND `type` != 'birthday' AND `start` < '%s' AND `start` > '%s'
+				WHERE `event`.`uid` = %d AND `type` != 'birthday' AND `start` < '%s' AND `start` >= '%s'
 				ORDER BY `start` ASC ",
 				intval(local_user()),
-				dbesc(datetime_convert('UTC','UTC','now + 6 days')),
+				dbesc(datetime_convert('UTC','UTC','now + 7 days')),
 				dbesc(datetime_convert('UTC','UTC','now - 1 days'))
 		);
 
@@ -1835,6 +1850,7 @@ if(! function_exists('get_events')) {
 			}
 			$classtoday = (($istoday) ? 'event-today' : '');
 
+			$skip = 0;
 
 			foreach($r as &$rr) {
 				if($rr['adjust'])
@@ -1848,6 +1864,12 @@ if(! function_exists('get_events')) {
 					$title = t('[No description]');
 
 				$strt = datetime_convert('UTC',$rr['convert'] ? $a->timezone : 'UTC',$rr['start']);
+
+				if(substr($strt,0,10) < datetime_convert('UTC',$a->timezone,'now','Y-m-d')) {
+					$skip++;
+					continue;
+				}
+
 				$today = ((substr($strt,0,10) === datetime_convert('UTC',$a->timezone,'now','Y-m-d')) ? true : false);
 
 				$rr['link'] = $md;
@@ -1862,7 +1884,7 @@ if(! function_exists('get_events')) {
 		return replace_macros($tpl, array(
 			'$baseurl' => $a->get_baseurl(),
 			'$classtoday' => $classtoday,
-			'$count' => count($r),
+			'$count' => count($r) - $skip,
 			'$event_reminders' => t('Event Reminders'),
 			'$event_title' => t('Events this week:'),
 			'$events' => $r,
@@ -2071,7 +2093,7 @@ if(! function_exists('load_contact_links')) {
 		if(! $uid || x($a->contacts,'empty'))
 			return;
 
-		$r = q("SELECT `id`,`network`,`url`,`thumb` FROM `contact` WHERE `uid` = %d AND `self` = 0 AND `blocked` = 0 ",
+		$r = q("SELECT `id`,`network`,`url`,`thumb` FROM `contact` WHERE `uid` = %d AND `self` = 0 AND `blocked` = 0 AND `thumb` != ''",
 				intval($uid)
 		);
 		if(count($r)) {
@@ -2166,6 +2188,20 @@ function get_my_url() {
 function zrl_init(&$a) {
 	$tmp_str = get_my_url();
 	if(validate_url($tmp_str)) {
+
+		// Is it a DDoS attempt?
+		// The check fetches the cached value from gprobe to reduce the load for this system
+		$urlparts = parse_url($tmp_str);
+
+		$result = Cache::get("gprobe:".$urlparts["host"]);
+		if (!is_null($result)) {
+			$result = unserialize($result);
+			if ($result["network"] == NETWORK_FEED) {
+				logger("DDoS attempt detected for ".$urlparts["host"]." by ".$_SERVER["REMOTE_ADDR"].". server data: ".print_r($_SERVER, true), LOGGER_DEBUG);
+				return;
+			}
+		}
+
 		proc_run('php','include/gprobe.php',bin2hex($tmp_str));
 		$arr = array('zrl' => $tmp_str, 'url' => $a->cmd);
 		call_hooks('zrl_init',$arr);
@@ -2332,7 +2368,9 @@ function get_itemcachepath() {
 
 	if ($temppath != "") {
 		$itemcache = $temppath."/itemcache";
-		mkdir($itemcache);
+		if(!file_exists($itemcache) && !is_dir($itemcache)) {
+			mkdir($itemcache);
+		}
 
 		if (is_dir($itemcache) AND is_writable($itemcache)) {
 			set_config("system", "itemcache", $itemcache);

doc/Account-Basics.md

@@ -59,7 +59,7 @@ After your first login, please visit the 'Settings' page from the top menu bar a
 
 **Getting Started**
 
-A ['Tips for New Members'](newmember) link will show up on your home page for two weeks to provide some important Getting Started information.
+A ['Tips for New Members'](newmember) link will show up on your network and home pages for two weeks to provide some important Getting Started information.
 
 
 **Retrieving Personal Data**

doc/Bugs-and-Issues.md

@@ -11,21 +11,3 @@ If you're a technical user, or your site doesn't have a support page, you'll nee
 Try to provide as much information as you can about the bug, including the **full** text of any error messages or notices, and any steps required to replicate the problem in as much detail as possible.  It's generally better to provide too much information than not enough.
 
 <a href="http://www.chiark.greenend.org.uk/~sgtatham/bugs.html">See this article</a> to learn more about submitting **good** bug reports.
-
-
-**Bug Sponsorship**
-
-If you find a bug, and it is caused by a problem in main branch (ie, is not specific to our site), you may sponsor it.
-
-The bug/issue database allows you to sponsor issues. This provides an incentive for developers to work on your issue. This isn't necessary - we don't like bugs and will try to fix them. This has more importance for future development projects and feature requests.
-
-Bug sponsorship works on the honour system. If you agree to pay $10 to fix a bug, when the fix has been checked in and verified you should send a paypal payment to the developer assigned to the bug. Don't ever think you can get away with not paying a developer for work performed. Some of these guys could hack into your credit card account if you make them mad.
-
-At the present time, one has to be approved as a "developer" to be able to assign themselves to a sponsored bug. This requires the developer to have some history fixing Friendica bugs. This is for everybody's assurance that the bug fix will work well with Friendica. If you wish to become approved as a developer, work on and check in some non-sponsored issues or your own projects and we will move you up the ladder.
-
-If you truly feel you have the solution to a sponsored bug but aren't an approved developer, you risk a sponsored developer assigning the bug to themselves before you check it in, but if they haven't done so - include a short note with your pull request. Assuming that it meets our code standards, we'll see that you get credit. 
-
-If you sponsor a project at greater than a $50 level, you may be requested by the developer for payment up front before work has begun (typically half). Again this is on the honour system - and is mostly to avoid payment issues and disagreements later. You should also expect to see some progress updates or demonstrations if the work takes more than a week or two. If the work is not completed within a reasonable time (as decided by those involved), you are entitled to get your money back. 
-
-Friendica is not involved in these transactions. It is purely a personal agreement between sponsors and developers. If there are any issues, the parties will need to work it out between themselves. We're just providing some guidelines to help avoid potential problems.     
-

doc/Developers-Intro.md

@@ -0,0 +1,49 @@
+Where to get started to help improve Friendica?
+===============================================
+
+* [Home](help)
+
+Do you want to help us improve Friendica? Here we have compiled some hints on how to get started and some tasks to help you choose. A project like Friendica is the sum of many different contributions. **Very different skills are required to make good software. Some of them involve coding, others do not.** We are looking for helpers in all areas, whether you write text or code, whether you spread the word to convince people or design new icons. Whether you feel like an expert or like a newbie - join us with your ideas!
+
+**Contact us**
+
+The discussion of Friendica development takes place in the following Friendica forums:
+
+* The main [forum for Friendica development](https://friendika.openmindspace.org/profile/friendicadevelopers)
+* The [forum for Friendica theme development](https://friendica.eu/profile/ftdevs)
+
+**Help other users**
+
+Remember the questions you had when you first tried Friendica? A good place to start can be to help new people find their way around Friendica in the [general support forum](https://helpers.pyxis.uberspace.de/profile/helpers). Welcome them, answer their questions, point them to documentation or ping other helpers directly if you can't help but think you know who can.
+
+**Translations**
+
+The documentation contains help on how to translate Friendica in the [at Transifex](/help/translations) where the UI is translated.
+If you don't want to translate the UI, or it is already done to your satisfaction, you might want to work on the translation of the /help files?
+
+**Design**
+
+Are you good at designing things? If you have seen Friendica you probably have ideas to improve it, haven't you?
+
+* If you would like to work with us on enhancing the user interface, please join the [UX Watchdogs forum](https://fc.oscp.info/profile/ux-watchdogs)
+* Make plans for a better Friendica interface design and share them with us.
+* Tell us if you are able to realize your ideas or what kind of help you need. We can't promise we have the right skills in the group but we'll try.
+* Choose a thing to start with, e.g. work on the icon set of your favourite theme
+
+**Programming**
+
+* **Issues:** Have a look at our [issue tracker](https://github.com/friendica/friendica) on gihub!
+	* Try to reproduce a bug that needs more inquries and write down what you find out.
+	* If a bug looks fixed, ask the bug reporters for feedback to find out if the bug can be closed.
+	* Fix a bug if you can. Please make the pull request against the *develop* branch of the repository.
+
+* **Web interface:** The thing many people want most is a better interface, preferably a responsive Friendica theme. This is a piece of work! If you want to get involved here:
+	* Look at the first steps that were made (e.g. the clean theme). Ask us to find out whom to talk to about their experiences.
+	* Talk to design people if you know any.
+	* Let us know about your plans [in the dev forum](https://friendika.openmindspace.org/profile/friendicadevelopers) and the [theme developer forum](https://friendica.eu/profile/ftdevs). Do not worry about cross-posting.
+
+* **Client software:** There are free software clients that do somehow work with Friendica but most of them need love and maintenance. Also, they were mostly made for other platforms using the StatusNet API. This means they lack:w
+ the features that are really specific to Friendica. Popular clients you might want to have a look at are:
+	* [Hotot (Linux)](http://hotot.org/) - abandoned
+	* [Friendica for Android](https://github.com/max-weller/friendica-for-android) - abandoned
+	* You can find more working client software in [Wikipedia](https://en.wikipedia.org/wiki/Friendica).

doc/Developers.md

@@ -1,23 +0,0 @@
-Friendica Developer Guide
-===================
-
-**Here is how you can join us.**
-
-First, get yourself a working git package on the system where you will be
-doing development.
-
-Create your own github account.
-
-You may fork/clone the Friendica repository from [https://github.com/friendica/friendica.git](https://github.com/friendica/friendica.git).
-
-Follow the instructions provided here: [http://help.github.com/fork-a-repo/](http://help.github.com/fork-a-repo/)
-to create and use your own tracking fork on github
-
-Then go to your github page and create a "Pull request" when you are ready
-to notify us to merge your work.
-
-**Important**
-
-Please pull in any changes from the project repository and merge them with your work **before** issuing a pull request. We reserve the right to reject any patch which results in a large number of merge conflicts. This is especially true in the case of language translations - where we may not be able to understand the subtle differences between conflicting versions.
-
-Also - **test your changes**. Don't assume that a simple fix won't break something else. If possible get an experienced Friendica developer to review the code. 

doc/Github.md

@@ -0,0 +1,28 @@
+Friendica on Github
+===================
+
+* [Home](help)
+
+**Here is how you can work on the code with us**
+
+1. Install git on the system you will be developing on.
+2. Create your own [github](https://github.com) account.
+3. Fork the Friendica repository from [https://github.com/friendica/friendica.git](https://github.com/friendica/friendica.git).
+4. Clone your fork from your Github account to your machine. Follow the instructions provided here: [http://help.github.com/fork-a-repo/](http://help.github.com/fork-a-repo/) to create and use your own tracking fork on github
+5. Commit your changes to your fork. Then go to your github page and create a "Pull request" to notify us to merge your work.
+
+**Branches**
+
+There are two branches in the main repo on Github:
+
+1. master: This branch contains stable releases only.
+2. develop: This branch contains the latest code. This is what you want to work with.
+
+**Important**
+
+Please pull in any changes from the project repository and merge them with your work **before** issuing a pull request. We reserve the right to reject any patch which results in a large number of merge conflicts. This is especially true in the case of language translations - where we may not be able to understand the subtle differences between conflicting versions.
+
+Also - **test your changes**. Don't assume that a simple fix won't break something else. If possible get an experienced Friendica developer to review the code. 
+
+Check out how to work with [our Vagrant](help/Vagrant) to save a lot of setup time!
+

doc/Home.md

@@ -1,7 +1,7 @@
 Friendica Documentation and Resources
 =====================================
 
-**Contents**
+**User Manual**
 
 * General functions - first steps
 	* [Account Basics](help/Account-Basics)
@@ -10,7 +10,7 @@ Friendica Documentation and Resources
         * [BBCode tag reference](help/BBCode)
 	* [Comment, sort and delete posts](help/Text_comment)
 	* [Profiles](help/Profiles)
-* You and other user
+* You and other users
 	* [Connectors](help/Connectors)
 	* [Making Friends](help/Making-Friends)
 	* [Groups and Privacy](help/Groups-and-Privacy)
@@ -19,12 +19,11 @@ Friendica Documentation and Resources
 	* [Chats](help/Chats)
 * Further information
 	* [Improve Performance](help/Improve-Performance)
-	* [Move Account](help/Move-Account)
+	* [Move your account](help/Move-Account)
-	* [Remove Account](help/Remove-Account)
+	* [Delete your account](help/Remove-Account)
-	* [Bugs and Issues](help/Bugs-and-Issues)
 	* [Frequently asked questions (FAQ)](help/FAQ)
 
-**Technical Documentation**
+**Admin Manual**
 
 * [Install](help/Install)
 * [Settings](help/Settings)
@@ -32,10 +31,16 @@ Friendica Documentation and Resources
 * [Installing Connectors (Facebook/Twitter/StatusNet)](help/Installing-Connectors)
 * [Message Flow](help/Message-Flow)
 * [Using SSL with Friendica](help/SSL)
-* [Developers](help/Developers)
 * [Twitter/StatusNet API Functions](help/api)
-* [Translation of Friendica](help/translations)
 
+**Developer Manual**
+
+* [Where to get started?](help/Developers-Intro)
+* [Help on Github](help/Github)
+* [Help on Vagrant](help/Vagrant)
+* [How to translate Friendica](help/translations)
+* [Bugs and Issues](help/Bugs-and-Issues)
+* [Smarty 3 Templates](help/smarty3-templates)
 
 **External Resources**
 

doc/Plugins.md

@@ -196,6 +196,24 @@ Current hooks:
         'email' => email to look up the avatar for
         'url' => the (string) generated URL of the avatar
 
+**'emailer_send_prepare'** - called from Emailer::send() before building the mime message
+    $b is (array) , params to Emailer::send()
+     'fromName' => name of the sender
+     'fromEmail' => email fo the sender
+     'replyTo' => replyTo address to direct responses
+     'toEmail' => destination email address
+     'messageSubject' => subject of the message
+     'htmlVersion' => html version of the message
+     'textVersion' => text only version of the message
+     'additionalMailHeader' => additions to the smtp mail header
+
+**'emailer_send'** - called before calling PHP's mail()
+    $b is (array) , params to mail()
+     'to'
+     'subject'
+     'body'
+     'headers'
+
 
 A complete list of all hook callbacks with file locations (generated 14-Feb-2012): Please see the source for details of any hooks not documented above.
 

doc/Vagrant.md

@@ -0,0 +1,39 @@
+Vagrant for Friendica Developers
+===================
+
+* [Home](help)
+
+**Getting started**
+
+[Vagrant](https://www.vagrantup.com/) is a virtualization solution for developers. No need to setup up a webserver, database etc. before actually starting. Vagrant creates a virtual machine (an Ubuntu 12.04) for you that you can just run inside VirtualBox and start to work directly on Friendica. What you need to do:
+
+1. Install VirtualBox and vagrant.
+2. Git clone your Friendica repository. Inside, you'll find a "Vagrantfile" and some scripts in the utils folder.
+3. Run "vagrant up" from inside the friendica clone. Be patient: When it runs for the first time, it downloads an Ubuntu Server image.
+4. Run "vagrant ssh" to log into the virtual machine to log in to the VM.
+5. Open 192.168.22.10 in a browser to finish the Friendica installation. The mysql database is called "friendica", the mysql user and password both are "root".
+6. Work on Friendica's code in your git clone on your machine (not in the VM).
+7. Check the changes in your browser in the VM. Debug via the "vagrant ssh" login.
+8. Commit and push your changes directly back to Github.
+
+If you want to stop vagrant after finishing your work, run the following command
+
+		$> vagrant halt
+
+in the development directory.
+
+**Import test data**
+
+If you want some test data in your vagrant Friendica instance import the database dump friendica_test_data.sql like so (inside the VM):
+
+		$> mysql -u root -p friendica < /vagrant/friendica_test_data.sql
+
+You will then have the following accounts to login:
+
+  * admin, password admin
+  * friendica1, password friendica
+  * friendica2, password friendica2 and so on until friendica5
+  * friendica1 is connected to all others. friendica1 has two groups: group1 with friendica2 and friendica4, group2 with friendica3 and friendica5.
+  * friendica2 and friendica3 are conntected. friendica4 and friendica5 are connected. 
+
+For further documentation of vagrant, please see [the vagrant*docs*](https://docs.vagrantup.com/v2/).

doc/snarty3-templates.md

@@ -0,0 +1,173 @@
+Friendica Templating Documentation
+==================================
+
+* [Home](help)
+
+Friendica uses [Smarty 3](http://www.smarty.net/) as PHP templating engine. The main templates are found in
+
+		/view/templates
+
+theme authors may overwrite the default templates by putting a files with the same name into the
+
+		/view/themes/$themename/templates
+
+directory.
+
+Templates that are only used by addons shall be placed in the
+
+		/addon/$addonname/templates
+
+directory.
+
+To render a template use the function *get_markup_template* to load the template and *replace_macros* to replace the macros/variables in the just loaded template file.
+
+		$tpl = get_markup_template('install_settings.tpl');
+        $o .= replace_macros($tpl, array( ... ));
+
+the array consists of an association of an identifier and the value for that identifier, i.e.
+
+		'$title' => $install_title,
+
+where the value may as well be an array by its own.
+
+Form Templates
+--------------
+
+To guarantee a consistent look and feel for input forms, i.e. in the settings sections, there are templates for the basic form fields. They are initialized with an array of data, depending on the tyle of the field.
+
+All of these take an array for holding the values, i.e. for an one line text input field, which is required and should be used to type email addesses use something along
+
+		'$adminmail' => array('adminmail', t('Site administrator email address'), $adminmail, t('Your account email address must match this in order to use the web admin panel.'), 'required', '', 'email'),
+
+To evaluate the input value, you can then use the $_POST array, more precisely the $_POST['adminemail'] variable.
+
+Listed below are the template file names, the general purpose of the template and their field parameters.
+
+### field_checkbox.tpl
+
+A checkbox. If the checkbox is checked its value is **1**. Field parameter:
+
+0. Name of the checkbox,
+1. Label for the checkbox,
+2. State checked? if true then the checkbox will be marked as checked,
+3. Help text for the checkbox.
+
+### field_combobox.tpl
+
+A combobox, combining a pull down selection and a textual input field. Field parameter:
+
+0. Name of the combobox,
+1. Label for the combobox,
+2. Current value of the variable,
+3. Help text for the combobox,
+4. Array holding the possible values for the textual input,
+5. Array holding the possible values for the pull down selection.
+
+### field_custom.tpl
+
+A customizeable template to include a custom element in the form with the usual surroundings, Field parameter:
+
+0. Name of the field,
+1. Label for the field,
+2. the field,
+3. Help text for the field.
+
+### field_input.tpl
+
+A single line input field for textual input. Field parameter:
+
+0. Name of the field,
+1. Label for the input box,
+2. Current value of the variable,
+3. Help text for the input box,
+4. if set to "required" modern browser will check that this input box is filled when submitting the form,
+5. if set to "autofocus" modern browser will put the cursur into this box once the page is loaded,
+6. if set to "email" or "url" modern browser will check that the filled in value corresponds to an email address or URL.
+
+### field_intcheckbox.tpl
+
+A checkbox (see above) but you can define the value of it. Field parameter:
+
+0. Name of the checkbox,
+1. Label for the checkbox,
+2. State checked? if true then the checkbox will be marked as checked,
+3. Value of the checkbox,
+4. Help text for the checkbox.
+
+### field_openid.tpl
+
+An input box (see above) but prepared for special CSS styling for openID input. Field parameter:
+
+0. Name of the field,
+1. Label for the input box,
+2. Current value of the variable,
+3. Help text for the input field.
+
+### field_password.tpl
+
+A single line input field (see above) for textual input. The characters typed in will not be shown by the browser. Field parameter:
+
+0. Name of the field,
+1. Label for the field,
+2. Value for the field, e.g. the old password,
+3. Help text for the input field,
+4. if set to "required" modern browser will check that this field is filled out,
+5. if set to "autofocus" modern browser will put the cursor automatically into this input field.
+
+### field_radio.tpl
+
+A radio button. Field parameter:
+
+0. Name of the radio button,
+1. Label for the radio button,
+2. Current value of the variable,
+3. Help text for the button,
+4. if set, the radio button will be checked.
+
+### field_richtext.tpl
+
+A multi-line input field for *rich* textual content. Field parameter:
+
+0. Name of the input field,
+1. Label for the input box,
+2. Current text for the box,
+3. Help text for the input box.
+
+### field_select.tpl
+
+A drop down selection box. Field parameter:
+
+0. Name of the field,
+1. Label of the selection box,
+2. Current selected value,
+3. Help text for the selection box,
+4. Array holding the possible values of the selection drop down.
+
+### field_select_raw.tpl
+
+A drop down selection box (see above) but you have to prepare the values yourself. Field parameter:
+
+0. Name of the field,
+1. Label of the selection box,
+2. Current selected value,
+3. Help text for the selection box,
+4. Possible values of the selection drop down.
+
+### field_textarea.tpl
+
+A multi-line input field for (plain) textual content. Field parameter:
+
+0. Name of the input field,
+1. Label for the input box,
+2. Current text for the box,
+3. Help text for the input box.
+
+### field_yesno.tpl
+
+A button that has two states *yes* or *no*. Field parameter:
+
+0. Name of the input field,
+1. Label for the button,
+2. Current value,
+3. Help text for the button
+4. if set to an array of two values, these two will be used, otherwise "off" and "on".

include/Contact.php

@@ -15,7 +15,7 @@ function user_remove($uid) {
 
 	call_hooks('remove_user',$r[0]);
 
-	// save username (actually the nickname as it is guaranteed 
+	// save username (actually the nickname as it is guaranteed
 	// unique), so it cannot be re-registered in the future.
 
 	q("insert into userd ( username ) values ( '%s' )",
@@ -46,6 +46,10 @@ function user_remove($uid) {
 	// q("DELETE FROM `user` WHERE `uid` = %d", intval($uid));
 	q("UPDATE `user` SET `account_removed` = 1, `account_expires_on` = UTC_TIMESTAMP() WHERE `uid` = %d", intval($uid));
 	proc_run('php', "include/notifier.php", "removeme", $uid);
+
+	// Send an update to the directory
+	proc_run('php', "include/directory.php", $r[0]['url']);
+
 	if($uid == local_user()) {
 		unset($_SESSION['authenticated']);
 		unset($_SESSION['uid']);
@@ -191,7 +195,7 @@ if(! function_exists('contact_photo_menu')){
 function contact_photo_menu($contact) {
 
 	$a = get_app();
-	
+
 	$contact_url="";
 	$pm_url="";
 	$status_link="";
@@ -222,24 +226,24 @@ function contact_photo_menu($contact) {
 	$contact_url = $a->get_baseurl() . '/contacts/' . $contact['id'];
 	$posts_link = $a->get_baseurl() . '/network/0?nets=all&cid=' . $contact['id'];
 	$contact_drop_link = $a->get_baseurl() . "/contacts/" . $contact['id'] . '/drop?confirm=1';
-	
+
 
 	$menu = Array(
 		'poke' => array(t("Poke"), $poke_link),
 		'status' => array(t("View Status"), $status_link),
 		'profile' => array(t("View Profile"), $profile_link),
-		'photos' => array(t("View Photos"), $photos_link),		
+		'photos' => array(t("View Photos"), $photos_link),
-		'network' => array(t("Network Posts"), $posts_link), 
+		'network' => array(t("Network Posts"), $posts_link),
 		'edit' => array(t("Edit Contact"), $contact_url),
 		'drop' => array(t("Drop Contact"), $contact_drop_link),
 		'pm' => array(t("Send PM"), $pm_url),
 	);
-	
+
-	
+
 	$args = array('contact' => $contact, 'menu' => &$menu);
-	
+
 	call_hooks('contact_photo_menu', $args);
-	
+
 /*	$o = "";
 	foreach($menu as $k=>$v){
 		if ($v!="") {
@@ -293,3 +297,107 @@ function contacts_not_grouped($uid,$start = 0,$count = 0) {
 	return $r;
 }
 
+function get_contact($url, $uid = 0) {
+	require_once("include/Scrape.php");
+
+	$data = array();
+	$contactid = 0;
+
+	// is it an address in the format user@server.tld?
+	if (!strstr($url, "http") OR strstr($url, "@")) {
+		$data = probe_url($url);
+		$url = $data["url"];
+		if ($url == "")
+			return 0;
+	}
+
+	$contact = q("SELECT `id`, `avatar-date` FROM `contact` WHERE `nurl` = '%s' AND `uid` = %d",
+			dbesc(normalise_link($url)),
+			intval($uid));
+
+	if (!$contact)
+		$contact = q("SELECT `id`, `avatar-date` FROM `contact` WHERE `alias` IN ('%s', '%s') AND `uid` = %d",
+				dbesc($url),
+				dbesc(normalise_link($url)),
+				intval($uid));
+
+	if ($contact) {
+		$contactid = $contact[0]["id"];
+
+		// Update the contact every 7 days
+		$update_photo = ($contact[0]['avatar-date'] < datetime_convert('','','now -7 days'));
+		//$update_photo = ($contact[0]['avatar-date'] < datetime_convert('','','now -12 hours'));
+
+		if (!$update_photo)
+			return($contactid);
+	}
+
+	if (!count($data))
+		$data = probe_url($url);
+
+	// Does this address belongs to a valid network?
+	if (!in_array($data["network"], array(NETWORK_DFRN, NETWORK_OSTATUS, NETWORK_DIASPORA)))
+		return 0;
+
+	// tempory programming. Can be deleted after 2015-02-07
+	if (($data["alias"] == "") AND (normalise_link($data["url"]) != normalise_link($url)))
+		$data["alias"] = normalise_link($url);
+
+	if ($contactid == 0) {
+		q("INSERT INTO `contact` (`uid`, `created`, `url`, `nurl`, `addr`, `alias`, `notify`, `poll`,
+					`name`, `nick`, `photo`, `network`, `pubkey`, `rel`, `priority`,
+					`batch`, `request`, `confirm`, `poco`,
+					`writable`, `blocked`, `readonly`, `pending`)
+					VALUES (%d, '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d, %d, '%s', '%s', '%s', '%s', 1, 0, 0, 0)",
+			intval($uid),
+			dbesc(datetime_convert()),
+			dbesc($data["url"]),
+			dbesc(normalise_link($data["url"])),
+			dbesc($data["addr"]),
+			dbesc($data["alias"]),
+			dbesc($data["notify"]),
+			dbesc($data["poll"]),
+			dbesc($data["name"]),
+			dbesc($data["nick"]),
+			dbesc($data["photo"]),
+			dbesc($data["network"]),
+			dbesc($data["pubkey"]),
+			intval(CONTACT_IS_SHARING),
+			intval($data["priority"]),
+			dbesc($data["batch"]),
+			dbesc($data["request"]),
+			dbesc($data["confirm"]),
+			dbesc($data["poco"])
+		);
+
+		$contact = q("SELECT `id` FROM `contact` WHERE `nurl` = '%s' AND `uid` = %d",
+				dbesc(normalise_link($data["url"])),
+				intval($uid));
+		if (!$contact)
+			return 0;
+
+		$contactid = $contact[0]["id"];
+	}
+
+	require_once("Photo.php");
+
+	$photos = import_profile_photo($data["photo"],$uid,$contactid);
+
+	q("UPDATE `contact` SET `photo` = '%s', `thumb` = '%s', `micro` = '%s',
+		`addr` = '%s', `alias` = '%s', `name` = '%s', `nick` = '%s',
+		`name-date` = '%s', `uri-date` = '%s', `avatar-date` = '%s' WHERE `id` = %d",
+		dbesc($photos[0]),
+		dbesc($photos[1]),
+		dbesc($photos[2]),
+		dbesc($data["addr"]),
+		dbesc($data["alias"]),
+		dbesc($data["name"]),
+		dbesc($data["nick"]),
+		dbesc(datetime_convert()),
+		dbesc(datetime_convert()),
+		dbesc(datetime_convert()),
+		intval($contactid)
+	);
+
+	return $contactid;
+}

include/Emailer.php

@@ -14,16 +14,24 @@ class Emailer {
 	 * @param htmlVersion		html version of the message
 	 * @param textVersion		text only version of the message
 	 * @param additionalMailHeader	additions to the smtp mail header
+	 * @param optional uid      user id of the destination user
 	 */
 	static public function send($params) {
 
+		call_hooks('emailer_send_prepare', $params);
+
+		$email_textonly = False;
+		if (x($params,"uid")) {
+			$email_textonly = get_pconfig($params['uid'], "system", "email_textonly");
+		}
+
 		$fromName = email_header_encode(html_entity_decode($params['fromName'],ENT_QUOTES,'UTF-8'),'UTF-8');
 		$messageSubject = email_header_encode(html_entity_decode($params['messageSubject'],ENT_QUOTES,'UTF-8'),'UTF-8');
 
 		// generate a mime boundary
 		$mimeBoundary   =rand(0,9)."-"
-				.rand(10000000000,9999999999)."-"
+				.rand(10000000000,99999999999)."-"
-				.rand(10000000000,9999999999)."=:"
+				.rand(10000000000,99999999999)."=:"
 				.rand(10000,99999);
 
 		// generate a multipart/alternative message header
@@ -41,19 +49,32 @@ class Emailer {
 			"--" . $mimeBoundary . "\n" .					// plain text section
 			"Content-Type: text/plain; charset=UTF-8\n" .
 			"Content-Transfer-Encoding: base64\n\n" .
-			$textBody . "\n" .
+			$textBody . "\n";
-			"--" . $mimeBoundary . "\n" .					// text/html section
+
-			"Content-Type: text/html; charset=UTF-8\n" .
+		if (!$email_textonly && !is_null($params['htmlVersion'])){
-			"Content-Transfer-Encoding: base64\n\n" .
+			$multipartMessageBody .=
-			$htmlBody . "\n" .
+				"--" . $mimeBoundary . "\n" .				// text/html section
+				"Content-Type: text/html; charset=UTF-8\n" .
+				"Content-Transfer-Encoding: base64\n\n" .
+				$htmlBody . "\n";
+		}
+		$multipartMessageBody .=
 			"--" . $mimeBoundary . "--\n";					// message ending
 
 		// send the message
+		$hookdata = array(
+			'to' => $params['toEmail'],
+			'subject' => $messageSubject,
+			'body' => $multipartMessageBody,
+			'headers' => $messageHeader
+		);
+		//echo "<pre>"; var_dump($hookdata); killme();
+		call_hooks("emailer_send", $hookdata);
 		$res = mail(
-			$params['toEmail'],	 									// send to address
+			$hookdata['to'],							// send to address
-			$messageSubject,								// subject
+			$hookdata['subject'],						// subject
-			$multipartMessageBody,	 						// message body
+			$hookdata['body'], 	 						// message body
-			$messageHeader									// message headers
+			$hookdata['headers']						// message headers
 		);
 		logger("header " . 'To: ' . $params['toEmail'] . "\n" . $messageHeader, LOGGER_DEBUG);
 		logger("return value " . (($res)?"true":"false"), LOGGER_DEBUG);

include/Photo.php

@@ -516,7 +516,12 @@ class Photo {
 	    return FALSE;
 
 	$string = $this->imageString();
+
+	$a = get_app();
+
+	$stamp1 = microtime(true);
 	file_put_contents($path, $string);
+	$a->save_timestamp($stamp1, "file");
     }
 
     public function imageString() {
@@ -764,11 +769,21 @@ function get_photo_info($url) {
 	if (is_null($data)) {
 		$img_str = fetch_url($url, true, $redirects, 4);
 
+		$filesize = strlen($img_str);
+
 		$tempfile = tempnam(get_temppath(), "cache");
+
+		$a = get_app();
+		$stamp1 = microtime(true);
 		file_put_contents($tempfile, $img_str);
+		$a->save_timestamp($stamp1, "file");
+
 		$data = getimagesize($tempfile);
 		unlink($tempfile);
 
+		if ($data)
+			$data["size"] = $filesize;
+
 		Cache::set($url, serialize($data));
 	} else
 		$data = unserialize($data);
@@ -846,7 +861,10 @@ function store_photo($a, $uid, $imagedata = "", $url = "") {
 		return(array());
 	} elseif (strlen($imagedata) == 0) {
 		logger("Uploading picture from ".$url, LOGGER_DEBUG);
+
+		$stamp1 = microtime(true);
 		$imagedata = @file_get_contents($url);
+		$a->save_timestamp($stamp1, "file");
 	}
 
 	$maximagesize = get_config('system','maximagesize');
@@ -870,7 +888,11 @@ function store_photo($a, $uid, $imagedata = "", $url = "") {
 */
 
 	$tempfile = tempnam(get_temppath(), "cache");
+
+	$stamp1 = microtime(true);
 	file_put_contents($tempfile, $imagedata);
+	$a->save_timestamp($stamp1, "file");
+
 	$data = getimagesize($tempfile);
 
 	if (!isset($data["mime"])) {

include/Scrape.php

@@ -343,6 +343,12 @@ function probe_url($url, $mode = PROBE_NORMAL) {
 	if(! $url)
 		return $result;
 
+	$result = Cache::get("probe_url:".$mode.":".$url);
+	if (!is_null($result)) {
+		$result = unserialize($result);
+		return $result;
+	}
+
 	$network = null;
 	$diaspora = false;
 	$diaspora_base = '';
@@ -350,6 +356,13 @@ function probe_url($url, $mode = PROBE_NORMAL) {
 	$diaspora_key = '';
 	$has_lrdd = false;
 	$email_conversant = false;
+	$connectornetworks = false;
+	$appnet = false;
+
+	if (strpos($url,'twitter.com')) {
+		$connectornetworks = true;
+		$network = NETWORK_TWITTER;
+	}
 
 	// Twitter is deactivated since twitter closed its old API
 	//$twitter = ((strpos($url,'twitter.com') !== false) ? true : false);
@@ -357,7 +370,7 @@ function probe_url($url, $mode = PROBE_NORMAL) {
 
 	$at_addr = ((strpos($url,'@') !== false) ? true : false);
 
-	if((! $twitter) && (! $lastfm)) {
+	if((!$appnet) && (!$lastfm) && !$connectornetworks) {
 
 		if(strpos($url,'mailto:') !== false && $at_addr) {
 			$url = str_replace('mailto:','',$url);
@@ -401,13 +414,16 @@ function probe_url($url, $mode = PROBE_NORMAL) {
 						$pubkey = $diaspora_key;
 					$diaspora = true;
 				}
+				if($link['@attributes']['rel'] === 'http://ostatus.org/schema/1.0/subscribe') {
+					$diaspora = false;
+				}
 			}
 
 			// Status.Net can have more than one profile URL. We need to match the profile URL
 			// to a contact on incoming messages to prevent spam, and we won't know which one
 			// to match. So in case of two, one of them is stored as an alias. Only store URL's
 			// and not webfinger user@host aliases. If they've got more than two non-email style
-			// aliases, let's hope we're lucky and get one that matches the feed author-uri because 
+			// aliases, let's hope we're lucky and get one that matches the feed author-uri because
 			// otherwise we're screwed.
 
 			foreach($links as $link) {
@@ -422,6 +438,10 @@ function probe_url($url, $mode = PROBE_NORMAL) {
 					}
 				}
 			}
+
+			// If the profile is different from the url then the url is abviously an alias
+			if (($alias == "") AND ($profile != "") AND !$at_addr AND (normalise_link($profile) != normalise_link($url)))
+				$alias = $url;
 		}
 		elseif($mode == PROBE_NORMAL) {
 
@@ -496,8 +516,8 @@ function probe_url($url, $mode = PROBE_NORMAL) {
 				if($j) {
 					$network = NETWORK_ZOT;
 					$vcard   = array(
-						'fn'    => $j->fullname, 
+						'fn'    => $j->fullname,
-						'nick'  => $j->nickname, 
+						'nick'  => $j->nickname,
 						'photo' => $j->photo
 					);
 					$profile  = $j->url;
@@ -539,6 +559,10 @@ function probe_url($url, $mode = PROBE_NORMAL) {
 			$network = NETWORK_DIASPORA;
 		elseif($has_lrdd)
 			$network  = NETWORK_OSTATUS;
+
+		if(strpos($url,'@'))
+			$addr = str_replace('acct:', '', $url);
+
 		$priority = 0;
 
 		if($hcard && ! $vcard) {
@@ -597,13 +621,16 @@ function probe_url($url, $mode = PROBE_NORMAL) {
 			// Will leave it to others to figure out how to grab the avatar, which is on the $url page in the open graph meta links
 		}
 
-		if($twitter || ! $poll)
+		if($appnet || ! $poll)
 			$check_feed = true;
 		if((! isset($vcard)) || (! x($vcard,'fn')) || (! $profile))
 			$check_feed = true;
 		if(($at_addr) && (! count($links)))
 			$check_feed = false;
 
+		if ($connectornetworks)
+			$check_feed = false;
+
 		if($check_feed) {
 
 			$feedret = scrape_feed(($poll) ? $poll : $url);
@@ -729,6 +756,22 @@ function probe_url($url, $mode = PROBE_NORMAL) {
 	if(($network === NETWORK_FEED) && ($poll) && (! x($vcard,'fn')))
 		$vcard['fn'] = $url;
 
+	if (($notify != "") AND ($poll != "")) {
+		$baseurl = matching($notify, $poll);
+
+		$baseurl2 = matching($baseurl, $profile);
+		if ($baseurl2 != "")
+			$baseurl = $baseurl2;
+	}
+
+	if (($baseurl == "") AND ($notify != ""))
+		$baseurl = matching($profile, $notify);
+
+	if (($baseurl == "") AND ($poll != ""))
+		$baseurl = matching($profile, $poll);
+
+	$baseurl = rtrim($baseurl, "/");
+
 	$vcard['fn'] = notags($vcard['fn']);
 	$vcard['nick'] = str_replace(' ','',notags($vcard['nick']));
 
@@ -747,8 +790,37 @@ function probe_url($url, $mode = PROBE_NORMAL) {
 	$result['network'] = $network;
 	$result['alias'] = $alias;
 	$result['pubkey'] = $pubkey;
+	$result['baseurl'] = $baseurl;
 
 	logger('probe_url: ' . print_r($result,true), LOGGER_DEBUG);
 
+	// Trying if it maybe a diaspora account
+	if (($result['network'] == NETWORK_FEED) OR ($result['addr'] == "")) {
+		require_once('include/bbcode.php');
+		$address = GetProfileUsername($url, "", true);
+		$result2 = probe_url($address, $mode);
+		if ($result2['network'] != "")
+			$result = $result2;
+	}
+
+	Cache::set("probe_url:".$mode.":".$url,serialize($result));
+
 	return $result;
 }
+
+function matching($part1, $part2) {
+	$len = min(strlen($part1), strlen($part2));
+
+	$match = "";
+	$matching = true;
+	$i = 0;
+	while (($i <= $len) AND $matching) {
+		if (substr($part1, $i, 1) == substr($part2, $i, 1))
+			$match .= substr($part1, $i, 1);
+		else
+			$matching = false;
+
+		$i++;
+	}
+	return($match);
+}

include/acl_selectors.php

@@ -2,13 +2,14 @@
 
 require_once("include/contact_selectors.php");
 require_once("include/features.php");
+require_once("mod/proxy.php");
 
 /**
- * 
+ *
  */
 
 /**
- * @package acl_selectors 
+ * @package acl_selectors
  */
 function group_select($selname,$selclass,$preselected = false,$size = 4) {
 
@@ -35,11 +36,12 @@ function group_select($selname,$selclass,$preselected = false,$size = 4) {
 				$selected = " selected=\"selected\" ";
 			else
 				$selected = '';
+
 			$trimmed = mb_substr($rr['name'],0,12);
 
 			$o .= "<option value=\"{$rr['id']}\" $selected title=\"{$rr['name']}\" >$trimmed</option>\r\n";
 		}
-	
+
 	}
 	$o .= "</select>\r\n";
 
@@ -89,13 +91,13 @@ function contact_selector($selname, $selclass, $preselected = false, $options) {
 						$networks = array('dfrn','mail','dspr');
 					else
 						$networks = array('dfrn','face','mail','dspr','stat');
-					break;					
+					break;
 				default:
 					break;
 			}
 		}
 	}
-		
+
 	$x = array('options' => $options, 'size' => $size, 'single' => $single, 'mutual' => $mutual, 'exclude' => $exclude, 'networks' => $networks);
 
 	call_hooks('contact_select_options', $x);
@@ -117,15 +119,15 @@ function contact_selector($selname, $selclass, $preselected = false, $options) {
 		$str_nets = implode(',',$x['networks']);
 		$sql_extra .= " AND `network` IN ( $str_nets ) ";
 	}
-	
+
 	$tabindex = (x($options, 'tabindex') ? "tabindex=\"" . $options["tabindex"] . "\"" : "");
 
 	if($x['single'])
 		$o .= "<select name=\"$selname\" id=\"$selclass\" class=\"$selclass\" size=\"" . $x['size'] . "\" $tabindex >\r\n";
-	else 
+	else
 		$o .= "<select name=\"{$selname}[]\" id=\"$selclass\" class=\"$selclass\" multiple=\"multiple\" size=\"" . $x['size'] . "$\" $tabindex >\r\n";
 
-	$r = q("SELECT `id`, `name`, `url`, `network` FROM `contact` 
+	$r = q("SELECT `id`, `name`, `url`, `network` FROM `contact`
 		WHERE `uid` = %d AND `self` = 0 AND `blocked` = 0 AND `pending` = 0 AND `archive` = 0 AND `notify` != ''
 		$sql_extra
 		ORDER BY `name` ASC ",
@@ -150,7 +152,7 @@ function contact_selector($selname, $selclass, $preselected = false, $options) {
 
 			$o .= "<option value=\"{$rr['id']}\" $selected title=\"{$rr['name']}|{$rr['url']}\" >$trimmed</option>\r\n";
 		}
-	
+
 	}
 
 	$o .= "</select>\r\n";
@@ -164,6 +166,8 @@ function contact_selector($selname, $selclass, $preselected = false, $options) {
 
 function contact_select($selname, $selclass, $preselected = false, $size = 4, $privmail = false, $celeb = false, $privatenet = false, $tabindex = null) {
 
+	require_once("include/bbcode.php");
+
 	$a = get_app();
 
 	$o = '';
@@ -180,7 +184,7 @@ function contact_select($selname, $selclass, $preselected = false, $size = 4, $p
 	if($privmail) {
 		$sql_extra .= " AND `network` IN ( 'dfrn', 'dspr' ) ";
 	}
-	elseif($privatenet) {	
+	elseif($privatenet) {
 		$sql_extra .= " AND `network` IN ( 'dfrn', 'mail', 'face', 'dspr' ) ";
 	}
 
@@ -188,10 +192,10 @@ function contact_select($selname, $selclass, $preselected = false, $size = 4, $p
 
 	if($privmail)
 		$o .= "<select name=\"$selname\" id=\"$selclass\" class=\"$selclass\" size=\"$size\" $tabindex >\r\n";
-	else 
+	else
 		$o .= "<select name=\"{$selname}[]\" id=\"$selclass\" class=\"$selclass\" multiple=\"multiple\" size=\"$size\" $tabindex >\r\n";
 
-	$r = q("SELECT `id`, `name`, `url`, `network` FROM `contact` 
+	$r = q("SELECT `id`, `name`, `url`, `network` FROM `contact`
 		WHERE `uid` = %d AND `self` = 0 AND `blocked` = 0 AND `pending` = 0 AND `archive` = 0 AND `notify` != ''
 		$sql_extra
 		ORDER BY `name` ASC ",
@@ -212,11 +216,14 @@ function contact_select($selname, $selclass, $preselected = false, $size = 4, $p
 			else
 				$selected = '';
 
-			$trimmed = mb_substr($rr['name'],0,20);
+			if($privmail)
+				$trimmed = GetProfileUsername($rr['url'], $rr['name'], false);
+			else
+				$trimmed = mb_substr($rr['name'],0,20);
 
 			$o .= "<option value=\"{$rr['id']}\" $selected title=\"{$rr['name']}|{$rr['url']}\" >$trimmed</option>\r\n";
 		}
-	
+
 	}
 
 	$o .= "</select>\r\n";
@@ -239,7 +246,7 @@ function prune_deadguys($arr) {
 	$r = q("select id from contact where id in ( " . $str . ") and blocked = 0 and pending = 0 and archive = 0 ");
 	if($r) {
 		$ret = array();
-		foreach($r as $rr) 
+		foreach($r as $rr)
 			$ret[] = intval($rr['id']);
 		return $ret;
 	}
@@ -251,7 +258,7 @@ function get_acl_permissions($user = null) {
 	$allow_cid = $allow_gid = $deny_cid = $deny_gid = false;
 
 	if(is_array($user)) {
-		$allow_cid = ((strlen($user['allow_cid'])) 
+		$allow_cid = ((strlen($user['allow_cid']))
 			? explode('><', $user['allow_cid']) : array() );
 		$allow_gid = ((strlen($user['allow_gid']))
 			? explode('><', $user['allow_gid']) : array() );
@@ -411,21 +418,21 @@ function acl_lookup(&$a, $out_type = 'json') {
 	}
 
 	if ($type=='' || $type=='c'){
-		$r = q("SELECT COUNT(*) AS c FROM `contact` 
+		$r = q("SELECT COUNT(*) AS c FROM `contact`
-				WHERE `uid` = %d AND `self` = 0 
+				WHERE `uid` = %d AND `self` = 0
 				AND `blocked` = 0 AND `pending` = 0 AND `archive` = 0
 				AND `notify` != '' $sql_extra2" ,
 			intval(local_user())
 		);
 		$contact_count = (int)$r[0]['c'];
-	} 
+	}
 	elseif ($type == 'm') {
 
 		// autocomplete for Private Messages
 
-		$r = q("SELECT COUNT(*) AS c FROM `contact` 
+		$r = q("SELECT COUNT(*) AS c FROM `contact`
-				WHERE `uid` = %d AND `self` = 0 
+				WHERE `uid` = %d AND `self` = 0
-				AND `blocked` = 0 AND `pending` = 0 AND `archive` = 0 
+				AND `blocked` = 0 AND `pending` = 0 AND `archive` = 0
 				AND `network` IN ('%s','%s','%s') $sql_extra2" ,
 			intval(local_user()),
 			dbesc(NETWORK_DFRN),
@@ -439,8 +446,8 @@ function acl_lookup(&$a, $out_type = 'json') {
 
 		// autocomplete for Contacts
 
-		$r = q("SELECT COUNT(*) AS c FROM `contact` 
+		$r = q("SELECT COUNT(*) AS c FROM `contact`
-				WHERE `uid` = %d AND `self` = 0 
+				WHERE `uid` = %d AND `self` = 0
 				AND `pending` = 0 $sql_extra2" ,
 			intval(local_user())
 		);
@@ -449,22 +456,22 @@ function acl_lookup(&$a, $out_type = 'json') {
 	} else {
 		$contact_count = 0;
 	}
-	
+
-	
+
 	$tot = $group_count+$contact_count;
-	
+
 	$groups = array();
 	$contacts = array();
-	
+
 	if ($type=='' || $type=='g'){
-		
+
 		$r = q("SELECT `group`.`id`, `group`.`name`, GROUP_CONCAT(DISTINCT `group_member`.`contact-id` SEPARATOR ',') as uids
-				FROM `group`,`group_member` 
+				FROM `group`,`group_member`
-				WHERE `group`.`deleted` = 0 AND `group`.`uid` = %d 
+				WHERE `group`.`deleted` = 0 AND `group`.`uid` = %d
 					AND `group_member`.`gid`=`group`.`id`
 					$sql_extra
 				GROUP BY `group`.`id`
-				ORDER BY `group`.`name` 
+				ORDER BY `group`.`name`
 				LIMIT %d,%d",
 			intval(local_user()),
 			intval($start),
@@ -472,7 +479,7 @@ function acl_lookup(&$a, $out_type = 'json') {
 		);
 
 		foreach($r as $g){
-//		logger('acl: group: ' . $g['name'] . ' members: ' . $g['uids']);		
+//		logger('acl: group: ' . $g['name'] . ' members: ' . $g['uids']);
 			$groups[] = array(
 				"type"  => "g",
 				"photo" => "images/twopeople.png",
@@ -484,10 +491,10 @@ function acl_lookup(&$a, $out_type = 'json') {
 			);
 		}
 	}
-	
+
 	if ($type=='' || $type=='c'){
-	
+
-		$r = q("SELECT `id`, `name`, `nick`, `micro`, `network`, `url`, `attag`, forum FROM `contact` 
+		$r = q("SELECT `id`, `name`, `nick`, `micro`, `network`, `url`, `attag`, forum FROM `contact`
 			WHERE `uid` = %d AND `self` = 0 AND `blocked` = 0 AND `pending` = 0 AND `archive` = 0 AND `notify` != ''
 			$sql_extra2
 			ORDER BY `name` ASC ",
@@ -495,7 +502,7 @@ function acl_lookup(&$a, $out_type = 'json') {
 		);
 	}
 	elseif($type == 'm') {
-		$r = q("SELECT `id`, `name`, `nick`, `micro`, `network`, `url`, `attag` FROM `contact` 
+		$r = q("SELECT `id`, `name`, `nick`, `micro`, `network`, `url`, `attag` FROM `contact`
 			WHERE `uid` = %d AND `self` = 0 AND `blocked` = 0 AND `pending` = 0 AND `archive` = 0
 			AND `network` IN ('%s','%s','%s')
 			$sql_extra2
@@ -507,7 +514,7 @@ function acl_lookup(&$a, $out_type = 'json') {
 		);
 	}
 	elseif($type == 'a') {
-		$r = q("SELECT `id`, `name`, `nick`, `micro`, `network`, `url`, `attag` FROM `contact` 
+		$r = q("SELECT `id`, `name`, `nick`, `micro`, `network`, `url`, `attag` FROM `contact`
 			WHERE `uid` = %d AND `pending` = 0
 			$sql_extra2
 			ORDER BY `name` ASC ",
@@ -527,7 +534,7 @@ function acl_lookup(&$a, $out_type = 'json') {
 		$x['data'] = array();
 		if(count($r)) {
 			foreach($r as $g) {
-				$x['photos'][] = $g['micro'];
+				$x['photos'][] = proxy_url($g['micro']);
 				$x['links'][] = $g['url'];
 				$x['suggestions'][] = $g['name'];
 				$x['data'][] = intval($g['id']);
@@ -541,7 +548,7 @@ function acl_lookup(&$a, $out_type = 'json') {
 		foreach($r as $g){
 			$contacts[] = array(
 				"type"  => "c",
-				"photo" => $g['micro'],
+				"photo" => proxy_url($g['micro']),
 				"name"  => $g['name'],
 				"id"	=> intval($g['id']),
 				"network" => $g['network'],
@@ -549,28 +556,28 @@ function acl_lookup(&$a, $out_type = 'json') {
 				"nick" => ($g['attag']) ? $g['attag'] : $g['nick'],
 				"forum" => $g['forum']
 			);
-		}			
+		}
 	}
-		
+
 	$items = array_merge($groups, $contacts);
 
 	if ($conv_id) {
-		/* if $conv_id is set, get unknow contacts in thread */ 
+		/* if $conv_id is set, get unknow contacts in thread */
 		/* but first get know contacts url to filter them out */
 		function _contact_link($i){ return dbesc($i['link']); }
 		$known_contacts = array_map(_contact_link, $contacts);
 		$unknow_contacts=array();
-		$r = q("select 
+		$r = q("select
 					`author-avatar`,`author-name`,`author-link`
 				from item where parent=%d
 				and (
 					`author-name` LIKE '%%%s%%' OR
 					`author-link` LIKE '%%%s%%'
-				) and 
+				) and
 				`author-link` NOT IN ('%s')
 				GROUP BY `author-link`
 				ORDER BY `author-name` ASC
-				", 
+				",
 				intval($conv_id),
 				dbesc($search),
 				dbesc($search),
@@ -586,7 +593,7 @@ function acl_lookup(&$a, $out_type = 'json') {
 				// /nickname
 				$unknow_contacts[] = array(
 					"type"  => "c",
-					"photo" => $row['author-avatar'],
+					"photo" => proxy_url($row['author-avatar']),
 					"name"  => $row['author-name'],
 					"id"	=> '',
 					"network" => "unknown",
@@ -611,14 +618,14 @@ function acl_lookup(&$a, $out_type = 'json') {
 		);
 		return $o;
 	}
-	
+
 	$o = array(
 		'tot'	=> $tot,
 		'start' => $start,
 		'count'	=> $count,
 		'items'	=> $items,
 	);
-	
+
 	echo json_encode($o);
 
 	killme();

include/api.php

@@ -96,24 +96,49 @@
 		}
 
 		$user = $_SERVER['PHP_AUTH_USER'];
-		$encrypted = hash('whirlpool',trim($_SERVER['PHP_AUTH_PW']));
+		$password = $_SERVER['PHP_AUTH_PW'];
+		$encrypted = hash('whirlpool',trim($password));
 
 
 		/**
 		 *  next code from mod/auth.php. needs better solution
 		 */
+		$record = null;
 
-		// process normal login request
+		$addon_auth = array(
-
+			'username' => trim($user), 
-		$r = q("SELECT * FROM `user` WHERE ( `email` = '%s' OR `nickname` = '%s' )
+			'password' => trim($password),
-			AND `password` = '%s' AND `blocked` = 0 AND `account_expired` = 0 AND `account_removed` = 0 AND `verified` = 1 LIMIT 1",
+			'authenticated' => 0,
-			dbesc(trim($user)),
+			'user_record' => null
-			dbesc(trim($user)),
-			dbesc($encrypted)
 		);
-		if(count($r)){
+
-			$record = $r[0];
+		/**
-		} else {
+		 *
+		 * A plugin indicates successful login by setting 'authenticated' to non-zero value and returning a user record
+		 * Plugins should never set 'authenticated' except to indicate success - as hooks may be chained
+		 * and later plugins should not interfere with an earlier one that succeeded.
+		 *
+		 */
+
+		call_hooks('authenticate', $addon_auth);
+
+		if(($addon_auth['authenticated']) && (count($addon_auth['user_record']))) {
+			$record = $addon_auth['user_record'];
+		}
+		else {
+			// process normal login request
+
+			$r = q("SELECT * FROM `user` WHERE ( `email` = '%s' OR `nickname` = '%s' )
+				AND `password` = '%s' AND `blocked` = 0 AND `account_expired` = 0 AND `account_removed` = 0 AND `verified` = 1 LIMIT 1",
+				dbesc(trim($user)),
+				dbesc(trim($user)),
+				dbesc($encrypted)
+			);
+			if(count($r))
+				$record = $r[0];
+		}
+
+		if((! $record) || (! count($record))) {
 			logger('API_login failure: ' . print_r($_SERVER,true), LOGGER_DEBUG);
 			header('WWW-Authenticate: Basic realm="Friendica"');
 			header('HTTP/1.0 401 Unauthorized');
@@ -197,6 +222,7 @@
 	}
 
 	function api_error(&$a, $type, $error) {
+		# TODO:  https://dev.twitter.com/overview/api/response-codes
 		$r = "<status><error>".$error."</error><request>".$a->query_string."</request></status>";
 		switch($type){
 			case "xml":
@@ -707,8 +733,7 @@
 				$_REQUEST['body'] = html2bbcode($txt);
 			}
 
-		}
+		} else
-		else
 			$_REQUEST['body'] = requestdata('status');
 
 		$_REQUEST['title'] = requestdata('title');
@@ -785,14 +810,15 @@
 			}
 
 			$_REQUEST['type'] = 'wall';
-			if(x($_FILES,'media')) {
+		}
-				// upload the image if we have one
+
-				$_REQUEST['hush']='yeah'; //tell wall_upload function to return img info instead of echo
+		if(x($_FILES,'media')) {
-				require_once('mod/wall_upload.php');
+			// upload the image if we have one
-				$media = wall_upload_post($a);
+			$_REQUEST['hush']='yeah'; //tell wall_upload function to return img info instead of echo
-				if(strlen($media)>0)
+			require_once('mod/wall_upload.php');
-					$_REQUEST['body'] .= "\n\n".$media;
+			$media = wall_upload_post($a);
-			}
+			if(strlen($media)>0)
+				$_REQUEST['body'] .= "\n\n".$media;
 		}
 
 		// set this so that the item_post() function is quiet and doesn't redirect or emit json
@@ -822,7 +848,7 @@
 		// get last public wall message
 		$lastwall = q("SELECT `item`.*, `i`.`contact-id` as `reply_uid`, `i`.`author-link` AS `item-author`
 				FROM `item`, `item` as `i`
-				WHERE `item`.`contact-id` = %d
+				WHERE `item`.`contact-id` = %d AND `item`.`uid` = %d
 					AND ((`item`.`author-link` IN ('%s', '%s')) OR (`item`.`owner-link` IN ('%s', '%s')))
 					AND `i`.`id` = `item`.`parent`
 					AND `item`.`type`!='activity'
@@ -830,6 +856,7 @@
 				ORDER BY `item`.`created` DESC
 				LIMIT 1",
 				intval($user_info['cid']),
+				intval(api_user()),
 				dbesc($user_info['url']),
 				dbesc(normalise_link($user_info['url'])),
 				dbesc($user_info['url']),
@@ -871,8 +898,10 @@
 				$in_reply_to_screen_name = NULL;
 			}
 
+			$converted = api_convert_item($item);
+
 			$status_info = array(
-				'text' => trim(html2plain(bbcode(api_clean_plain_items($lastwall['body']), false, false, 2, true), 0)),
+				'text' => $converted["text"],
 				'truncated' => false,
 				'created_at' => api_date($lastwall['created']),
 				'in_reply_to_status_id' => $in_reply_to_status_id,
@@ -884,19 +913,17 @@
 				'in_reply_to_user_id_str' => $in_reply_to_user_id_str,
 				'in_reply_to_screen_name' => $in_reply_to_screen_name,
 				'geo' => NULL,
-				'favorited' => false,
+				'favorited' => $lastwall['starred'] ? true : false,
-				// attachments
 				'user' => $user_info,
-				'statusnet_html'		=> trim(bbcode($lastwall['body'], false, false)),
+				'statusnet_html'		=> $converted["html"],
 				'statusnet_conversation_id'	=> $lastwall['parent'],
 			);
 
-			if ($lastwall['title'] != "")
+			if (count($converted["attachments"]) > 0)
-				$status_info['statusnet_html'] = "<h4>".bbcode($lastwall['title'])."</h4>\n".$status_info['statusnet_html'];
+				$status_info["attachments"] = $converted["attachments"];
 
-			$entities = api_get_entitities($status_info['text'], $lastwall['body']);
+			if (count($converted["entities"]) > 0)
-			if (count($entities) > 0)
+				$status_info["entities"] = $converted["entities"];
-				$status_info['entities'] = $entities;
 
 			if (($lastwall['item_network'] != "") AND ($status["source"] == 'web'))
 				$status_info["source"] = network_to_name($lastwall['item_network']);
@@ -970,8 +997,11 @@
 					}
 				}
 			}
+
+			$converted = api_convert_item($item);
+
 			$user_info['status'] = array(
-				'text' => trim(html2plain(bbcode(api_clean_plain_items($lastwall['body']), false, false, 2, true), 0)),
+				'text' => $converted["text"],
 				'truncated' => false,
 				'created_at' => api_date($lastwall['created']),
 				'in_reply_to_status_id' => $in_reply_to_status_id,
@@ -983,17 +1013,16 @@
 				'in_reply_to_user_id_str' => $in_reply_to_user_id_str,
 				'in_reply_to_screen_name' => $in_reply_to_screen_name,
 				'geo' => NULL,
-				'favorited' => false,
+				'favorited' => $lastwall['starred'] ? true : false,
-				'statusnet_html'		=> trim(bbcode($lastwall['body'], false, false)),
+				'statusnet_html'		=> $converted["html"],
 				'statusnet_conversation_id'	=> $lastwall['parent'],
 			);
 
-			if ($lastwall['title'] != "")
+			if (count($converted["attachments"]) > 0)
-				$user_info['statusnet_html'] = "<h4>".bbcode($lastwall['title'])."</h4>\n".$user_info['statusnet_html'];
+				$user_info["status"]["attachments"] = $converted["attachments"];
 
-			$entities = api_get_entitities($user_info['text'], $lastwall['body']);
+			if (count($converted["entities"]) > 0)
-			if (count($entities) > 0)
+				$user_info["status"]["entities"] = $converted["entities"];
-				$user_info['entities'] = $entities;
 
 			if (($lastwall['item_network'] != "") AND ($user_info["status"]["source"] == 'web'))
 				$user_info["status"]["source"] = network_to_name($lastwall['item_network']);
@@ -1100,15 +1129,14 @@
 
 		$ret = api_format_items($r,$user_info);
 
-		// We aren't going to try to figure out at the item, group, and page
+		// Set all posts from the query above to seen
-		// level which items you've seen and which you haven't. If you're looking
+		$idarray = array();
-		// at the network timeline just mark everything seen. 
+		foreach ($r AS $item)
+			$idarray[] = intval($item["id"]);
 
-		$r = q("UPDATE `item` SET `unseen` = 0 
+		$idlist = implode(",", $idarray);
-			WHERE `unseen` = 1 AND `uid` = %d",
+
-			//intval($user_info['uid'])
+		$r = q("UPDATE `item` SET `unseen` = 0 WHERE `unseen` AND `id` IN (%s)", $idlist);
-			intval(api_user())
-		);
 
 
 		$data = array('$statuses' => $ret);
@@ -1198,7 +1226,7 @@
 	api_register_func('api/statuses/public_timeline','api_statuses_public_timeline', true);
 
 	/**
-	 * 
+	 *
 	 */
 	function api_statuses_show(&$a, $type){
 		if (api_user()===false) return false;
@@ -1419,9 +1447,9 @@
 	api_register_func('api/statuses/destroy','api_statuses_destroy', true);
 
 	/**
-	 * 
+	 *
 	 * http://developer.twitter.com/doc/get/statuses/mentions
-	 * 
+	 *
 	 */
 	function api_statuses_mentions(&$a, $type){
 		if (api_user()===false) return false;
@@ -1568,6 +1596,69 @@
 	api_register_func('api/statuses/user_timeline','api_statuses_user_timeline', true);
 
 
+	/**
+	 * Star/unstar an item
+	 * param: id : id of the item
+	 *
+	 * api v1 : https://web.archive.org/web/20131019055350/https://dev.twitter.com/docs/api/1/post/favorites/create/%3Aid
+	 */
+	function api_favorites_create_destroy(&$a, $type){
+		if (api_user()===false) return false;
+
+		# for versioned api.
+		# TODO: we need a better global soluton
+		$action_argv_id=2;
+		if ($a->argv[1]=="1.1") $action_argv_id=3;
+
+		if ($a->argc<=$action_argv_id) die(api_error($a, $type, t("Invalid request.")));
+		$action = str_replace(".".$type,"",$a->argv[$action_argv_id]);
+		if ($a->argc==$action_argv_id+2) {
+			$itemid = intval($a->argv[$action_argv_id+1]);
+		} else {
+			$itemid = intval($_REQUEST['id']);
+		}
+
+		$item = q("SELECT * FROM item WHERE id=%d AND uid=%d",
+				$itemid, api_user());
+
+		if ($item===false || count($item)==0) die(api_error($a, $type, t("Invalid item.")));
+
+		switch($action){
+			case "create":
+				$item[0]['starred']=1;
+				break;
+			case "destroy":
+				$item[0]['starred']=0;
+				break;
+			default:
+				die(api_error($a, $type, t("Invalid action. ".$action)));
+		}
+		$r = q("UPDATE item SET starred=%d WHERE id=%d AND uid=%d",
+				$item[0]['starred'], $itemid, api_user());
+
+		q("UPDATE thread SET starred=%d WHERE iid=%d AND uid=%d",
+			$item[0]['starred'], $itemid, api_user());
+
+		if ($r===false) die(api_error($a, $type, t("DB error")));
+
+
+		$user_info = api_get_user($a);
+		$rets = api_format_items($item,$user_info);
+		$ret = $rets[0];
+
+		$data = array('$status' => $ret);
+		switch($type){
+			case "atom":
+			case "rss":
+				$data = api_rss_extra($a, $data, $user_info);
+		}
+
+		return api_apply_template("status", $type, $data);
+	}
+
+	api_register_func('api/favorites/create', 'api_favorites_create_destroy', true);
+	api_register_func('api/favorites/destroy', 'api_favorites_create_destroy', true);
+
 	function api_favorites(&$a, $type){
 		global $called_api;
 
@@ -1603,7 +1694,7 @@
 				`contact`.`network`, `contact`.`thumb`, `contact`.`dfrn-id`, `contact`.`self`,
 				`contact`.`id` AS `cid`, `contact`.`uid` AS `contact-uid`
 				FROM `item`, `contact`
-				WHERE `item`.`uid` = %d AND `verb` = '%s'
+				WHERE `item`.`uid` = %d
 				AND `item`.`visible` = 1 and `item`.`moderated` = 0 AND `item`.`deleted` = 0
 				AND `item`.`starred` = 1
 				AND `contact`.`id` = `item`.`contact-id`
@@ -1612,7 +1703,6 @@
 				AND `item`.`id`>%d
 				ORDER BY `item`.`id` DESC LIMIT %d ,%d ",
 				intval(api_user()),
-				dbesc(ACTIVITY_POST),
 				intval($since_id),
 				intval($start),	intval($count)
 			);
@@ -1633,6 +1723,9 @@
 
 	api_register_func('api/favorites','api_favorites', true);
 
+
+
+
 	function api_format_as($a, $ret, $user_info) {
 
 		$as = array();
@@ -1740,6 +1833,67 @@
 		return $ret;
 	}
 
+	function api_convert_item($item) {
+
+		$body = $item['body'];
+		$attachments = api_get_attachments($body);
+
+		// Workaround for ostatus messages where the title is identically to the body
+		$html = bbcode(api_clean_plain_items($body), false, false, 2, true);
+		$statusbody = trim(html2plain($html, 0));
+
+		// handle data: images
+		$statusbody = api_format_items_embeded_images($item,$statusbody);
+
+		$statustitle = trim($item['title']);
+
+		if (($statustitle != '') and (strpos($statusbody, $statustitle) !== false))
+			$statustext = trim($statusbody);
+		else
+			$statustext = trim($statustitle."\n\n".$statusbody);
+
+		if (($item["network"] == NETWORK_FEED) and (strlen($statustext)> 1000))
+			$statustext = substr($statustext, 0, 1000)."... \n".$item["plink"];
+
+		$statushtml = trim(bbcode($body, false, false));
+
+		if ($item['title'] != "")
+			$statushtml = "<h4>".bbcode($item['title'])."</h4>\n".$statushtml;
+
+		$entities = api_get_entitities($statustext, $body);
+
+		return(array("text" => $statustext, "html" => $statushtml, "attachments" => $attachments, "entities" => $entities));
+	}
+
+	function api_get_attachments(&$body) {
+
+		$text = $body;
+		$text = preg_replace("/\[img\=([0-9]*)x([0-9]*)\](.*?)\[\/img\]/ism", '[img]$3[/img]', $text);
+
+		$URLSearchString = "^\[\]";
+		$ret = preg_match_all("/\[img\]([$URLSearchString]*)\[\/img\]/ism", $text, $images);
+
+		if (!$ret)
+			return false;
+
+		require_once("include/Photo.php");
+
+		$attachments = array();
+
+		foreach ($images[1] AS $image) {
+			$imagedata = get_photo_info($image);
+
+			if ($imagedata)
+				$attachments[] = array("url" => $image, "mimetype" => $imagedata["mime"], "size" => $imagedata["size"]);
+		}
+
+		if (strstr($_SERVER['HTTP_USER_AGENT'], "AndStatus"))
+			foreach ($images[0] AS $orig)
+				$body = str_replace($orig, "", $body);
+
+		return $attachments;
+	}
+
 	function api_get_entitities(&$text, $bbcode) {
 		/*
 		To-Do:
@@ -1907,14 +2061,14 @@
 				$text);
 		return $text;
 	}
-	
+
 	function api_format_items($r,$user_info, $filter_user = false) {
 
 		$a = get_app();
 		$ret = Array();
 
 		foreach($r as $item) {
-			api_share_as_retweet($a, api_user(), $item);
+			api_share_as_retweet($item);
 
 			localize_item($item);
 			$status_user = api_item_get_user($a,$item);
@@ -1961,29 +2115,10 @@
 				$in_reply_to_status_id_str = NULL;
 			}
 
-			// Workaround for ostatus messages where the title is identically to the body
+			$converted = api_convert_item($item);
-			//$statusbody = trim(html2plain(bbcode(api_clean_plain_items($item['body']), false, false, 5, true), 0));
-			$html = bbcode(api_clean_plain_items($item['body']), false, false, 2, true);
-			$statusbody = trim(html2plain($html, 0));
-			
-			// handle data: images
-			$statusbody = api_format_items_embeded_images($item,$statusbody);
-			
-			$statustitle = trim($item['title']);
 
-			if (($statustitle != '') and (strpos($statusbody, $statustitle) !== false))
-				$statustext = trim($statusbody);
-			else
-				$statustext = trim($statustitle."\n\n".$statusbody);
-				
-			if (($item["network"] == NETWORK_FEED) and (strlen($statustext)> 1000))
-				$statustext = substr($statustext, 0, 1000)."... \n".$item["plink"];
-
-			$statushtml = trim(bbcode($item['body'], false, false));
-			
-			
 			$status = array(
-				'text'		=> $statustext,
+				'text'		=> $converted["text"],
 				'truncated' => False,
 				'created_at'=> api_date($item['created']),
 				'in_reply_to_status_id' => $in_reply_to_status_id,
@@ -1996,19 +2131,17 @@
 				'in_reply_to_screen_name' => $in_reply_to_screen_name,
 				'geo' => NULL,
 				'favorited' => $item['starred'] ? true : false,
-				//'attachments' => array(),
 				'user' =>  $status_user ,
 				//'entities' => NULL,
-				'statusnet_html'		=> $statushtml,
+				'statusnet_html'		=> $converted["html"],
 				'statusnet_conversation_id'	=> $item['parent'],
 			);
 
-			if ($item['title'] != "")
+			if (count($converted["attachments"]) > 0)
-				$status['statusnet_html'] = "<h4>".bbcode($item['title'])."</h4>\n".$status['statusnet_html'];
+				$status["attachments"] = $converted["attachments"];
 
-			$entities = api_get_entitities($status['text'], $item['body']);
+			if (count($converted["entities"]) > 0)
-			if (count($entities) > 0)
+				$status["entities"] = $converted["entities"];
-				$status['entities'] = $entities;
 
 			if (($item['item_network'] != "") AND ($status["source"] == 'web'))
 				$status["source"] = network_to_name($item['item_network']);
@@ -2367,7 +2500,7 @@
 
 		if ($user_id !="") {
 			$sql_extra .= ' AND `mail`.`contact-id` = ' . intval($user_id);
-		} 
+		}
 		elseif($screen_name !=""){
 			$sql_extra .= " AND `contact`.`nick` = '" . dbesc($screen_name). "'";
 		}
@@ -2377,7 +2510,7 @@
 				intval($since_id),
 				intval($start),	intval($count)
 		);
-	
+
 
 		$ret = Array();
 		foreach($r as $item) {
@@ -2479,7 +2612,7 @@
 			echo json_encode($r[0]);
 		}
 
-		killme();	
+		killme();
 	}
 
 	api_register_func('api/friendica/photos/list', 'api_fr_photos_list', true);
@@ -2487,7 +2620,7 @@
 
 
 
-function api_share_as_retweet($a, $uid, &$item) {
+function api_share_as_retweet(&$item) {
 	$body = trim($item["body"]);
 
 	// Skip if it isn't a pure repeated messages
@@ -2531,6 +2664,15 @@ function api_share_as_retweet($a, $uid, &$item) {
 	if ($matches[1] != "")
 		$avatar = $matches[1];
 
+	$link = "";
+	preg_match("/link='(.*?)'/ism", $attributes, $matches);
+	if ($matches[1] != "")
+		$link = $matches[1];
+
+	preg_match('/link="(.*?)"/ism', $attributes, $matches);
+	if ($matches[1] != "")
+		$link = $matches[1];
+
 	$shared_body = preg_replace("/\[share(.*?)\]\s?(.*?)\s?\[\/share\]\s?/ism","$2",$body);
 
 	if (($shared_body == "") OR ($profile == "") OR ($author == "") OR ($avatar == ""))
@@ -2540,6 +2682,7 @@ function api_share_as_retweet($a, $uid, &$item) {
 	$item["author-name"] = $author;
 	$item["author-link"] = $profile;
 	$item["author-avatar"] = $avatar;
+	$item["plink"] = $link;
 
 	return(true);
 
@@ -2697,9 +2840,6 @@ function api_best_nickname(&$contacts) {
 
 /*
 Not implemented by now:
-favorites
-favorites/create
-favorites/destroy
 statuses/retweets_of_me
 friendships/create
 friendships/destroy

include/bb2diaspora.php

@@ -5,7 +5,7 @@ require_once("include/event.php");
 require_once("library/markdown.php");
 require_once("include/html2bbcode.php");
 require_once("include/bbcode.php");
-require_once("include/markdownify/markdownify.php");
+require_once("library/html-to-markdown/HTML_To_Markdown.php");
 
 
 // we don't want to support a bbcode specific markdown interpreter
@@ -17,19 +17,21 @@ function diaspora2bb($s) {
 
 	$s = html_entity_decode($s,ENT_COMPAT,'UTF-8');
 
-	// Simply remove cr.
+	// Remove CR to avoid problems with following code
 	$s = str_replace("\r","",$s);
 
-	// <br/> is invalid. Replace it with the valid expression
+	$s = str_replace("\n"," \n",$s);
-	$s = str_replace(array("<br/>", "</p>", "<p>", '<p dir="ltr">'),array("<br />", "<br />", "<br />", "<br />"),$s);
 
-	$s = preg_replace('/\@\{(.+?)\; (.+?)\@(.+?)\}/','@[url=https://$3/u/$2]$1[/url]',$s);
+	// The parser cannot handle paragraphs correctly
+	$s = str_replace(array("</p>", "<p>", '<p dir="ltr">'),array("<br>", "<br>", "<br>"),$s);
 
 	// Escaping the hash tags
 	$s = preg_replace('/\#([^\s\#])/','&#35;$1',$s);
 
 	$s = Markdown($s);
 
+	$s = preg_replace('/\@\{(.+?)\; (.+?)\@(.+?)\}/','@[url=https://$3/u/$2]$1[/url]',$s);
+
 	$s = str_replace('&#35;','#',$s);
 
 	$s = html2bbcode($s);
@@ -56,6 +58,8 @@ function diaspora2bb($s) {
 
 function bb2diaspora($Text,$preserve_nl = false, $fordiaspora = true) {
 
+	$a = get_app();
+
 	$OriginalText = $Text;
 
 	// Since Diaspora is creating a summary for links, this function removes them before posting
@@ -88,40 +92,22 @@ function bb2diaspora($Text,$preserve_nl = false, $fordiaspora = true) {
 			$Text = $Text."<br />".$tagline;
 		}
 
-	} else {
+	} else
 		$Text = bbcode($Text, $preserve_nl, false, 4);
-		// Libertree doesn't convert a harizontal rule if there isn't a linefeed
+
-		$Text = str_replace("<hr />", "<br /><hr />", $Text);
+	// If a link is followed by a quote then there should be a newline before it
-	}
+	// Maybe we should make this newline at every time before a quote.
+	$Text = str_replace(array("</a><blockquote>"), array("</a><br><blockquote>"), $Text);
+
+	$stamp1 = microtime(true);
 
 	// Now convert HTML to Markdown
-	$md = new Markdownify(false, false, false);
+	$Text = new HTML_To_Markdown($Text);
-	$Text = $md->parseString($Text);
 
-	// The Markdownify converter converts underscores '_' in URLs to '\_', which
+	$a->save_timestamp($stamp1, "parser");
-	// messes up the URL. Manually fix these
-	$count = 1;
-	$pos = bb_find_open_close($Text, '[', ']', $count);
-	while($pos !== false) {
-		$start = substr($Text, 0, $pos['start']);
-		$subject = substr($Text, $pos['start'], $pos['end'] - $pos['start'] + 1);
-		$end = substr($Text, $pos['end'] + 1);
 
-		$subject = str_replace('\_', '_', $subject);
+	// Libertree has a problem with escaped hashtags.
-		$Text = $start . $subject . $end;
+	$Text = str_replace(array('\#'), array('#'), $Text);
-
-		$count++;
-		$pos = bb_find_open_close($Text, '[', ']', $count);
-	}
-
-	// If the text going into bbcode() has a plain URL in it, i.e.
-	// with no [url] tags around it, it will come out of parseString()
-	// looking like: <http://url.com>, which gets removed by strip_tags().
-	// So take off the angle brackets of any such URL
-	$Text = preg_replace("/<http(.*?)>/is", "http$1", $Text);
-
-	// Remove all unconverted tags
-	$Text = strip_tags($Text);
 
 	// Remove any leading or trailing whitespace, as this will mess up
 	// the Diaspora signature verification and cause the item to disappear

include/bbcode.php

@@ -41,7 +41,9 @@ function bb_attachment($Text, $plaintext = false, $tryoembed = true) {
 			if ($matches[1] != "")
 				$title = $matches[1];
 
-			$title = htmlentities($title, ENT_QUOTES, 'UTF-8', false);
+			//$title = htmlentities($title, ENT_QUOTES, 'UTF-8', false);
+			$title = bbcode(html_entity_decode($title, ENT_QUOTES, 'UTF-8'), false, false, true);
+			$title = str_replace(array("[", "]"), array("[", "]"), $title);
 
 			$image = "";
 			if ($type != "video") {
@@ -166,6 +168,8 @@ function bb_remove_share_information($Text, $plaintext = false, $nolink = false)
 }
 
 function bb_cleanup_share($shared, $plaintext, $nolink) {
+	$shared[1] = trim($shared[1]);
+
 	if (!in_array($shared[2], array("type-link", "type-video")))
 		return($shared[0]);
 
@@ -176,7 +180,7 @@ function bb_cleanup_share($shared, $plaintext, $nolink) {
 		return($shared[0]);
 
 	if ($nolink)
-		return(trim($shared[1]));
+		return($shared[1]);
 
 	$title = "";
 	$link = "";
@@ -187,6 +191,9 @@ function bb_cleanup_share($shared, $plaintext, $nolink) {
 	if (isset($bookmark[1][0]))
 		$link = $bookmark[1][0];
 
+	if (($shared[1] != "") AND (strpos($title, $shared[1]) !== false))
+		$shared[1] = $title;
+
 	if (($title != "") AND ((strpos($shared[1],$title) !== false) OR
 		(similar_text($shared[1],$title) / strlen($title)) > 0.9))
 		$title = "";
@@ -502,9 +509,7 @@ function bb_ShareAttributes($share, $simplehtml) {
 			$text = $preshare.html_entity_decode("&#x2672; ", ENT_QUOTES, 'UTF-8').' '.$userid_compact.": <br />".$share[3];
 			break;
 		case 3: // Diaspora
-			$headline = '<div class="shared_header">';
+			$headline .= '<b>'.html_entity_decode("&#x2672; ", ENT_QUOTES, 'UTF-8').$userid.':</b><br />';
-			$headline .= '<span><b>'.html_entity_decode("&#x2672; ", ENT_QUOTES, 'UTF-8').$userid.':</b></span>';
-			$headline .= "</div>";
 
 			$text = trim($share[1]);
 
@@ -512,7 +517,7 @@ function bb_ShareAttributes($share, $simplehtml) {
 				$text .= "<hr />";
 
 			if (substr(normalise_link($link), 0, 19) != "http://twitter.com/") {
-				$text .= $headline.'<blockquote class="shared_content">'.trim($share[3])."</blockquote><br />";
+				$text .= $headline.'<blockquote>'.trim($share[3])."</blockquote><br />";
 
 				if ($link != "")
 					$text .= '<br /><a href="'.$link.'">[l]</a>';
@@ -617,6 +622,17 @@ function GetProfileUsername($profile, $username, $compact = false, $getnetwork =
 			return($username." (".$diaspora.")");
 	}
 
+	$red = preg_replace("=https?://(.*)/channel/(.*)=ism", "$2@$1", $profile);
+	if ($red != $profile) {
+		if ($getnetwork)
+			// red is identified as Diaspora - friendica can't connect directly to it
+			return(NETWORK_DIASPORA);
+		elseif ($compact)
+			return($red);
+		else
+			return($username." (".$red.")");
+	}
+
 	$StatusnetHost = preg_replace("=https?://(.*)/user/(.*)=ism", "$1", $profile);
 	if ($StatusnetHost != $profile) {
 		$StatusnetUser = preg_replace("=https?://(.*)/user/(.*)=ism", "$2", $profile);
@@ -651,12 +667,20 @@ function GetProfileUsername($profile, $username, $compact = false, $getnetwork =
 	return($username);
 }
 
+function bb_DiasporaLinks($match) {
+	$a = get_app();
+
+	return "[url=".$a->get_baseurl()."/display/".$match[1]."]".$match[2]."[/url]";
+}
+
 function bb_RemovePictureLinks($match) {
 	$text = Cache::get($match[1]);
 
 	if(is_null($text)){
 		$a = get_app();
 
+		$stamp1 = microtime(true);
+
 		$ch = @curl_init($match[1]);
 		@curl_setopt($ch, CURLOPT_NOBODY, true);
 		@curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
@@ -664,6 +688,8 @@ function bb_RemovePictureLinks($match) {
 		@curl_exec($ch);
 		$curl_info = @curl_getinfo($ch);
 
+		$a->save_timestamp($stamp1, "network");
+
 		if (substr($curl_info["content_type"], 0, 6) == "image/")
 			$text = "[url=".$match[1]."]".$match[1]."[/url]";
 		else {
@@ -695,7 +721,7 @@ function bb_RemovePictureLinks($match) {
 }
 
 function bb_expand_links($match) {
-	if (stristr($match[2], $match[3]) OR ($match[2] == $match[3]))
+	if (($match[3] == "") OR ($match[2] == $match[3]) OR stristr($match[2], $match[3]))
 		return ($match[1]."[url]".$match[2]."[/url]");
 	else
 		return ($match[1].$match[3]." [url]".$match[2]."[/url]");
@@ -707,6 +733,8 @@ function bb_CleanPictureLinksSub($match) {
 	if(is_null($text)){
 		$a = get_app();
 
+		$stamp1 = microtime(true);
+
 		$ch = @curl_init($match[1]);
 		@curl_setopt($ch, CURLOPT_NOBODY, true);
 		@curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
@@ -714,6 +742,8 @@ function bb_CleanPictureLinksSub($match) {
 		@curl_exec($ch);
 		$curl_info = @curl_getinfo($ch);
 
+		$a->save_timestamp($stamp1, "network");
+
 		// if its a link to a picture then embed this picture
 		if (substr($curl_info["content_type"], 0, 6) == "image/")
 			$text = "[img]".$match[1]."[/img]";
@@ -755,8 +785,6 @@ function bb_CleanPictureLinks($text) {
 
 function bbcode($Text,$preserve_nl = false, $tryoembed = true, $simplehtml = false, $forplaintext = false) {
 
-	$stamp1 = microtime(true);
-
 	$a = get_app();
 
 	// Hide all [noparse] contained bbtags by spacefying them
@@ -818,8 +846,10 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true, $simplehtml = fal
 
 	// removing multiplicated newlines
 	if (get_config("system", "remove_multiplicated_lines")) {
-		$search = array("\n\n\n", "\n ", " \n", "[/quote]\n\n", "\n[/quote]", "[/li]\n", "\n[li]", "\n[ul]", "[/ul]\n", "\n\n[share ", "[/attachment]\n");
+		$search = array("\n\n\n", "\n ", " \n", "[/quote]\n\n", "\n[/quote]", "[/li]\n", "\n[li]", "\n[ul]", "[/ul]\n", "\n\n[share ", "[/attachment]\n",
-		$replace = array("\n\n", "\n", "\n", "[/quote]\n", "[/quote]", "[/li]", "[li]", "[ul]", "[/ul]", "\n[share ", "[/attachment]");
+				"\n[h1]", "[/h1]\n", "\n[h2]", "[/h2]\n", "\n[h3]", "[/h3]\n", "\n[h4]", "[/h4]\n", "\n[h5]", "[/h5]\n", "\n[h6]", "[/h6]\n");
+		$replace = array("\n\n", "\n", "\n", "[/quote]\n", "[/quote]", "[/li]", "[li]", "[ul]", "[/ul]", "\n[share ", "[/attachment]",
+				"[h1]", "[/h1]", "[h2]", "[/h2]", "[h3]", "[/h3]", "[h4]", "[/h4]", "[h5]", "[/h5]", "[h6]", "[/h6]");
 		do {
 			$oldtext = $Text;
 			$Text = str_replace($search, $replace, $Text);
@@ -867,6 +897,9 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true, $simplehtml = fal
 	else
 		$Text = preg_replace("/\[bookmark\=([^\]]*)\](.*?)\[\/bookmark\]/ism",'[url=$1]$2[/url]',$Text);
 
+	// Handle Diaspora posts
+	$Text = preg_replace_callback("&\[url=/posts/([^\[\]]*)\](.*)\[\/url\]&Usi", 'bb_DiasporaLinks', $Text);
+
 	// if the HTML is used to generate plain text, then don't do this search, but replace all URL of that kind to text
 	if (!$forplaintext)
 		$Text = preg_replace("/([^\]\='".'"'."]|^)(https?\:\/\/[a-zA-Z0-9\:\/\-\?\&\;\.\=\_\~\#\%\$\!\+\,]+)/ism", '$1<a href="$2" target="_blank">$2</a>', $Text);
@@ -896,6 +929,14 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true, $simplehtml = fal
 	$Text = preg_replace("/\[mail\]([$MAILSearchString]*)\[\/mail\]/", '<a href="mailto:$1">$1</a>', $Text);
 	$Text = preg_replace("/\[mail\=([$MAILSearchString]*)\](.*?)\[\/mail\]/", '<a href="mailto:$1">$2</a>', $Text);
 
+	// Check for headers
+	$Text = preg_replace("(\[h1\](.*?)\[\/h1\])ism",'<h1>$1</h1>',$Text);
+	$Text = preg_replace("(\[h2\](.*?)\[\/h2\])ism",'<h2>$1</h2>',$Text);
+	$Text = preg_replace("(\[h3\](.*?)\[\/h3\])ism",'<h3>$1</h3>',$Text);
+	$Text = preg_replace("(\[h4\](.*?)\[\/h4\])ism",'<h4>$1</h4>',$Text);
+	$Text = preg_replace("(\[h5\](.*?)\[\/h5\])ism",'<h5>$1</h5>',$Text);
+	$Text = preg_replace("(\[h6\](.*?)\[\/h6\])ism",'<h6>$1</h6>',$Text);
+
 	// Check for bold text
 	$Text = preg_replace("(\[b\](.*?)\[\/b\])ism",'<strong>$1</strong>',$Text);
 
@@ -1076,8 +1117,8 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true, $simplehtml = fal
 					'<a href="https://www.youtube.com/watch?v=$1" target="_blank">https://www.youtube.com/watch?v=$1</a>', $Text);
 
 	if ($tryoembed) {
-		$Text = preg_replace_callback("/\[vimeo\](https?:\/\/player.vimeo.com\/video\/[0-9]+).*?\[\/vimeo\]/ism",'tryoembed',$Text); 
+		$Text = preg_replace_callback("/\[vimeo\](https?:\/\/player.vimeo.com\/video\/[0-9]+).*?\[\/vimeo\]/ism",'tryoembed',$Text);
-		$Text = preg_replace_callback("/\[vimeo\](https?:\/\/vimeo.com\/[0-9]+).*?\[\/vimeo\]/ism",'tryoembed',$Text); 
+		$Text = preg_replace_callback("/\[vimeo\](https?:\/\/vimeo.com\/[0-9]+).*?\[\/vimeo\]/ism",'tryoembed',$Text);
 	}
 
 	$Text = preg_replace("/\[vimeo\]https?:\/\/player.vimeo.com\/video\/([0-9]+)(.*?)\[\/vimeo\]/ism",'[vimeo]$1[/vimeo]',$Text); 
@@ -1091,7 +1132,6 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true, $simplehtml = fal
 
 //	$Text = preg_replace("/\[youtube\](.*?)\[\/youtube\]/", '<object width="425" height="350" type="application/x-shockwave-flash" data="http://www.youtube.com/v/$1" ><param name="movie" value="http://www.youtube.com/v/$1"></param><!--[if IE]><embed src="http://www.youtube.com/v/$1" type="application/x-shockwave-flash" width="425" height="350" /><![endif]--></object>', $Text);
 
-
 	// oembed tag
 	$Text = oembed_bbcode2html($Text);
 
@@ -1161,16 +1201,8 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true, $simplehtml = fal
 	//$Text = str_replace('<br /><li>','<li>', $Text);
 	//	$Text = str_replace('<br /><ul','<ul ', $Text);
 
-	// Remove all hashtag addresses
-/*	if (!$tryoembed AND get_config("system", "remove_hashtags_on_export")) {
-		$pattern = '/#<a.*?href="(.*?)".*?>(.*?)<\/a>/is';
-		$Text = preg_replace($pattern, '#$2', $Text);
-	}
-*/
 	call_hooks('bbcode',$Text);
 
-	$a->save_timestamp($stamp1, "parser");
+	return trim($Text);
-
-	return $Text;
 }
 ?>

include/conversation.php

@@ -203,12 +203,11 @@ function localize_item(&$item){
 
 		// we can't have a translation string with three positions but no distinguishable text
 		// So here is the translate string.
-
 		$txt = t('%1$s poked %2$s');
-
+		
 		// now translate the verb
-
+		$poked_t = trim(sprintf($txt, "",""));
-		$txt = str_replace( t('poked'), t($verb), $txt);
+		$txt = str_replace( $poked_t, t($verb), $txt);
 
 		// then do the sprintf on the translation string
 
@@ -1102,16 +1101,16 @@ function status_editor($a,$x, $notes_cid = 0, $popup=false) {
 		'$shortsetloc' => t('set location'),
 		'$noloc' => t('Clear browser location'),
 		'$shortnoloc' => t('clear location'),
-		'$title' => "",
+		'$title' => $x['title'],
 		'$placeholdertitle' => t('Set title'),
-		'$category' => "",
+		'$category' => $x['category'],
 		'$placeholdercategory' => (feature_enabled(local_user(),'categories') ? t('Categories (comma-separated list)') : ''),
 		'$wait' => t('Please wait'),
 		'$permset' => t('Permission settings'),
 		'$shortpermset' => t('permissions'),
 		'$ptyp' => (($notes_cid) ? 'note' : 'wall'),
-		'$content' => '',
+		'$content' => $x['content'],
-		'$post_id' => '',
+		'$post_id' => $x['post_id'],
 		'$baseurl' => $a->get_baseurl(true),
 		'$defloc' => $x['default_location'],
 		'$visitor' => $x['visitor'],

include/cronhooks.php

@@ -66,6 +66,6 @@ function cronhooks_run(&$argv, &$argc){
 }
 
 if (array_search(__file__,get_included_files())===0){
-  cronhooks_run($argv,$argc);
+  cronhooks_run($_SERVER["argv"],$_SERVER["argc"]);
   killme();
 }

include/dbstructure.php

@@ -106,29 +106,17 @@ function table_structure($table) {
 }
 
 function print_structure($database) {
+	echo "-- ------------------------------------------\n";
+	echo "-- ".FRIENDICA_PLATFORM." ".FRIENDICA_VERSION." (".FRIENDICA_CODENAME,")\n";
+	echo "-- DB_UPDATE_VERSION ".DB_UPDATE_VERSION."\n";
+	echo "-- ------------------------------------------\n\n\n";
 	foreach ($database AS $name => $structure) {
-		echo "\t".'$database["'.$name."\"] = array(\n";
+		echo "--\n";
+		echo "-- TABLE $name\n";
+		echo "--\n";
+		db_create_table($name, $structure['fields'], true, false, $structure["indexes"]);
 
-		echo "\t\t\t".'"fields" => array('."\n";
+		echo "\n";
-		foreach ($structure["fields"] AS $fieldname => $parameters) {
-			echo "\t\t\t\t\t".'"'.$fieldname.'" => array(';
-
-			$data = "";
-			foreach ($parameters AS $name => $value) {
-				if ($data != "")
-					$data .= ", ";
-				$data .= '"'.$name.'" => "'.$value.'"';
-			}
-
-			echo $data."),\n";
-		}
-		echo "\t\t\t\t\t),\n";
-		echo "\t\t\t".'"indexes" => array('."\n";
-		foreach ($structure["indexes"] AS $indexname => $fieldnames) {
-			echo "\t\t\t\t\t".'"'.$indexname.'" => array("'.implode($fieldnames, '","').'"'."),\n";
-		}
-		echo "\t\t\t\t\t)\n";
-		echo "\t\t\t);\n";
 	}
 }
 
@@ -231,9 +219,13 @@ function db_field_command($parameters, $create = true) {
 	if ($parameters["not null"])
 		$fieldstruct .= " NOT NULL";
 
-	if (isset($parameters["default"]))
+	if (isset($parameters["default"])){
-		$fieldstruct .= " DEFAULT '".$parameters["default"]."'";
+		if (strpos(strtolower($parameters["type"]),"int")!==false) {
-
+			$fieldstruct .= " DEFAULT ".$parameters["default"];
+		} else {
+			$fieldstruct .= " DEFAULT '".$parameters["default"]."'";
+		}
+	}
 	if ($parameters["extra"] != "")
 		$fieldstruct .= " ".$parameters["extra"];
 
@@ -243,20 +235,28 @@ function db_field_command($parameters, $create = true) {
 	return($fieldstruct);
 }
 
-function db_create_table($name, $fields, $verbose, $action) {
+function db_create_table($name, $fields, $verbose, $action, $indexes=null) {
 	global $a, $db;
 
 	$r = true;
 
 	$sql = "";
+	$sql_rows = array();
 	foreach($fields AS $fieldname => $field) {
-		if ($sql != "")
+		$sql_rows[] = "`".dbesc($fieldname)."` ".db_field_command($field);
-			$sql .= ",\n";
-
-		$sql .= "`".dbesc($fieldname)."` ".db_field_command($field);
 	}
 
-	$sql = sprintf("CREATE TABLE IF NOT EXISTS `%s` (\n", dbesc($name)).$sql."\n) DEFAULT CHARSET=utf8";
+	if (!is_null($indexes)) {
+
+		foreach ($indexes AS $indexname => $fieldnames) {
+			$sql_index = db_create_index($indexname, $fieldnames, "");
+			if (!is_null($sql_index)) $sql_rows[] = $sql_index;
+		}
+	}
+
+	$sql = implode(",\n\t", $sql_rows);
+
+	$sql = sprintf("CREATE TABLE IF NOT EXISTS `%s` (\n\t", dbesc($name)).$sql."\n) DEFAULT CHARSET=utf8";
 
 	if ($verbose)
 		echo $sql.";\n";
@@ -282,7 +282,7 @@ function db_drop_index($indexname) {
 	return($sql);
 }
 
-function db_create_index($indexname, $fieldnames) {
+function db_create_index($indexname, $fieldnames, $method="ADD") {
 
 	if ($indexname == "PRIMARY")
 		return;
@@ -298,7 +298,13 @@ function db_create_index($indexname, $fieldnames) {
 			$names .= "`".dbesc($fieldname)."`";
 	}
 
-	$sql = sprintf("ADD INDEX `%s` (%s)", dbesc($indexname), $names);
+	$method = strtoupper(trim($method));
+	if ($method!="" && $method!="ADD") {
+		throw new Exception("Invalid parameter 'method' in db_create_index(): '$method'");
+		killme();
+	}
+
+	$sql = sprintf("%s INDEX `%s` (%s)", $method, dbesc($indexname), $names);
 	return($sql);
 }
 
@@ -413,6 +419,10 @@ function db_definition() {
 					"network" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
 					"name" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
 					"nick" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
+					"location" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
+					"about" => array("type" => "text", "not null" => "1"),
+					"keywords" => array("type" => "text", "not null" => "1"),
+					"gender" => array("type" => "varchar(32)", "not null" => "1", "default" => ""),
 					"attag" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
 					"photo" => array("type" => "text", "not null" => "1"),
 					"thumb" => array("type" => "text", "not null" => "1"),
@@ -616,6 +626,13 @@ function db_definition() {
 					"nurl" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
 					"photo" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
 					"connect" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
+					"updated" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"),
+					"location" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
+					"about" => array("type" => "text", "not null" => "1"),
+					"keywords" => array("type" => "text", "not null" => "1"),
+					"gender" => array("type" => "varchar(32)", "not null" => "1", "default" => ""),
+					"network" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
+					"generation" => array("type" => "tinyint(3)", "not null" => "1", "default" => "0"),
 					),
 			"indexes" => array(
 					"PRIMARY" => array("id"),
@@ -765,6 +782,9 @@ function db_definition() {
 					"last-child" => array("type" => "tinyint(1) unsigned", "not null" => "1", "default" => "1"),
 					"mention" => array("type" => "tinyint(1)", "not null" => "1", "default" => "0"),
 					"network" => array("type" => "varchar(32)", "not null" => "1", "default" => ""),
+					"rendered-hash" => array("type" => "varchar(32)", "not null" => "1", "default" => ""),
+					"rendered-html" => array("type" => "mediumtext", "not null" => "1"),
+					"global" => array("type" => "tinyint(1)", "not null" => "1", "default" => "0"),
 					),
 			"indexes" => array(
 					"PRIMARY" => array("id"),
@@ -823,6 +843,7 @@ function db_definition() {
 					"id" => array("type" => "int(11)", "not null" => "1", "extra" => "auto_increment", "primary" => "1"),
 					"name" => array("type" => "varchar(128)", "not null" => "1", "default" => ""),
 					"locked" => array("type" => "tinyint(1)", "not null" => "1", "default" => "0"),
+					"created" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"),
 					),
 			"indexes" => array(
 					"PRIMARY" => array("id"),
@@ -1176,6 +1197,10 @@ function db_definition() {
 					"type" => array("type" => "tinyint(3) unsigned", "not null" => "1", "default" => "0"),
 					"term" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
 					"url" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
+					"guid" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
+					"created" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"),
+					"received" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"),
+					"global" => array("type" => "tinyint(1)", "not null" => "1", "default" => "0"),
 					"aid" => array("type" => "int(10) unsigned", "not null" => "1", "default" => "0"),
 					"uid" => array("type" => "int(10) unsigned", "not null" => "1", "default" => "0"),
 					),
@@ -1184,8 +1209,9 @@ function db_definition() {
 					"oid_otype_type_term" => array("oid","otype","type","term"),
 					"uid_term_tid" => array("uid","term","tid"),
 					"type_term" => array("type","term"),
-					"uid_otype_type_term_tid" => array("uid","otype","type","term","tid"),
+					"uid_otype_type_term_global_created" => array("uid","otype","type","term","global","created"),
 					"otype_type_term_tid" => array("otype","type","term","tid"),
+					"guid" => array("guid"),
 					)
 			);
 	$database["thread"] = array(
@@ -1247,6 +1273,8 @@ function db_definition() {
 					"nick" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
 					"name" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
 					"avatar" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
+					"location" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
+					"about" => array("type" => "text", "not null" => "1"),
 					),
 			"indexes" => array(
 					"PRIMARY" => array("id"),
@@ -1335,10 +1363,32 @@ function dbstructure_run(&$argv, &$argc) {
 			unset($db_host, $db_user, $db_pass, $db_data);
 	}
 
-	update_structure(true, true);
+	if ($argc==2) {
+		switch ($argv[1]) {
+			case "update":
+				update_structure(true, true);
+				return;
+			case "dumpsql":
+				print_structure(db_definition());
+				return;
+		}
+	}
+
+
+	// print help
+	echo $argv[0]." <command>\n";
+	echo "\n";
+	echo "commands:\n";
+	echo "update		update database schema\n";
+	echo "dumpsql		dump database schema\n";
+	return;
+
+
+
+
 }
 
 if (array_search(__file__,get_included_files())===0){
-	dbstructure_run($argv,$argc);
+	dbstructure_run($_SERVER["argv"],$_SERVER["argc"]);
 	killme();
 }

include/dbupdate.php

@@ -8,7 +8,7 @@ function dbupdate_run(&$argv, &$argc) {
 	if(is_null($a)){
 		$a = new App;
 	}
-  
+
 	if(is_null($db)) {
 		@include(".htconfig.php");
 		require_once("include/dba.php");
@@ -23,7 +23,6 @@ function dbupdate_run(&$argv, &$argc) {
 }
 
 if (array_search(__file__,get_included_files())===0){
-  dbupdate_run($argv,$argc);
+  dbupdate_run($_SERVER["argv"],$_SERVER["argc"]);
   killme();
 }
-

include/delivery.php

@@ -565,6 +565,6 @@ function delivery_run(&$argv, &$argc){
 }
 
 if (array_search(__file__,get_included_files())===0){
-  delivery_run($argv,$argc);
+  delivery_run($_SERVER["argv"],$_SERVER["argc"]);
   killme();
 }

include/diaspora.php

@@ -6,6 +6,7 @@ require_once('include/bb2diaspora.php');
 require_once('include/contact_selectors.php');
 require_once('include/queue_fn.php');
 require_once('include/lock.php');
+require_once('include/threads.php');
 
 function diaspora_dispatch_public($msg) {
 
@@ -589,7 +590,7 @@ function diaspora_request($importer,$xml) {
 			intval($importer['uid'])
 		);
 
-		if((count($r)) && (! $r[0]['hide-friends']) && (! $contact['hidden'])) {
+		if((count($r)) && (!$r[0]['hide-friends']) && (!$contact['hidden']) && intval(get_pconfig($importer['uid'],'system','post_newfriend'))) {
 			require_once('include/items.php');
 
 			$self = q("SELECT * FROM `contact` WHERE `self` = 1 AND `uid` = %d LIMIT 1",
@@ -833,31 +834,6 @@ function diaspora_post($importer,$xml,$msg) {
 
 	$str_tags = '';
 
-	$tags = get_tags($body);
-
-	if(count($tags)) {
-		foreach($tags as $tag) {
-			if(strpos($tag,'#') === 0) {
-				if(strpos($tag,'[url='))
-					continue;
-
-				// don't link tags that are already embedded in links
-
-				if(preg_match('/\[(.*?)' . preg_quote($tag,'/') . '(.*?)\]/',$body))
-					continue;
-				if(preg_match('/\[(.*?)\]\((.*?)' . preg_quote($tag,'/') . '(.*?)\)/',$body))
-					continue;
-
-				$basetag = str_replace('_',' ',substr($tag,1));
-				$body = str_replace($tag,'#[url=' . $a->get_baseurl() . '/search?tag=' . rawurlencode($basetag) . ']' . $basetag . '[/url]',$body);
-				if(strlen($str_tags))
-					$str_tags .= ',';
-				$str_tags .= '#[url=' . $a->get_baseurl() . '/search?tag=' . rawurlencode($basetag) . ']' . $basetag . '[/url]';
-				continue;
-			}
-		}
-	}
-
 	$cnt = preg_match_all('/@\[url=(.*?)\[\/url\]/ism',$body,$matches,PREG_SET_ORDER);
 	if($cnt) {
 		foreach($matches as $mtch) {
@@ -895,19 +871,170 @@ function diaspora_post($importer,$xml,$msg) {
 
 	$datarray['visible'] = ((strlen($body)) ? 1 : 0);
 
+	DiasporaFetchGuid($datarray);
 	$message_id = item_store($datarray);
 
-	//if($message_id) {
-	//	q("update item set plink = '%s' where id = %d",
-	//		dbesc($a->get_baseurl() . '/display/' . $importer['nickname'] . '/' . $message_id),
-	//		intval($message_id)
-	//	);
-	//}
-
 	return;
 
 }
 
+function DiasporaFetchGuid($item) {
+	preg_replace_callback("&\[url=/posts/([^\[\]]*)\](.*)\[\/url\]&Usi",
+		function ($match) use ($item){
+			return(DiasporaFetchGuidSub($match, $item));
+		},$item["body"]);
+}
+
+function DiasporaFetchGuidSub($match, $item) {
+	$a = get_app();
+
+	$author = parse_url($item["author-link"]);
+	$authorserver = $author["scheme"]."://".$author["host"];
+
+	$owner = parse_url($item["owner-link"]);
+	$ownerserver = $owner["scheme"]."://".$owner["host"];
+
+	if (!diaspora_store_by_guid($match[1], $authorserver))
+		diaspora_store_by_guid($match[1], $ownerserver);
+}
+
+function diaspora_store_by_guid($guid, $server) {
+	require_once("include/Contact.php");
+
+	logger("fetching item ".$guid." from ".$server, LOGGER_DEBUG);
+
+	$item = diaspora_fetch_message($guid, $server);
+
+	if (!$item)
+		return false;
+
+	$body = $item["body"];
+	$str_tags = $item["tag"];
+	$app = $item["app"];
+	$created = $item["created"];
+	$author = $item["author"];
+	$guid = $item["guid"];
+	$private = $item["private"];
+
+	$message_id = $author.':'.$guid;
+	$r = q("SELECT `id` FROM `item` WHERE `uid` = 0 AND `uri` = '%s' AND `guid` = '%s' LIMIT 1",
+		dbesc($message_id),
+		dbesc($guid)
+	);
+	if(count($r))
+		return $r[0]["id"];
+
+	$person = find_diaspora_person_by_handle($author);
+
+        $datarray = array();
+	$datarray['uid'] = 0;
+	$datarray['contact-id'] = get_contact($person['url'], 0);
+	$datarray['wall'] = 0;
+	$datarray['network']  = NETWORK_DIASPORA;
+	$datarray['guid'] = $guid;
+	$datarray['uri'] = $datarray['parent-uri'] = $message_id;
+	$datarray['changed'] = $datarray['created'] = $datarray['edited'] = datetime_convert('UTC','UTC',$created);
+	$datarray['private'] = $private;
+	$datarray['parent'] = 0;
+	$datarray['plink'] = 'https://'.substr($author,strpos($author,'@')+1).'/posts/'.$guid;
+	$datarray['author-name'] = $person['name'];
+	$datarray['author-link'] = $person['url'];
+	$datarray['author-avatar'] = ((x($person,'thumb')) ? $person['thumb'] : $person['photo']);
+	$datarray['owner-name'] = $datarray['author-name'];
+	$datarray['owner-link'] = $datarray['author-link'];
+	$datarray['owner-avatar'] = $datarray['author-avatar'];
+	$datarray['body'] = $body;
+	$datarray['tag'] = $str_tags;
+	$datarray['app']  = $app;
+	$datarray['visible'] = ((strlen($body)) ? 1 : 0);
+
+	DiasporaFetchGuid($datarray);
+	$message_id = item_store($datarray);
+
+	// To-Do:
+	// Looking if there is some subscribe mechanism in Diaspora to get all comments for this post
+
+	return $message_id;
+}
+
+function diaspora_fetch_message($guid, $server, $level = 0) {
+
+	if ($level > 5)
+		return false;
+
+	$a = get_app();
+
+	// This will not work if the server is not a Diaspora server
+	$source_url = $server.'/p/'.$guid.'.xml';
+	$x = fetch_url($source_url);
+	if(!$x)
+		return false;
+
+	$x = str_replace(array('<activity_streams-photo>','</activity_streams-photo>'),array('<asphoto>','</asphoto>'),$x);
+	$source_xml = parse_xml_string($x,false);
+
+	$item = array();
+	$item["app"] = 'Diaspora';
+	$item["guid"] = $guid;
+	$body = "";
+
+	if ($source_xml->post->status_message->created_at)
+		$item["created"] = unxmlify($source_xml->post->status_message->created_at);
+
+	if ($source_xml->post->status_message->provider_display_name)
+		$item["app"] = unxmlify($source_xml->post->status_message->provider_display_name);
+
+	if ($source_xml->post->status_message->diaspora_handle)
+		$item["author"] = unxmlify($source_xml->post->status_message->diaspora_handle);
+
+	if ($source_xml->post->status_message->guid)
+		$item["guid"] = unxmlify($source_xml->post->status_message->guid);
+
+	$item["private"] = (unxmlify($source_xml->post->status_message->public) == 'false');
+
+	if(strlen($source_xml->post->asphoto->objectId) && ($source_xml->post->asphoto->objectId != 0) && ($source_xml->post->asphoto->image_url)) {
+		$body = '[url=' . notags(unxmlify($source_xml->post->asphoto->image_url)) . '][img]' . notags(unxmlify($source_xml->post->asphoto->objectId)) . '[/img][/url]' . "\n";
+		$body = scale_external_images($body,false);
+	} elseif($source_xml->post->asphoto->image_url) {
+		$body = '[img]' . notags(unxmlify($source_xml->post->asphoto->image_url)) . '[/img]' . "\n";
+		$body = scale_external_images($body);
+	} elseif($source_xml->post->status_message) {
+		$body = diaspora2bb($source_xml->post->status_message->raw_message);
+
+		// Checking for embedded pictures
+		if($source_xml->post->status_message->photo->remote_photo_path AND
+			$source_xml->post->status_message->photo->remote_photo_name) {
+
+			$remote_photo_path = notags(unxmlify($source_xml->post->status_message->photo->remote_photo_path));
+			$remote_photo_name = notags(unxmlify($source_xml->post->status_message->photo->remote_photo_name));
+
+			$body = '[img]'.$remote_photo_path.$remote_photo_name.'[/img]'."\n".$body;
+
+			logger('embedded picture link found: '.$body, LOGGER_DEBUG);
+		}
+
+		$body = scale_external_images($body);
+
+		// Add OEmbed and other information to the body
+		$body = add_page_info_to_body($body, false, true);
+	} elseif($source_xml->post->reshare) {
+		// Reshare of a reshare
+		return diaspora_fetch_message($source_xml->post->reshare->root_guid, $server, ++$level);
+	} else {
+		// Maybe it is a reshare of a photo that will be delivered at a later time (testing)
+		logger('no content found: '.print_r($source_xml,true));
+		$body = "";
+	}
+
+	if ($body == "")
+		return false;
+
+	$item["tag"] = '';
+	$item["body"] = $body;
+
+	return $item;
+}
+
 function diaspora_reshare($importer,$xml,$msg) {
 
 	logger('diaspora_reshare: init: ' . print_r($xml,true));
@@ -945,70 +1072,66 @@ function diaspora_reshare($importer,$xml,$msg) {
 	$orig_author = notags(unxmlify($xml->root_diaspora_id));
 	$orig_guid = notags(unxmlify($xml->root_guid));
 
-	$source_url = 'https://' . substr($orig_author,strpos($orig_author,'@')+1) . '/p/' . $orig_guid . '.xml';
+	$create_original_post = false;
-	$orig_url = 'https://'.substr($orig_author,strpos($orig_author,'@')+1).'/posts/'.$orig_guid;
+
-	$x = fetch_url($source_url);
+	// Do we already have this item?
-	if(! $x)
+	$r = q("SELECT `body`, `tag`, `app`, `author-link`, `plink` FROM `item` WHERE `guid` = '%s' AND `visible` AND NOT `deleted` AND `body` != '' LIMIT 1",
-		$x = fetch_url(str_replace('https://','http://',$source_url));
+		dbesc($orig_guid),
-	if(! $x) {
+		dbesc(NETWORK_DIASPORA)
-		logger('diaspora_reshare: unable to fetch source url ' . $source_url);
+	);
-		return;
+	if(count($r)) {
+		logger('reshared message '.orig_guid." reshared by ".$guid.' already exists on system: '.$orig_url);
+
+		// Maybe it is already a reshared item?
+		// Then refetch the content, since there can be many side effects with reshared posts from other networks or reshares from reshares
+		require_once('include/api.php');
+		if (api_share_as_retweet($r[0]))
+			$r = array();
+		else
+			$orig_url = $a->get_baseurl().'/display/'.$orig_guid;
 	}
-	logger('diaspora_reshare: source: ' . $x);
 
-	$x = str_replace(array('<activity_streams-photo>','</activity_streams-photo>'),array('<asphoto>','</asphoto>'),$x);
+	if (!count($r)) {
-	$source_xml = parse_xml_string($x,false);
+		$body = "";
+		$str_tags = "";
+		$app = "";
 
-	if(strlen($source_xml->post->asphoto->objectId) && ($source_xml->post->asphoto->objectId != 0) && ($source_xml->post->asphoto->image_url)) {
+		$orig_url = 'https://'.substr($orig_author,strpos($orig_author,'@')+1).'/posts/'.$orig_guid;
-		$body = '[url=' . notags(unxmlify($source_xml->post->asphoto->image_url)) . '][img]' . notags(unxmlify($source_xml->post->asphoto->objectId)) . '[/img][/url]' . "\n";
-		$body = scale_external_images($body,false);
-	}
-	elseif($source_xml->post->asphoto->image_url) {
-		$body = '[img]' . notags(unxmlify($source_xml->post->asphoto->image_url)) . '[/img]' . "\n";
-		$body = scale_external_images($body);
-	}
-	elseif($source_xml->post->status_message) {
-		$body = diaspora2bb($source_xml->post->status_message->raw_message);
 
-		// Checking for embedded pictures
+		$server = 'https://'.substr($orig_author,strpos($orig_author,'@')+1);
-		if($source_xml->post->status_message->photo->remote_photo_path AND
+		logger('1st try: reshared message '.$orig_guid." reshared by ".$guid.' will be fetched from original server: '.$server);
-			$source_xml->post->status_message->photo->remote_photo_name) {
+		$item = diaspora_fetch_message($orig_guid, $server);
 
-			$remote_photo_path = notags(unxmlify($source_xml->post->status_message->photo->remote_photo_path));
+		if (!$item) {
-			$remote_photo_name = notags(unxmlify($source_xml->post->status_message->photo->remote_photo_name));
+			$server = 'https://'.substr($diaspora_handle,strpos($diaspora_handle,'@')+1);
-
+			logger('2nd try: reshared message '.$orig_guid." reshared by ".$guid." will be fetched from sharer's server: ".$server);
-			$body = '[img]'.$remote_photo_path.$remote_photo_name.'[/img]'."\n".$body;
+			$item = diaspora_fetch_message($orig_guid, $server);
-
+		}
-			logger('diaspora_reshare: embedded picture link found: '.$body, LOGGER_DEBUG);
+		if (!$item) {
+			$server = 'http://'.substr($orig_author,strpos($orig_author,'@')+1);
+			logger('3rd try: reshared message '.$orig_guid." reshared by ".$guid.' will be fetched from original server: '.$server);
+			$item = diaspora_fetch_message($orig_guid, $server);
+		}
+		if (!$item) {
+			$server = 'http://'.substr($diaspora_handle,strpos($diaspora_handle,'@')+1);
+			logger('4th try: reshared message '.$orig_guid." reshared by ".$guid." will be fetched from sharer's server: ".$server);
+			$item = diaspora_fetch_message($orig_guid, $server);
 		}
 
-		$body = scale_external_images($body);
+		if ($item) {
-
+			$body = $item["body"];
-		// Add OEmbed and other information to the body
+			$str_tags = $item["tag"];
-		$body = add_page_info_to_body($body, false, true);
+			$app = $item["app"];
+			$orig_created = $item["created"];
+			$orig_author = $item["author"];
+			$orig_guid = $item["guid"];
+			$create_original_post = ($body != "");
+			$orig_url = $a->get_baseurl()."/display/".$orig_guid;
+		}
 	}
-	else {
-		// Maybe it is a reshare of a photo that will be delivered at a later time (testing)
-		logger('diaspora_reshare: no reshare content found: ' . print_r($source_xml,true));
-		$body = "";
-		//return;
-	}
-
-	//if(! $body) {
-	//	logger('diaspora_reshare: empty body: source= ' . $x);
-	//	return;
-	//}
 
 	$person = find_diaspora_person_by_handle($orig_author);
 
-	/*if(is_array($person) && x($person,'name') && x($person,'url'))
-		$details = '[url=' . $person['url'] . ']' . $person['name'] . '[/url]';
-	else
-		$details = $orig_author;
-
-	$prefix = html_entity_decode("&#x2672; ", ENT_QUOTES, 'UTF-8') . $details . "\n";*/
-
-
 	// allocate a guid on our system - we aren't fixing any collisions.
 	// we're ignoring them
 
@@ -1026,34 +1149,6 @@ function diaspora_reshare($importer,$xml,$msg) {
 
 	$datarray = array();
 
-	$str_tags = '';
-
-	$tags = get_tags($body);
-
-	if(count($tags)) {
-		foreach($tags as $tag) {
-			if(strpos($tag,'#') === 0) {
-				if(strpos($tag,'[url='))
-					continue;
-
-				// don't link tags that are already embedded in links
-
-				if(preg_match('/\[(.*?)' . preg_quote($tag,'/') . '(.*?)\]/',$body))
-					continue;
-				if(preg_match('/\[(.*?)\]\((.*?)' . preg_quote($tag,'/') . '(.*?)\)/',$body))
-					continue;
-
-
-				$basetag = str_replace('_',' ',substr($tag,1));
-				$body = str_replace($tag,'#[url=' . $a->get_baseurl() . '/search?tag=' . rawurlencode($basetag) . ']' . $basetag . '[/url]',$body);
-				if(strlen($str_tags))
-					$str_tags .= ',';
-				$str_tags .= '#[url=' . $a->get_baseurl() . '/search?tag=' . rawurlencode($basetag) . ']' . $basetag . '[/url]';
-				continue;
-			}
-		}
-	}
-
 	$plink = 'https://'.substr($diaspora_handle,strpos($diaspora_handle,'@')+1).'/posts/'.$guid;
 
 	$datarray['uid'] = $importer['uid'];
@@ -1073,6 +1168,8 @@ function diaspora_reshare($importer,$xml,$msg) {
 		$prefix = "[share author='".str_replace(array("'", "[", "]"), array("&#x27;", "&#x5B;", "&#x5D;"),$person['name']).
 				"' profile='".$person['url'].
 				"' avatar='".((x($person,'thumb')) ? $person['thumb'] : $person['photo']).
+				"' guid='".$orig_guid.
+				"' posted='".$orig_created.
 				"' link='".str_replace(array("'", "[", "]"), array("&#x27;", "&#x5B;", "&#x5D;"),$orig_url)."']";
 		$datarray['author-name'] = $contact['name'];
 		$datarray['author-link'] = $contact['url'];
@@ -1087,19 +1184,40 @@ function diaspora_reshare($importer,$xml,$msg) {
 	}
 
 	$datarray['tag'] = $str_tags;
-	$datarray['app']  = 'Diaspora';
+	$datarray['app']  = $app;
 
 	// if empty content it might be a photo that hasn't arrived yet. If a photo arrives, we'll make it visible. (testing)
 	$datarray['visible'] = ((strlen($body)) ? 1 : 0);
 
-	$message_id = item_store($datarray);
+	// Store the original item of a reshare
+	if ($create_original_post) {
+		require_once("include/Contact.php");
 
-	//if($message_id) {
+		$datarray2 = $datarray;
-	//	q("update item set plink = '%s' where id = %d",
+
-	//		dbesc($a->get_baseurl() . '/display/' . $importer['nickname'] . '/' . $message_id),
+		$datarray2['uid'] = 0;
-	//		intval($message_id)
+		$datarray2['contact-id'] = get_contact($person['url'], 0);
-	//	);
+		$datarray2['guid'] = $orig_guid;
-	//}
+		$datarray2['uri'] = $datarray2['parent-uri'] = $orig_author.':'.$orig_guid;
+		$datarray2['changed'] = $datarray2['created'] = $datarray2['edited'] = datetime_convert('UTC','UTC',$orig_created);
+		$datarray2['plink'] = 'https://'.substr($orig_author,strpos($orig_author,'@')+1).'/posts/'.$orig_guid;
+
+		$datarray2['author-name'] = $person['name'];
+		$datarray2['author-link'] = $person['url'];
+		$datarray2['author-avatar'] = ((x($person,'thumb')) ? $person['thumb'] : $person['photo']);
+		$datarray2['owner-name'] = $datarray2['author-name'];
+		$datarray2['owner-link'] = $datarray2['author-link'];
+		$datarray2['owner-avatar'] = $datarray2['author-avatar'];
+		$datarray2['body'] = $body;
+
+		DiasporaFetchGuid($datarray2);
+		$message_id = item_store($datarray2);
+
+		logger("Store original item ".$orig_guid." under message id ".$message_id);
+	}
+
+	DiasporaFetchGuid($datarray);
+	$message_id = item_store($datarray);
 
 	return;
 
@@ -1191,6 +1309,7 @@ function diaspora_asphoto($importer,$xml,$msg) {
 
 	$datarray['app']  = 'Diaspora/Cubbi.es';
 
+	DiasporaFetchGuid($datarray);
 	$message_id = item_store($datarray);
 
 	//if($message_id) {
@@ -1314,34 +1433,6 @@ function diaspora_comment($importer,$xml,$msg) {
 
 	$datarray = array();
 
-	$str_tags = '';
-
-	$tags = get_tags($body);
-
-	if(count($tags)) {
-		foreach($tags as $tag) {
-			if(strpos($tag,'#') === 0) {
-				if(strpos($tag,'[url='))
-					continue;
-
-				// don't link tags that are already embedded in links
-
-				if(preg_match('/\[(.*?)' . preg_quote($tag,'/') . '(.*?)\]/',$body))
-					continue;
-				if(preg_match('/\[(.*?)\]\((.*?)' . preg_quote($tag,'/') . '(.*?)\)/',$body))
-					continue;
-
-
-				$basetag = str_replace('_',' ',substr($tag,1));
-				$body = str_replace($tag,'#[url=' . $a->get_baseurl() . '/search?tag=' . rawurlencode($basetag) . ']' . $basetag . '[/url]',$body);
-				if(strlen($str_tags))
-					$str_tags .= ',';
-				$str_tags .= '#[url=' . $a->get_baseurl() . '/search?tag=' . rawurlencode($basetag) . ']' . $basetag . '[/url]';
-				continue;
-			}
-		}
-	}
-
 	$datarray['uid'] = $importer['uid'];
 	$datarray['contact-id'] = $contact['id'];
 	$datarray['type'] = 'remote-comment';
@@ -1365,12 +1456,12 @@ function diaspora_comment($importer,$xml,$msg) {
 	$datarray['author-link'] = $person['url'];
 	$datarray['author-avatar'] = ((x($person,'thumb')) ? $person['thumb'] : $person['photo']);
 	$datarray['body'] = $body;
-	$datarray['tag'] = $str_tags;
 
 	// We can't be certain what the original app is if the message is relayed.
 	if(($parent_item['origin']) && (! $parent_author_signature))
 		$datarray['app']  = 'Diaspora';
 
+	DiasporaFetchGuid($datarray);
 	$message_id = item_store($datarray);
 
 	//if($message_id) {
@@ -1765,11 +1856,12 @@ function diaspora_photo($importer,$xml,$msg,$attempt=1) {
 	                                   array($remote_photo_name, 'scaled_full_' . $remote_photo_name));
 
 	if(strpos($parent_item['body'],$link_text) === false) {
-		$r = q("update item set `body` = '%s', `visible` = 1 where `id` = %d and `uid` = %d",
+		$r = q("UPDATE `item` SET `body` = '%s', `visible` = 1 WHERE `id` = %d AND `uid` = %d",
 			dbesc($link_text . $parent_item['body']),
 			intval($parent_item['id']),
 			intval($parent_item['uid'])
 		);
+		update_thread($parent_item['id']);
 	}
 
 	return;
@@ -1844,7 +1936,7 @@ function diaspora_like($importer,$xml,$msg) {
 	if($positive === 'false') {
 		logger('diaspora_like: received a like with positive set to "false"');
 		logger('diaspora_like: unlike received with no corresponding like...ignoring');
-		return;	
+		return;
 	}
 
 
@@ -1860,26 +1952,28 @@ function diaspora_like($importer,$xml,$msg) {
 	     who sent the salmon
 	*/
 
-	$signed_data = $guid . ';' . $target_type . ';' . $parent_guid . ';' . $positive . ';' . $diaspora_handle;
+	// Diaspora has changed the way they are signing the likes.
+	// Just to make sure that we don't miss any likes we will check the old and the current way.
+	$old_signed_data = $guid . ';' . $target_type . ';' . $parent_guid . ';' . $positive . ';' . $diaspora_handle;
+
+	$signed_data = $positive . ';' . $guid . ';' . $target_type . ';' . $parent_guid . ';' . $diaspora_handle;
+
 	$key = $msg['key'];
 
-	if($parent_author_signature) {
+	if ($parent_author_signature) {
 		// If a parent_author_signature exists, then we've received the like
 		// relayed from the top-level post owner. There's no need to check the
 		// author_signature if the parent_author_signature is valid
 
 		$parent_author_signature = base64_decode($parent_author_signature);
 
-		if(! rsa_verify($signed_data,$parent_author_signature,$key,'sha256')) {
+		if (!rsa_verify($signed_data,$parent_author_signature,$key,'sha256') AND
-			if (intval(get_config('system','ignore_diaspora_like_signature')))
+			!rsa_verify($old_signed_data,$parent_author_signature,$key,'sha256')) {
-				logger('diaspora_like: top-level owner verification failed. Proceeding anyway.');
+
-			else {
+			logger('diaspora_like: top-level owner verification failed.');
-				logger('diaspora_like: top-level owner verification failed.');
+			return;
-				return;
-			}
 		}
-	}
+	} else {
-	else {
 		// If there's no parent_author_signature, then we've received the like
 		// from the like creator. In that case, the person is "like"ing
 		// our post, so he/she must be a contact of ours and his/her public key
@@ -1887,13 +1981,11 @@ function diaspora_like($importer,$xml,$msg) {
 
 		$author_signature = base64_decode($author_signature);
 
-		if(! rsa_verify($signed_data,$author_signature,$key,'sha256')) {
+		if (!rsa_verify($signed_data,$author_signature,$key,'sha256') AND
-			if (intval(get_config('system','ignore_diaspora_like_signature')))
+			!rsa_verify($old_signed_data,$author_signature,$key,'sha256')) {
-				logger('diaspora_like: like creator verification failed. Proceeding anyway');
+
-			else {
+			logger('diaspora_like: like creator verification failed.');
-				logger('diaspora_like: like creator verification failed.');
+			return;
-				return;
-			}
 		}
 	}
 
@@ -2028,6 +2120,7 @@ function diaspora_retraction($importer,$xml) {
 					dbesc(datetime_convert()),
 					intval($r[0]['id'])
 				);
+				delete_thread($r[0]['id'], $r[0]['parent-uri']);
 			}
 		}
 	}
@@ -2100,6 +2193,7 @@ function diaspora_signed_retraction($importer,$xml,$msg) {
 					dbesc(datetime_convert()),
 					intval($r[0]['id'])
 				);
+				delete_thread($r[0]['id'], $r[0]['parent-uri']);
 
 				// Now check if the retraction needs to be relayed by us
 				//
@@ -2159,14 +2253,27 @@ function diaspora_profile($importer,$xml,$msg) {
 	$name = unxmlify($xml->first_name) . ((strlen($xml->last_name)) ? ' ' . unxmlify($xml->last_name) : '');
 	$image_url = unxmlify($xml->image_url);
 	$birthday = unxmlify($xml->birthday);
+	$location = diaspora2bb(unxmlify($xml->location));
+	$about = diaspora2bb(unxmlify($xml->bio));
+	$gender = unxmlify($xml->gender);
+	$tags = unxmlify($xml->tag_string);
 
+	$tags = explode("#", $tags);
+
+	$keywords = array();
+	foreach ($tags as $tag) {
+		$tag = trim(strtolower($tag));
+		if ($tag != "")
+			$keywords[] = $tag;
+	}
+
+	$keywords = implode(", ", $keywords);
 
 	$handle_parts = explode("@", $diaspora_handle);
 	if($name === '') {
 		$name = $handle_parts[0];
 	}
-	
+
-	 
 	if( preg_match("|^https?://|", $image_url) === 0) {
 		$image_url = "http://" . $handle_parts[1] . $image_url;
 	}
@@ -2180,8 +2287,8 @@ function diaspora_profile($importer,$xml,$msg) {
 	require_once('include/Photo.php');
 
 	$images = import_profile_photo($image_url,$importer['uid'],$contact['id']);
-	
+
-	// Generic birthday. We don't know the timezone. The year is irrelevant. 
+	// Generic birthday. We don't know the timezone. The year is irrelevant.
 
 	$birthday = str_replace('1000','1901',$birthday);
 
@@ -2194,9 +2301,9 @@ function diaspora_profile($importer,$xml,$msg) {
 		$birthday = $contact['bd'];
 
 	// TODO: update name on item['author-name'] if the name changed. See consume_feed()
-	// Not doing this currently because D* protocol is scheduled for revision soon. 
+	// Not doing this currently because D* protocol is scheduled for revision soon.
 
-	$r = q("UPDATE `contact` SET `name` = '%s', `name-date` = '%s', `photo` = '%s', `thumb` = '%s', `micro` = '%s', `avatar-date` = '%s' , `bd` = '%s' WHERE `id` = %d AND `uid` = %d",
+	$r = q("UPDATE `contact` SET `name` = '%s', `name-date` = '%s', `photo` = '%s', `thumb` = '%s', `micro` = '%s', `avatar-date` = '%s' , `bd` = '%s', `location` = '%s', `about` = '%s', `keywords` = '%s', `gender` = '%s' WHERE `id` = %d AND `uid` = %d",
 		dbesc($name),
 		dbesc(datetime_convert()),
 		dbesc($images[0]),
@@ -2204,9 +2311,34 @@ function diaspora_profile($importer,$xml,$msg) {
 		dbesc($images[2]),
 		dbesc(datetime_convert()),
 		dbesc($birthday),
+		dbesc($location),
+		dbesc($about),
+		dbesc($keywords),
+		dbesc($gender),
 		intval($contact['id']),
 		intval($importer['uid'])
-	); 
+	);
+
+	if (unxmlify($xml->searchable) == "true") {
+		require_once('include/socgraph.php');
+		poco_check($contact['url'], $name, NETWORK_DIASPORA, $images[0], $about, $location, $gender, $keywords, "",
+			datetime_convert(), 2, $contact['id'], $importer['uid']);
+	}
+
+	$profileurl = "";
+	$author = q("SELECT * FROM `unique_contacts` WHERE `url`='%s' LIMIT 1",
+			dbesc(normalise_link($contact['url'])));
+
+	if (count($author) == 0) {
+		q("INSERT INTO `unique_contacts` (`url`, `name`, `avatar`, `location`, `about`) VALUES ('%s', '%s', '%s', '%s', '%s')",
+			dbesc(normalise_link($contact['url'])), dbesc($name), dbesc($location), dbesc($about), dbesc($images[0]));
+
+		$author = q("SELECT id FROM unique_contacts WHERE url='%s' LIMIT 1",
+			dbesc(normalise_link($contact['url'])));
+	} else if (normalise_link($contact['url']).$name.$location.$about != normalise_link($author[0]["url"]).$author[0]["name"].$author[0]["location"].$author[0]["about"]) {
+		q("UPDATE unique_contacts SET name = '%s', avatar = '%s', `location` = '%s', `about` = '%s' WHERE url = '%s'",
+		dbesc($name), dbesc($images[0]), dbesc($location), dbesc($about), dbesc(normalise_link($contact['url'])));
+	}
 
 /*	if($r) {
 		if($oldphotos) {
@@ -2380,6 +2512,26 @@ function diaspora_is_reshare($body) {
         if ($body == $attributes)
                 return(false);
 
+        $guid = "";
+        preg_match("/guid='(.*?)'/ism", $attributes, $matches);
+        if ($matches[1] != "")
+                $guid = $matches[1];
+
+        preg_match('/guid="(.*?)"/ism', $attributes, $matches);
+        if ($matches[1] != "")
+                $guid = $matches[1];
+
+	if ($guid != "") {
+		$r = q("SELECT `contact-id` FROM `item` WHERE `guid` = '%s' AND `network` IN ('%s', '%s') LIMIT 1",
+			dbesc($guid), NETWORK_DFRN, NETWORK_DIASPORA);
+		if ($r) {
+			$ret= array();
+			$ret["root_handle"] = diaspora_handle_from_contact($r[0]["contact-id"]);
+			$ret["root_guid"] = $guid;
+			return($ret);
+		}
+	}
+
         $profile = "";
         preg_match("/profile='(.*?)'/ism", $attributes, $matches);
         if ($matches[1] != "")

include/directory.php

@@ -46,6 +46,6 @@ function directory_run(&$argv, &$argc){
 }
 
 if (array_search(__file__,get_included_files())===0){
-  directory_run($argv,$argc);
+  directory_run($_SERVER["argv"],$_SERVER["argc"]);
   killme();
 }

include/dsprphotoq.php

@@ -45,6 +45,6 @@ function dsprphotoq_run($argv, $argc){
 
 
 if (array_search(__file__,get_included_files())===0){
-  dsprphotoq_run($argv,$argc);
+  dsprphotoq_run($_SERVER["argv"],$_SERVER["argc"]);
   killme();
 }

include/enotify.php

@@ -21,13 +21,21 @@ function notification($params) {
 	$thanks = t('Thank You,');
 	$sitename = $a->config['sitename'];
 	$site_admin = sprintf( t('%s Administrator'), $sitename);
+	$nickname = "";
 
-	$sender_name = $product;
+	$sender_name = $sitename;
 	$hostname = $a->get_hostname();
 	if(strpos($hostname,':'))
 		$hostname = substr($hostname,0,strpos($hostname,':'));
+	
+	$sender_email = $a->config['sender_email'];
+	if (empty($sender_email)) {
+		$sender_email = t('noreply') . '@' . $hostname;
+	}
 
-	$sender_email = t('noreply') . '@' . $hostname;
+	$user = q("SELECT `nickname` FROM `user` WHERE `uid` = %d", intval($params['uid']));
+	if ($user)
+		$nickname = $user[0]["nickname"];
 
 	// with $params['show_in_notification_page'] == false, the notification isn't inserted into
 	// the database, and an email is sent if applicable.
@@ -37,6 +45,7 @@ function notification($params) {
 	$additional_mail_header = "";
 	$additional_mail_header .= "Precedence: list\n";
 	$additional_mail_header .= "X-Friendica-Host: ".$hostname."\n";
+	$additional_mail_header .= "X-Friendica-Account: <".$nickname."@".$hostname.">\n";
 	$additional_mail_header .= "X-Friendica-Platform: ".FRIENDICA_PLATFORM."\n";
 	$additional_mail_header .= "X-Friendica-Version: ".FRIENDICA_VERSION."\n";
 	$additional_mail_header .= "List-ID: <notification.".$hostname.">\n";
@@ -344,7 +353,7 @@ function notification($params) {
 			$show_in_notification_page = false;
 	}
 
-
+	$subject .= " (".$nickname."@".$hostname.")";
 
 	$h = array(
 		'params'    => $params,
@@ -592,6 +601,7 @@ function notification($params) {
 		// use the Emailer class to send the message
 
 		return Emailer::send(array(
+			'uid' => $params['uid'],
 			'fromName' => $sender_name,
 			'fromEmail' => $sender_email,
 			'replyTo' => $sender_email,

include/expire.php

@@ -55,6 +55,6 @@ function expire_run(&$argv, &$argc){
 }
 
 if (array_search(__file__,get_included_files())===0){
-  expire_run($argv,$argc);
+  expire_run($_SERVER["argv"],$_SERVER["argc"]);
   killme();
 }

include/follow.php

@@ -37,7 +37,7 @@ function new_contact($uid,$url,$interactive = false) {
 
 	call_hooks('follow', $arr);
 
-	if(x($arr['contact'],'name')) 
+	if(x($arr['contact'],'name'))
 		$ret = $arr['contact'];
 	else
 		$ret = probe_url($url);
@@ -73,7 +73,7 @@ function new_contact($uid,$url,$interactive = false) {
 
 
 	// do we have enough information?
-	
+
 	if(! ((x($ret,'name')) && (x($ret,'poll')) && ((x($ret,'url')) || (x($ret,'addr'))))) {
 		$result['message'] .=  t('The profile address specified does not provide adequate information.') . EOL;
 		if(! x($ret,'poll'))
@@ -120,12 +120,12 @@ function new_contact($uid,$url,$interactive = false) {
 	// the poll url is more reliable than the profile url, as we may have
 	// indirect links or webfinger links
 
-	$r = q("SELECT * FROM `contact` WHERE `uid` = %d AND `poll` = '%s' LIMIT 1",
+	$r = q("SELECT * FROM `contact` WHERE `uid` = %d AND `poll` = '%s' AND `network` = '%s' LIMIT 1",
 		intval($uid),
-		dbesc($ret['poll'])
+		dbesc($ret['poll']),
+		dbesc($ret['network'])
 	);
 
-
 	if(count($r)) {
 		// update contact
 		if($r[0]['rel'] == CONTACT_IS_FOLLOWER || ($network === NETWORK_DIASPORA && $r[0]['rel'] == CONTACT_IS_SHARING)) {
@@ -169,10 +169,10 @@ function new_contact($uid,$url,$interactive = false) {
 		if($ret['network'] === NETWORK_DIASPORA)
 			$new_relation = CONTACT_IS_FOLLOWER;
 
-		// create contact record 
+		// create contact record
-		$r = q("INSERT INTO `contact` ( `uid`, `created`, `url`, `nurl`, `addr`, `alias`, `batch`, `notify`, `poll`, `poco`, `name`, `nick`, `photo`, `network`, `pubkey`, `rel`, `priority`,
+		$r = q("INSERT INTO `contact` ( `uid`, `created`, `url`, `nurl`, `addr`, `alias`, `batch`, `notify`, `poll`, `poco`, `name`, `nick`, `network`, `pubkey`, `rel`, `priority`,
 			`writable`, `hidden`, `blocked`, `readonly`, `pending`, `subhub` )
-			VALUES ( %d, '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d, %d, %d, %d, 0, 0, 0, %d ) ",
+			VALUES ( %d, '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d, %d, %d, %d, 0, 0, 0, %d ) ",
 			intval($uid),
 			dbesc(datetime_convert()),
 			dbesc($ret['url']),
@@ -185,7 +185,6 @@ function new_contact($uid,$url,$interactive = false) {
 			dbesc($ret['poco']),
 			dbesc($ret['name']),
 			dbesc($ret['nick']),
-			dbesc($ret['photo']),
 			dbesc($ret['network']),
 			dbesc($ret['pubkey']),
 			intval($new_relation),
@@ -196,8 +195,9 @@ function new_contact($uid,$url,$interactive = false) {
 		);
 	}
 
-	$r = q("SELECT * FROM `contact` WHERE `url` = '%s' AND `uid` = %d LIMIT 1",
+	$r = q("SELECT * FROM `contact` WHERE `url` = '%s' AND `network` = '%s' AND `uid` = %d LIMIT 1",
 		dbesc($ret['url']),
+		dbesc($ret['network']),
 		intval($uid)
 	);
 
@@ -228,8 +228,7 @@ function new_contact($uid,$url,$interactive = false) {
 			`name-date` = '%s',
 			`uri-date` = '%s',
 			`avatar-date` = '%s'
-			WHERE `id` = %d
+			WHERE `id` = %d",
-		",
 			dbesc($photos[0]),
 			dbesc($photos[1]),
 			dbesc($photos[2]),

include/gprobe.php

@@ -41,7 +41,23 @@ function gprobe_run(&$argv, &$argc){
 
 	if(! count($r)) {
 
+		// Is it a DDoS attempt?
+		$urlparts = parse_url($url);
+
+		$result = Cache::get("gprobe:".$urlparts["host"]);
+		if (!is_null($result)) {
+			$result = unserialize($result);
+			if ($result["network"] == NETWORK_FEED) {
+				logger("DDoS attempt detected for ".$urlparts["host"]." by ".$_SERVER["REMOTE_ADDR"].". server data: ".print_r($_SERVER, true), LOGGER_DEBUG);
+				return;
+			}
+		}
+
 		$arr = probe_url($url);
+
+		if (is_null($result))
+			Cache::set("gprobe:".$urlparts["host"],serialize($arr));
+
 		if(count($arr) && x($arr,'network') && $arr['network'] === NETWORK_DFRN) {
 			q("insert into `gcontact` (`name`,`url`,`nurl`,`photo`)
 				values ( '%s', '%s', '%s', '%s') ",
@@ -63,6 +79,6 @@ function gprobe_run(&$argv, &$argc){
 }
 
 if (array_search(__file__,get_included_files())===0){
-  gprobe_run($argv,$argc);
+  gprobe_run($_SERVER["argv"],$_SERVER["argc"]);
   killme();
 }

include/html2bbcode.php

@@ -16,7 +16,7 @@ function node2bbcode(&$doc, $oldnode, $attributes, $startbb, $endbb)
 
 function node2bbcodesub(&$doc, $oldnode, $attributes, $startbb, $endbb)
 {
-	$savestart = str_replace('$', '%', $startbb);
+	$savestart = str_replace('$', '\x01', $startbb);
 	$replace = false;
 
 	$xpath = new DomXPath($doc);
@@ -37,7 +37,7 @@ function node2bbcodesub(&$doc, $oldnode, $attributes, $startbb, $endbb)
 
 		foreach ($attributes as $attribute => $value) {
 
-			$startbb = str_replace('%'.++$i, '$1', $startbb);
+			$startbb = str_replace('\x01'.++$i, '$1', $startbb);
 
 			if (strpos('*'.$startbb, '$1') > 0) {
 
@@ -76,20 +76,18 @@ function node2bbcodesub(&$doc, $oldnode, $attributes, $startbb, $endbb)
 	return($replace);
 }
 
+if(!function_exists('deletenode')) {
 function deletenode(&$doc, $node)
 {
 	$xpath = new DomXPath($doc);
 	$list = $xpath->query("//".$node);
 	foreach ($list as $child)
 		$child->parentNode->removeChild($child);
-}
+}}
 
 function html2bbcode($message)
 {
 
-	//$file = tempnam("/tmp/", "html");
-	//file_put_contents($file, $message);
-
 	$message = str_replace("\r", "", $message);
 
 	$message = str_replace(array(
@@ -206,12 +204,19 @@ function html2bbcode($message)
 	//node2bbcode($doc, 'tr', array(), "[tr]", "[/tr]");
 	//node2bbcode($doc, 'td', array(), "[td]", "[/td]");
 
-	node2bbcode($doc, 'h1', array(), "\n\n[size=xx-large][b]", "[/b][/size]\n");
+	//node2bbcode($doc, 'h1', array(), "\n\n[size=xx-large][b]", "[/b][/size]\n");
-	node2bbcode($doc, 'h2', array(), "\n\n[size=x-large][b]", "[/b][/size]\n");
+	//node2bbcode($doc, 'h2', array(), "\n\n[size=x-large][b]", "[/b][/size]\n");
-	node2bbcode($doc, 'h3', array(), "\n\n[size=large][b]", "[/b][/size]\n");
+	//node2bbcode($doc, 'h3', array(), "\n\n[size=large][b]", "[/b][/size]\n");
-	node2bbcode($doc, 'h4', array(), "\n\n[size=medium][b]", "[/b][/size]\n");
+	//node2bbcode($doc, 'h4', array(), "\n\n[size=medium][b]", "[/b][/size]\n");
-	node2bbcode($doc, 'h5', array(), "\n\n[size=small][b]", "[/b][/size]\n");
+	//node2bbcode($doc, 'h5', array(), "\n\n[size=small][b]", "[/b][/size]\n");
-	node2bbcode($doc, 'h6', array(), "\n\n[size=x-small][b]", "[/b][/size]\n");
+	//node2bbcode($doc, 'h6', array(), "\n\n[size=x-small][b]", "[/b][/size]\n");
+
+	node2bbcode($doc, 'h1', array(), "\n\n[h1]", "[/h1]\n");
+	node2bbcode($doc, 'h2', array(), "\n\n[h2]", "[/h2]\n");
+	node2bbcode($doc, 'h3', array(), "\n\n[h3]", "[/h3]\n");
+	node2bbcode($doc, 'h4', array(), "\n\n[h4]", "[/h4]\n");
+	node2bbcode($doc, 'h5', array(), "\n\n[h5]", "[/h5]\n");
+	node2bbcode($doc, 'h6', array(), "\n\n[h6]", "[/h6]\n");
 
 	node2bbcode($doc, 'a', array('href'=>'/mailto:(.+)/'), '[mail=$1]', '[/mail]');
 	node2bbcode($doc, 'a', array('href'=>'/(.+)/'), '[url=$1]', '[/url]');

include/html2plain.php

@@ -113,12 +113,6 @@ function html2plain($html, $wraplength = 75, $compact = false)
 
 	$message = str_replace("\r", "", $html);
 
-	// replace all hashtag addresses
-/*	if (get_config("system", "remove_hashtags_on_export")) {
-		$pattern = '/#<a.*?href="(.*?)".*?>(.*?)<\/a>/is';
-		$message = preg_replace($pattern, '#$2', $message);
-	}
-*/
 	$doc = new DOMDocument();
 	$doc->preserveWhiteSpace = false;
 

include/items.php

@@ -11,6 +11,7 @@ require_once('include/text.php');
 require_once('include/email.php');
 require_once('include/ostatus_conversation.php');
 require_once('include/threads.php');
+require_once('include/socgraph.php');
 
 function get_feed_for(&$a, $dfrn_id, $owner_nick, $last_update, $direction = 0) {
 
@@ -872,9 +873,19 @@ function get_atom_elements($feed, $item, $contact = array()) {
 	}
 
 	if (isset($contact["network"]) AND ($contact["network"] == NETWORK_FEED) AND $contact['fetch_further_information']) {
-		$res["body"] = $res["title"].add_page_info($res['plink'], false, "", ($contact['fetch_further_information'] == 2), $contact['ffi_keyword_blacklist']);
+		$preview = "";
+
+		// Handle enclosures and treat them as preview picture
+		if (isset($attach))
+			foreach ($attach AS $attachment)
+				if ($attachment->type == "image/jpeg")
+					$preview = $attachment->link;
+
+		$res["body"] = $res["title"].add_page_info($res['plink'], false, $preview, ($contact['fetch_further_information'] == 2), $contact['ffi_keyword_blacklist']);
+		$res["tag"] = add_page_keywords($res['plink'], false, $preview, ($contact['fetch_further_information'] == 2), $contact['ffi_keyword_blacklist']);
 		$res["title"] = "";
 		$res["object-type"] = ACTIVITY_OBJ_BOOKMARK;
+		unset($res["attach"]);
 	} elseif (isset($contact["network"]) AND ($contact["network"] == NETWORK_OSTATUS))
 		$res["body"] = add_page_info_to_body($res["body"]);
 	elseif (isset($contact["network"]) AND ($contact["network"] == NETWORK_FEED) AND strstr($res['plink'], ".app.net/")) {
@@ -936,12 +947,20 @@ function add_page_info_data($data) {
 	return("\n[class=type-".$data["type"]."]".$text."[/class]".$hashtags);
 }
 
-function add_page_info($url, $no_photos = false, $photo = "", $keywords = false, $keyword_blacklist = "") {
+function query_page_info($url, $no_photos = false, $photo = "", $keywords = false, $keyword_blacklist = "") {
 	require_once("mod/parse_url.php");
 
-	$data = parseurl_getsiteinfo($url, true);
+	$data = Cache::get("parse_url:".$url);
+	if (is_null($data)){
+		$data = parseurl_getsiteinfo($url, true);
+		Cache::set("parse_url:".$url,serialize($data));
+	} else
+		$data = unserialize($data);
 
-	logger('add_page_info: fetch page info for '.$url.' '.print_r($data, true), LOGGER_DEBUG);
+	if ($photo != "")
+		$data["images"][0]["src"] = $photo;
+
+	logger('fetch page info for '.$url.' '.print_r($data, true), LOGGER_DEBUG);
 
 	if (!$keywords AND isset($data["keywords"]))
 		unset($data["keywords"]);
@@ -956,6 +975,32 @@ function add_page_info($url, $no_photos = false, $photo = "", $keywords = false,
 		}
 	}
 
+	return($data);
+}
+
+function add_page_keywords($url, $no_photos = false, $photo = "", $keywords = false, $keyword_blacklist = "") {
+	$data = query_page_info($url, $no_photos, $photo, $keywords, $keyword_blacklist);
+
+	$tags = "";
+	if (isset($data["keywords"]) AND count($data["keywords"])) {
+		$a = get_app();
+		foreach ($data["keywords"] AS $keyword) {
+			$hashtag = str_replace(array(" ", "+", "/", ".", "#", "'"),
+						array("","", "", "", "", ""), $keyword);
+
+			if ($tags != "")
+				$tags .= ",";
+
+			$tags .= "#[url=".$a->get_baseurl()."/search?tag=".rawurlencode($hashtag)."]".$hashtag."[/url]";
+		}
+	}
+
+	return($tags);
+}
+
+function add_page_info($url, $no_photos = false, $photo = "", $keywords = false, $keyword_blacklist = "") {
+	$data = query_page_info($url, $no_photos, $photo, $keywords, $keyword_blacklist);
+
 	$text = add_page_info_data($data);
 
 	return($text);
@@ -1027,7 +1072,7 @@ function encode_rel_links($links) {
 
 
 
-function item_store($arr,$force_parent = false, $notify = false) {
+function item_store($arr,$force_parent = false, $notify = false, $dontcache = false) {
 
 	// If it is a posting where users should get notifications, then define it as wall posting
 	if ($notify) {
@@ -1165,8 +1210,8 @@ function item_store($arr,$force_parent = false, $notify = false) {
 	$arr['attach']        = ((x($arr,'attach'))        ? notags(trim($arr['attach']))        : '');
 	$arr['app']           = ((x($arr,'app'))           ? notags(trim($arr['app']))           : '');
 	$arr['origin']        = ((x($arr,'origin'))        ? intval($arr['origin'])              : 0 );
-	$arr['guid']          = ((x($arr,'guid'))          ? notags(trim($arr['guid']))          : get_guid(30));
 	$arr['network']       = ((x($arr,'network'))       ? trim($arr['network'])               : '');
+	$arr['guid']          = ((x($arr,'guid'))          ? notags(trim($arr['guid']))          : get_guid(32, $arr['network']));
 	$arr['postopts']      = ((x($arr,'postopts'))      ? trim($arr['postopts'])              : '');
 	$arr['resource-id']   = ((x($arr,'resource-id'))   ? trim($arr['resource-id'])           : '');
 	$arr['event-id']      = ((x($arr,'event-id'))      ? intval($arr['event-id'])            : 0 );
@@ -1194,6 +1239,9 @@ function item_store($arr,$force_parent = false, $notify = false) {
 		logger("item_store: Set network to ".$arr["network"]." for ".$arr["uri"], LOGGER_DEBUG);
 	}
 
+	// Check for hashtags in the body and repair or add hashtag links
+	item_body_set_hashtags($arr);
+
 	$arr['thr-parent'] = $arr['parent-uri'];
 	if($arr['parent-uri'] === $arr['uri']) {
 		$parent_id = 0;
@@ -1298,6 +1346,20 @@ function item_store($arr,$force_parent = false, $notify = false) {
 		return 0;
 	}
 
+	// Is this item available in the global items (with uid=0)?
+	if ($arr["uid"] == 0) {
+		$arr["global"] = true;
+
+		q("UPDATE `item` SET `global` = 1 WHERE `guid` = '%s'", dbesc($arr["guid"]));
+	}  else {
+		$isglobal = q("SELECT `global` FROM `item` WHERE `uid` = 0 AND `guid` = '%s'", dbesc($arr["guid"]));
+
+		$arr["global"] = (count($isglobal) > 0);
+	}
+
+	// Fill the cache field
+	put_item_in_cache($arr);
+
 	call_hooks('post_remote',$arr);
 
 	if(x($arr,'cancel')) {
@@ -1305,6 +1367,9 @@ function item_store($arr,$force_parent = false, $notify = false) {
 		return 0;
 	}
 
+	// Store the unescaped version
+	$unescaped = $arr;
+
 	dbesc_array($arr);
 
 	logger('item_store: ' . print_r($arr,true), LOGGER_DATA);
@@ -1315,10 +1380,12 @@ function item_store($arr,$force_parent = false, $notify = false) {
 			. implode("', '", array_values($arr))
 			. "')" );
 
-	// find the item we just created
+	// And restore it
+	$arr = $unescaped;
 
+	// find the item we just created
 	$r = q("SELECT `id` FROM `item` WHERE `uri` = '%s' AND `uid` = %d ORDER BY `id` ASC ",
-		$arr['uri'],           // already dbesc'd
+		dbesc($arr['uri']),
 		intval($arr['uid'])
 	);
 
@@ -1326,57 +1393,23 @@ function item_store($arr,$force_parent = false, $notify = false) {
 		$current_post = $r[0]['id'];
 		logger('item_store: created item ' . $current_post);
 
-		// Only check for notifications on start posts
+		// Set "success_update" to the date of the last time we heard from this contact
-		if ($arr['parent-uri'] === $arr['uri']) {
+		// This can be used to filter for inactive contacts and poco.
-			add_thread($r[0]['id']);
+		// Only do this for public postings to avoid privacy problems, since poco data is public.
-			logger('item_store: Check notification for contact '.$arr['contact-id'].' and post '.$current_post, LOGGER_DEBUG);
+		// Don't set this value if it isn't from the owner (could be an author that we don't know)
-
+		if (!$arr['private'] AND (($arr["author-link"] === $arr["owner-link"]) OR ($arr["parent-uri"] === $arr["uri"])))
-			// Send a notification for every new post?
+			q("UPDATE `contact` SET `success_update` = '%s' WHERE `id` = %d",
-			$r = q("SELECT `notify_new_posts` FROM `contact` WHERE `id` = %d AND `uid` = %d AND `notify_new_posts` LIMIT 1",
+				dbesc($arr['received']),
-				intval($arr['contact-id']),
+				intval($arr['contact-id'])
-				intval($arr['uid'])
 			);
-
-			if(count($r)) {
-				logger('item_store: Send notification for contact '.$arr['contact-id'].' and post '.$current_post, LOGGER_DEBUG);
-				$u = q("SELECT * FROM user WHERE uid = %d LIMIT 1",
-					intval($arr['uid']));
-
-				$item = q("SELECT * FROM `item` WHERE `id` = %d AND `uid` = %d",
-					intval($current_post),
-					intval($arr['uid'])
-				);
-
-				$a = get_app();
-
-				require_once('include/enotify.php');
-				notification(array(
-					'type'         => NOTIFY_SHARE,
-					'notify_flags' => $u[0]['notify-flags'],
-					'language'     => $u[0]['language'],
-					'to_name'      => $u[0]['username'],
-					'to_email'     => $u[0]['email'],
-					'uid'          => $u[0]['uid'],
-					'item'         => $item[0],
-					'link'         => $a->get_baseurl().'/display/'.urlencode($arr['guid']),
-					'source_name'  => $item[0]['author-name'],
-					'source_link'  => $item[0]['author-link'],
-					'source_photo' => $item[0]['author-avatar'],
-					'verb'         => ACTIVITY_TAG,
-					'otype'        => 'item'
-				));
-				logger('item_store: Notification sent for contact '.$arr['contact-id'].' and post '.$current_post, LOGGER_DEBUG);
-			}
-		}
-
 	} else {
 		logger('item_store: could not locate created item');
 		return 0;
 	}
 	if(count($r) > 1) {
-		logger('item_store: duplicated post occurred. Removing duplicates.');
+		logger('item_store: duplicated post occurred. Removing duplicates. uri = '.$arr['uri'].' uid = '.$arr['uid']);
 		q("DELETE FROM `item` WHERE `uri` = '%s' AND `uid` = %d AND `id` != %d ",
-			$arr['uri'],
+			dbesc($arr['uri']),
 			intval($arr['uid']),
 			intval($current_post)
 		);
@@ -1418,13 +1451,18 @@ function item_store($arr,$force_parent = false, $notify = false) {
 	$arr['deleted'] = $parent_deleted;
 
 	// update the commented timestamp on the parent
-
+	// Only update "commented" if it is really a comment
-	q("UPDATE `item` set `commented` = '%s', `changed` = '%s' WHERE `id` = %d",
+	if (($arr['verb'] == ACTIVITY_POST) OR !get_config("system", "like_no_comment"))
-		dbesc(datetime_convert()),
+		q("UPDATE `item` SET `commented` = '%s', `changed` = '%s' WHERE `id` = %d",
-		dbesc(datetime_convert()),
+			dbesc(datetime_convert()),
-		intval($parent_id)
+			dbesc(datetime_convert()),
-	);
+			intval($parent_id)
-	update_thread($parent_id);
+		);
+	else
+		q("UPDATE `item` SET `changed` = '%s' WHERE `id` = %d",
+			dbesc(datetime_convert()),
+			intval($parent_id)
+		);
 
 	if($dsprsig) {
 		q("insert into sign (`iid`,`signed_text`,`signature`,`signer`) values (%d,'%s','%s','%s') ",
@@ -1450,39 +1488,157 @@ function item_store($arr,$force_parent = false, $notify = false) {
 
 	$deleted = tag_deliver($arr['uid'],$current_post);
 
-	// current post can be deleted if is for a communuty page and no mention are
+	// current post can be deleted if is for a community page and no mention are
 	// in it.
-	if (!$deleted) {
+	if (!$deleted AND !$dontcache) {
-
-		// Store the fresh generated item into the cache
-		$cachefile = get_cachefile(urlencode($arr["guid"])."-".hash("md5", $arr['body']));
-
-		if (($cachefile != '') AND !file_exists($cachefile)) {
-			$s = prepare_text($arr['body']);
-			$a = get_app();
-			$stamp1 = microtime(true);
-			file_put_contents($cachefile, $s);
-			$a->save_timestamp($stamp1, "file");
-			logger('item_store: put item '.$current_post.' into cachefile '.$cachefile);
-		}
 
 		$r = q('SELECT * FROM `item` WHERE id = %d', intval($current_post));
 		if (count($r) == 1) {
 			call_hooks('post_remote_end', $r[0]);
-		} else {
+		} else
 			logger('item_store: new item not found in DB, id ' . $current_post);
-		}
 	}
 
+	// Add every contact of the post to the global contact table
+	poco_store($arr);
+
 	create_tags_from_item($current_post);
 	create_files_from_item($current_post);
 
+	// Only check for notifications on start posts
+	if ($arr['parent-uri'] === $arr['uri']) {
+		add_thread($current_post);
+		logger('item_store: Check notification for contact '.$arr['contact-id'].' and post '.$current_post, LOGGER_DEBUG);
+
+		// Send a notification for every new post?
+		$r = q("SELECT `notify_new_posts` FROM `contact` WHERE `id` = %d AND `uid` = %d AND `notify_new_posts` LIMIT 1",
+			intval($arr['contact-id']),
+			intval($arr['uid'])
+		);
+		$send_notification = count($r);
+
+		if (!$send_notification) {
+			$tags = q("SELECT `url` FROM `term` WHERE `otype` = %d AND `oid` = %d AND `type` = %d AND `uid` = %d",
+				intval(TERM_OBJ_POST), intval($current_post), intval(TERM_MENTION), intval($arr['uid']));
+
+			if (count($tags)) {
+				foreach ($tags AS $tag) {
+					$r = q("SELECT `id` FROM `contact` WHERE `nurl` = '%s' AND `uid` = %d AND `notify_new_posts`",
+						normalise_link($tag["url"]), intval($arr['uid']));
+					if (count($r))
+						$send_notification = true;
+				}
+			}
+		}
+
+		if ($send_notification) {
+			logger('item_store: Send notification for contact '.$arr['contact-id'].' and post '.$current_post, LOGGER_DEBUG);
+			$u = q("SELECT * FROM user WHERE uid = %d LIMIT 1",
+				intval($arr['uid']));
+
+			$item = q("SELECT * FROM `item` WHERE `id` = %d AND `uid` = %d",
+				intval($current_post),
+				intval($arr['uid'])
+			);
+
+			$a = get_app();
+
+			require_once('include/enotify.php');
+			notification(array(
+				'type'         => NOTIFY_SHARE,
+				'notify_flags' => $u[0]['notify-flags'],
+				'language'     => $u[0]['language'],
+				'to_name'      => $u[0]['username'],
+				'to_email'     => $u[0]['email'],
+				'uid'          => $u[0]['uid'],
+				'item'         => $item[0],
+				'link'         => $a->get_baseurl().'/display/'.urlencode($arr['guid']),
+				'source_name'  => $item[0]['author-name'],
+				'source_link'  => $item[0]['author-link'],
+				'source_photo' => $item[0]['author-avatar'],
+				'verb'         => ACTIVITY_TAG,
+				'otype'        => 'item'
+			));
+			logger('item_store: Notification sent for contact '.$arr['contact-id'].' and post '.$current_post, LOGGER_DEBUG);
+		}
+	} else {
+		update_thread($parent_id);
+		add_shadow_entry($arr);
+	}
+
 	if ($notify)
 		proc_run('php', "include/notifier.php", $notify_type, $current_post);
 
 	return $current_post;
 }
 
+function item_body_set_hashtags(&$item) {
+
+	$tags = get_tags($item["body"]);
+
+	// No hashtags?
+	if(!count($tags))
+		return(false);
+
+	// This sorting is important when there are hashtags that are part of other hashtags
+	// Otherwise there could be problems with hashtags like #test and #test2
+	rsort($tags);
+
+	$a = get_app();
+
+	$URLSearchString = "^\[\]";
+
+	// All hashtags should point to the home server
+	//$item["body"] = preg_replace("/#\[url\=([$URLSearchString]*)\](.*?)\[\/url\]/ism",
+	//		"#[url=".$a->get_baseurl()."/search?tag=$2]$2[/url]", $item["body"]);
+
+	//$item["tag"] = preg_replace("/#\[url\=([$URLSearchString]*)\](.*?)\[\/url\]/ism",
+	//		"#[url=".$a->get_baseurl()."/search?tag=$2]$2[/url]", $item["tag"]);
+
+	// mask hashtags inside of url, bookmarks and attachments to avoid urls in urls
+	$item["body"] = preg_replace_callback("/\[url\=([$URLSearchString]*)\](.*?)\[\/url\]/ism",
+		function ($match){
+			return("[url=".$match[1]."]".str_replace("#", "#", $match[2])."[/url]");
+		},$item["body"]);
+
+	$item["body"] = preg_replace_callback("/\[bookmark\=([$URLSearchString]*)\](.*?)\[\/bookmark\]/ism",
+		function ($match){
+			return("[bookmark=".$match[1]."]".str_replace("#", "#", $match[2])."[/bookmark]");
+		},$item["body"]);
+
+	$item["body"] = preg_replace_callback("/\[attachment (.*)\](.*?)\[\/attachment\]/ism",
+		function ($match){
+			return("[attachment ".str_replace("#", "#", $match[1])."]".$match[2]."[/attachment]");
+		},$item["body"]);
+
+	// Repair recursive urls
+	$item["body"] = preg_replace("/#\[url\=([$URLSearchString]*)\](.*?)\[\/url\]/ism",
+			"#$2", $item["body"]);
+
+	foreach($tags as $tag) {
+		if(strpos($tag,'#') !== 0)
+			continue;
+
+		if(strpos($tag,'[url='))
+			continue;
+
+		$basetag = str_replace('_',' ',substr($tag,1));
+
+		$newtag = '#[url='.$a->get_baseurl().'/search?tag='.rawurlencode($basetag).']'.$basetag.'[/url]';
+
+		$item["body"] = str_replace($tag, $newtag, $item["body"]);
+
+		if(!stristr($item["tag"],"/search?tag=".$basetag."]".$basetag."[/url]")) {
+			if(strlen($item["tag"]))
+				$item["tag"] = ','.$item["tag"];
+			$item["tag"] = $newtag.$item["tag"];
+		}
+	}
+
+	// Convert back the masked hashtags
+	$item["body"] = str_replace("#", "#", $item["body"]);
+}
+
 function get_item_guid($id) {
 	$r = q("SELECT `guid` FROM `item` WHERE `id` = %d LIMIT 1", intval($id));
 	if (count($r))
@@ -2008,6 +2164,7 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0, $pass = 0)
 	$photo_timestamp = '';
 	$photo_url = '';
 	$birthday = '';
+	$contact_updated = '';
 
 	$hubs = $feed->get_links('hub');
 	logger('consume_feed: hubs: ' . print_r($hubs,true), LOGGER_DATA);
@@ -2023,9 +2180,16 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0, $pass = 0)
 		if($elems['name'][0]['attribs'][NAMESPACE_DFRN]['updated']) {
 			$name_updated = $elems['name'][0]['attribs'][NAMESPACE_DFRN]['updated'];
 			$new_name = $elems['name'][0]['data'];
+
+			// Manually checking for changed contact names
+			if (($new_name != $contact['name']) AND ($new_name != "") AND ($name_updated <= $contact['name-date'])) {
+				$name_updated = date("c");
+				$photo_timestamp = date("c");
+			}
 		}
 		if((x($elems,'link')) && ($elems['link'][0]['attribs']['']['rel'] === 'photo') && ($elems['link'][0]['attribs'][NAMESPACE_DFRN]['updated'])) {
-			$photo_timestamp = datetime_convert('UTC','UTC',$elems['link'][0]['attribs'][NAMESPACE_DFRN]['updated']);
+			if ($photo_timestamp == "")
+				$photo_timestamp = datetime_convert('UTC','UTC',$elems['link'][0]['attribs'][NAMESPACE_DFRN]['updated']);
 			$photo_url = $elems['link'][0]['attribs']['']['href'];
 		}
 
@@ -2036,6 +2200,9 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0, $pass = 0)
 
 	if((is_array($contact)) && ($photo_timestamp) && (strlen($photo_url)) && ($photo_timestamp > $contact['avatar-date'])) {
 		logger('consume_feed: Updating photo for '.$contact['name'].' from '.$photo_url.' uid: '.$contact['uid']);
+
+		$contact_updated = $photo_timestamp;
+
 		require_once("include/Photo.php");
 		$photo_failure = false;
 		$have_photo = false;
@@ -2093,6 +2260,9 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0, $pass = 0)
 	}
 
 	if((is_array($contact)) && ($name_updated) && (strlen($new_name)) && ($name_updated > $contact['name-date'])) {
+		if ($name_updated > $contact_updated)
+			$contact_updated = $name_updated;
+
 		$r = q("select * from contact where uid = %d and id = %d limit 1",
 			intval($contact['uid']),
 			intval($contact['id'])
@@ -2117,6 +2287,9 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0, $pass = 0)
 		}
 	}
 
+	if ($contact_updated AND $new_name AND $photo_url)
+		poco_check($contact['url'], $new_name, NETWORK_DFRN, $photo_url, "", "", "", "", "", $contact_updated, 2, $contact['id'], $contact['uid']);
+
 	if(strlen($birthday)) {
 		if(substr($birthday,0,4) != $contact['bdyear']) {
 			logger('consume_feed: updating birthday: ' . $birthday);
@@ -2163,7 +2336,6 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0, $pass = 0)
 
 			$contact['bdyear'] = substr($birthday,0,4);
 		}
-
 	}
 
 	$community_page = 0;
@@ -2645,20 +2817,24 @@ function item_is_remote_self($contact, &$datarray) {
 	if ($datarray["app"] == $a->get_hostname())
 		return false;
 
+	// Only forward posts
+	if ($datarray["verb"] != ACTIVITY_POST)
+		return false;
+
 	if (($contact['network'] != NETWORK_FEED) AND $datarray['private'])
 		return false;
 
 	$datarray2 = $datarray;
 	logger('remote-self start - Contact '.$contact['url'].' - '.$contact['remote_self'].' Item '.print_r($datarray, true), LOGGER_DEBUG);
 	if ($contact['remote_self'] == 2) {
-		$r = q("SELECT `id`,`url`,`name`,`photo`,`network` FROM `contact` WHERE `uid` = %d AND `self`",
+		$r = q("SELECT `id`,`url`,`name`,`thumb` FROM `contact` WHERE `uid` = %d AND `self`",
 			intval($contact['uid']));
 		if (count($r)) {
 			$datarray['contact-id'] = $r[0]["id"];
 
 			$datarray['owner-name'] = $r[0]["name"];
 			$datarray['owner-link'] = $r[0]["url"];
-			$datarray['owner-avatar'] = $r[0]["avatar"];
+			$datarray['owner-avatar'] = $r[0]["thumb"];
 
 			$datarray['author-name']   = $datarray['owner-name'];
 			$datarray['author-link']   = $datarray['owner-link'];
@@ -2725,6 +2901,7 @@ function local_delivery($importer,$data) {
 	$new_name = '';
 	$photo_timestamp = '';
 	$photo_url = '';
+	$contact_updated = '';
 
 
 	$rawtags = $feed->get_feed_tags( NAMESPACE_DFRN, 'owner');
@@ -2738,14 +2915,24 @@ function local_delivery($importer,$data) {
 		if($elems['name'][0]['attribs'][NAMESPACE_DFRN]['updated']) {
 			$name_updated = $elems['name'][0]['attribs'][NAMESPACE_DFRN]['updated'];
 			$new_name = $elems['name'][0]['data'];
+
+			// Manually checking for changed contact names
+			if (($new_name != $importer['name']) AND ($new_name != "") AND ($name_updated <= $importer['name-date'])) {
+				$name_updated = date("c");
+				$photo_timestamp = date("c");
+			}
 		}
 		if((x($elems,'link')) && ($elems['link'][0]['attribs']['']['rel'] === 'photo') && ($elems['link'][0]['attribs'][NAMESPACE_DFRN]['updated'])) {
-			$photo_timestamp = datetime_convert('UTC','UTC',$elems['link'][0]['attribs'][NAMESPACE_DFRN]['updated']);
+			if ($photo_timestamp == "")
+				$photo_timestamp = datetime_convert('UTC','UTC',$elems['link'][0]['attribs'][NAMESPACE_DFRN]['updated']);
 			$photo_url = $elems['link'][0]['attribs']['']['href'];
 		}
 	}
 
 	if(($photo_timestamp) && (strlen($photo_url)) && ($photo_timestamp > $importer['avatar-date'])) {
+
+		$contact_updated = $photo_timestamp;
+
 		logger('local_delivery: Updating photo for ' . $importer['name']);
 		require_once("include/Photo.php");
 		$photo_failure = false;
@@ -2804,6 +2991,9 @@ function local_delivery($importer,$data) {
 	}
 
 	if(($name_updated) && (strlen($new_name)) && ($name_updated > $importer['name-date'])) {
+		if ($name_updated > $contact_updated)
+			$contact_updated = $name_updated;
+
 		$r = q("select * from contact where uid = %d and id = %d limit 1",
 			intval($importer['importer_uid']),
 			intval($importer['id'])
@@ -2828,7 +3018,8 @@ function local_delivery($importer,$data) {
 		}
 	}
 
-
+	if ($contact_updated AND $new_name AND $photo_url)
+		poco_check($importer['url'], $new_name, NETWORK_DFRN, $photo_url, "", "", "", "", "", $contact_updated, 2, $importer['id'], $importer['importer_uid']);
 
 	// Currently unsupported - needs a lot of work
 	$reloc = $feed->get_feed_tags( NAMESPACE_DFRN, 'relocate' );
@@ -2867,6 +3058,7 @@ function local_delivery($importer,$data) {
 					thumb = '%s',
 					micro = '%s',
 					url = '%s',
+					nurl = '%s',
 					request = '%s',
 					confirm = '%s',
 					notify = '%s',
@@ -2878,6 +3070,7 @@ function local_delivery($importer,$data) {
 					dbesc($newloc['thumb']),
 					dbesc($newloc['micro']),
 					dbesc($newloc['url']),
+					dbesc(normalise_link($newloc['url'])),
 					dbesc($newloc['request']),
 					dbesc($newloc['confirm']),
 					dbesc($newloc['notify']),
@@ -4037,6 +4230,7 @@ function atom_entry($item,$type,$author,$owner,$comment = false,$cid = 0) {
 	else
 		$body = $item['body'];
 
+
 	$o = "\r\n\r\n<entry>\r\n";
 
 	if(is_array($author))
@@ -4051,13 +4245,22 @@ function atom_entry($item,$type,$author,$owner,$comment = false,$cid = 0) {
 		$o .= '<thr:in-reply-to ref="' . xmlify($parent_item) . '" type="text/html" href="' .  xmlify($a->get_baseurl() . '/display/' . $owner['nickname'] . '/' . $item['parent']) . '" />' . "\r\n";
 	}
 
+	$htmlbody = $body;
+
+	if ($item['title'] != "")
+		$htmlbody = "[b]".$item['title']."[/b]\n\n".$htmlbody;
+
+	$htmlbody = bbcode(bb_remove_share_information($htmlbody), false, false, 7);
+
 	$o .= '<id>' . xmlify($item['uri']) . '</id>' . "\r\n";
 	$o .= '<title>' . xmlify($item['title']) . '</title>' . "\r\n";
 	$o .= '<published>' . xmlify(datetime_convert('UTC','UTC',$item['created'] . '+00:00',ATOM_TIME)) . '</published>' . "\r\n";
 	$o .= '<updated>' . xmlify(datetime_convert('UTC','UTC',$item['edited'] . '+00:00',ATOM_TIME)) . '</updated>' . "\r\n";
 	$o .= '<dfrn:env>' . base64url_encode($body, true) . '</dfrn:env>' . "\r\n";
-	$o .= '<content type="' . $type . '" >' . xmlify((($type === 'html') ? bbcode($body) : $body)) . '</content>' . "\r\n";
+	$o .= '<content type="' . $type . '" >' . xmlify((($type === 'html') ? $htmlbody : $body)) . '</content>' . "\r\n";
 	$o .= '<link rel="alternate" type="text/html" href="' . xmlify($a->get_baseurl() . '/display/' . $owner['nickname'] . '/' . $item['id']) . '" />' . "\r\n";
+
+
 	if($comment)
 		$o .= '<dfrn:comment-allow>' . intval($item['last-child']) . '</dfrn:comment-allow>' . "\r\n";
 
@@ -4475,7 +4678,7 @@ function drop_item($id,$interactive = true) {
 		);
 		create_tags_from_item($item['id']);
 		create_files_from_item($item['id']);
-		delete_thread($item['id']);
+		delete_thread($item['id'], $item['parent-uri']);
 
 		// clean up categories and tags so they don't end up as orphans
 
@@ -4569,8 +4772,8 @@ function drop_item($id,$interactive = true) {
 				dbesc($item['parent-uri']),
 				intval($item['uid'])
 			);
-			create_tags_from_item($item['parent-uri'], $item['uid']);
+			create_tags_from_itemuri($item['parent-uri'], $item['uid']);
-			create_files_from_item($item['parent-uri'], $item['uid']);
+			create_files_from_itemuri($item['parent-uri'], $item['uid']);
 			delete_thread_uri($item['parent-uri'], $item['uid']);
 			// ignore the result
 		}

include/lock.php

@@ -11,20 +11,22 @@ function lock_function($fn_name, $block = true, $wait_sec = 2, $timeout = 30) {
 	$start = time();
 
 	do {
-		q("LOCK TABLE locks WRITE");
+		q("LOCK TABLE `locks` WRITE");
-		$r = q("SELECT locked FROM locks WHERE name = '%s' LIMIT 1",
+		$r = q("SELECT `locked`, `created` FROM `locks` WHERE `name` = '%s' LIMIT 1",
 			dbesc($fn_name)
 		);
 
-		if((count($r)) && (! $r[0]['locked'])) {
+		if((count($r)) AND (!$r[0]['locked'] OR (strtotime($r[0]['created']) < time() - 3600))) {
-			q("UPDATE locks SET locked = 1 WHERE name = '%s'",
+			q("UPDATE `locks` SET `locked` = 1, `created` = '%s' WHERE `name` = '%s'",
+				dbesc(datetime_convert()),
 				dbesc($fn_name)
 			);
 			$got_lock = true;
 		}
 		elseif(! $r) { // the Boolean value for count($r) should be equivalent to the Boolean value of $r
-			q("INSERT INTO locks ( name, locked ) VALUES ( '%s', 1 )",
+			q("INSERT INTO `locks` (`name`, `created`, `locked`) VALUES ('%s', '%s', 1)",
-				dbesc($fn_name)
+				dbesc($fn_name),
+				dbesc(datetime_convert())
 			);
 			$got_lock = true;
 		}
@@ -37,7 +39,7 @@ function lock_function($fn_name, $block = true, $wait_sec = 2, $timeout = 30) {
 	} while(($block) && (! $got_lock) && ((time() - $start) < $timeout));
 
 	logger('lock_function: function ' . $fn_name . ' with blocking = ' . $block . ' got_lock = ' . $got_lock . ' time = ' . (time() - $start), LOGGER_DEBUG);
-	
+
 	return $got_lock;
 }}
 
@@ -65,7 +67,7 @@ function block_on_function_lock($fn_name, $wait_sec = 2, $timeout = 30) {
 
 if(! function_exists('unlock_function')) {
 function unlock_function($fn_name) {
-	$r = q("UPDATE locks SET locked = 0 WHERE name = '%s'",
+	$r = q("UPDATE `locks` SET `locked` = 0, `created` = '0000-00-00 00:00:00' WHERE `name` = '%s'",
 			dbesc($fn_name)
 	     );
 

include/markdownify/LICENSE_LGPL.txt

@@ -1,504 +0,0 @@
-		  GNU LESSER GENERAL PUBLIC LICENSE
-		       Version 2.1, February 1999
-
- Copyright (C) 1991, 1999 Free Software Foundation, Inc.
- 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-[This is the first released version of the Lesser GPL.  It also counts
- as the successor of the GNU Library Public License, version 2, hence
- the version number 2.1.]
-
-			    Preamble
-
-  The licenses for most software are designed to take away your
-freedom to share and change it.  By contrast, the GNU General Public
-Licenses are intended to guarantee your freedom to share and change
-free software--to make sure the software is free for all its users.
-
-  This license, the Lesser General Public License, applies to some
-specially designated software packages--typically libraries--of the
-Free Software Foundation and other authors who decide to use it.  You
-can use it too, but we suggest you first think carefully about whether
-this license or the ordinary General Public License is the better
-strategy to use in any particular case, based on the explanations below.
-
-  When we speak of free software, we are referring to freedom of use,
-not price.  Our General Public Licenses are designed to make sure that
-you have the freedom to distribute copies of free software (and charge
-for this service if you wish); that you receive source code or can get
-it if you want it; that you can change the software and use pieces of
-it in new free programs; and that you are informed that you can do
-these things.
-
-  To protect your rights, we need to make restrictions that forbid
-distributors to deny you these rights or to ask you to surrender these
-rights.  These restrictions translate to certain responsibilities for
-you if you distribute copies of the library or if you modify it.
-
-  For example, if you distribute copies of the library, whether gratis
-or for a fee, you must give the recipients all the rights that we gave
-you.  You must make sure that they, too, receive or can get the source
-code.  If you link other code with the library, you must provide
-complete object files to the recipients, so that they can relink them
-with the library after making changes to the library and recompiling
-it.  And you must show them these terms so they know their rights.
-
-  We protect your rights with a two-step method: (1) we copyright the
-library, and (2) we offer you this license, which gives you legal
-permission to copy, distribute and/or modify the library.
-
-  To protect each distributor, we want to make it very clear that
-there is no warranty for the free library.  Also, if the library is
-modified by someone else and passed on, the recipients should know
-that what they have is not the original version, so that the original
-author's reputation will not be affected by problems that might be
-introduced by others.
-
-  Finally, software patents pose a constant threat to the existence of
-any free program.  We wish to make sure that a company cannot
-effectively restrict the users of a free program by obtaining a
-restrictive license from a patent holder.  Therefore, we insist that
-any patent license obtained for a version of the library must be
-consistent with the full freedom of use specified in this license.
-
-  Most GNU software, including some libraries, is covered by the
-ordinary GNU General Public License.  This license, the GNU Lesser
-General Public License, applies to certain designated libraries, and
-is quite different from the ordinary General Public License.  We use
-this license for certain libraries in order to permit linking those
-libraries into non-free programs.
-
-  When a program is linked with a library, whether statically or using
-a shared library, the combination of the two is legally speaking a
-combined work, a derivative of the original library.  The ordinary
-General Public License therefore permits such linking only if the
-entire combination fits its criteria of freedom.  The Lesser General
-Public License permits more lax criteria for linking other code with
-the library.
-
-  We call this license the "Lesser" General Public License because it
-does Less to protect the user's freedom than the ordinary General
-Public License.  It also provides other free software developers Less
-of an advantage over competing non-free programs.  These disadvantages
-are the reason we use the ordinary General Public License for many
-libraries.  However, the Lesser license provides advantages in certain
-special circumstances.
-
-  For example, on rare occasions, there may be a special need to
-encourage the widest possible use of a certain library, so that it becomes
-a de-facto standard.  To achieve this, non-free programs must be
-allowed to use the library.  A more frequent case is that a free
-library does the same job as widely used non-free libraries.  In this
-case, there is little to gain by limiting the free library to free
-software only, so we use the Lesser General Public License.
-
-  In other cases, permission to use a particular library in non-free
-programs enables a greater number of people to use a large body of
-free software.  For example, permission to use the GNU C Library in
-non-free programs enables many more people to use the whole GNU
-operating system, as well as its variant, the GNU/Linux operating
-system.
-
-  Although the Lesser General Public License is Less protective of the
-users' freedom, it does ensure that the user of a program that is
-linked with the Library has the freedom and the wherewithal to run
-that program using a modified version of the Library.
-
-  The precise terms and conditions for copying, distribution and
-modification follow.  Pay close attention to the difference between a
-"work based on the library" and a "work that uses the library".  The
-former contains code derived from the library, whereas the latter must
-be combined with the library in order to run.
-
-		  GNU LESSER GENERAL PUBLIC LICENSE
-   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
-
-  0. This License Agreement applies to any software library or other
-program which contains a notice placed by the copyright holder or
-other authorized party saying it may be distributed under the terms of
-this Lesser General Public License (also called "this License").
-Each licensee is addressed as "you".
-
-  A "library" means a collection of software functions and/or data
-prepared so as to be conveniently linked with application programs
-(which use some of those functions and data) to form executables.
-
-  The "Library", below, refers to any such software library or work
-which has been distributed under these terms.  A "work based on the
-Library" means either the Library or any derivative work under
-copyright law: that is to say, a work containing the Library or a
-portion of it, either verbatim or with modifications and/or translated
-straightforwardly into another language.  (Hereinafter, translation is
-included without limitation in the term "modification".)
-
-  "Source code" for a work means the preferred form of the work for
-making modifications to it.  For a library, complete source code means
-all the source code for all modules it contains, plus any associated
-interface definition files, plus the scripts used to control compilation
-and installation of the library.
-
-  Activities other than copying, distribution and modification are not
-covered by this License; they are outside its scope.  The act of
-running a program using the Library is not restricted, and output from
-such a program is covered only if its contents constitute a work based
-on the Library (independent of the use of the Library in a tool for
-writing it).  Whether that is true depends on what the Library does
-and what the program that uses the Library does.
-  
-  1. You may copy and distribute verbatim copies of the Library's
-complete source code as you receive it, in any medium, provided that
-you conspicuously and appropriately publish on each copy an
-appropriate copyright notice and disclaimer of warranty; keep intact
-all the notices that refer to this License and to the absence of any
-warranty; and distribute a copy of this License along with the
-Library.
-
-  You may charge a fee for the physical act of transferring a copy,
-and you may at your option offer warranty protection in exchange for a
-fee.
-
-  2. You may modify your copy or copies of the Library or any portion
-of it, thus forming a work based on the Library, and copy and
-distribute such modifications or work under the terms of Section 1
-above, provided that you also meet all of these conditions:
-
-    a) The modified work must itself be a software library.
-
-    b) You must cause the files modified to carry prominent notices
-    stating that you changed the files and the date of any change.
-
-    c) You must cause the whole of the work to be licensed at no
-    charge to all third parties under the terms of this License.
-
-    d) If a facility in the modified Library refers to a function or a
-    table of data to be supplied by an application program that uses
-    the facility, other than as an argument passed when the facility
-    is invoked, then you must make a good faith effort to ensure that,
-    in the event an application does not supply such function or
-    table, the facility still operates, and performs whatever part of
-    its purpose remains meaningful.
-
-    (For example, a function in a library to compute square roots has
-    a purpose that is entirely well-defined independent of the
-    application.  Therefore, Subsection 2d requires that any
-    application-supplied function or table used by this function must
-    be optional: if the application does not supply it, the square
-    root function must still compute square roots.)
-
-These requirements apply to the modified work as a whole.  If
-identifiable sections of that work are not derived from the Library,
-and can be reasonably considered independent and separate works in
-themselves, then this License, and its terms, do not apply to those
-sections when you distribute them as separate works.  But when you
-distribute the same sections as part of a whole which is a work based
-on the Library, the distribution of the whole must be on the terms of
-this License, whose permissions for other licensees extend to the
-entire whole, and thus to each and every part regardless of who wrote
-it.
-
-Thus, it is not the intent of this section to claim rights or contest
-your rights to work written entirely by you; rather, the intent is to
-exercise the right to control the distribution of derivative or
-collective works based on the Library.
-
-In addition, mere aggregation of another work not based on the Library
-with the Library (or with a work based on the Library) on a volume of
-a storage or distribution medium does not bring the other work under
-the scope of this License.
-
-  3. You may opt to apply the terms of the ordinary GNU General Public
-License instead of this License to a given copy of the Library.  To do
-this, you must alter all the notices that refer to this License, so
-that they refer to the ordinary GNU General Public License, version 2,
-instead of to this License.  (If a newer version than version 2 of the
-ordinary GNU General Public License has appeared, then you can specify
-that version instead if you wish.)  Do not make any other change in
-these notices.
-
-  Once this change is made in a given copy, it is irreversible for
-that copy, so the ordinary GNU General Public License applies to all
-subsequent copies and derivative works made from that copy.
-
-  This option is useful when you wish to copy part of the code of
-the Library into a program that is not a library.
-
-  4. You may copy and distribute the Library (or a portion or
-derivative of it, under Section 2) in object code or executable form
-under the terms of Sections 1 and 2 above provided that you accompany
-it with the complete corresponding machine-readable source code, which
-must be distributed under the terms of Sections 1 and 2 above on a
-medium customarily used for software interchange.
-
-  If distribution of object code is made by offering access to copy
-from a designated place, then offering equivalent access to copy the
-source code from the same place satisfies the requirement to
-distribute the source code, even though third parties are not
-compelled to copy the source along with the object code.
-
-  5. A program that contains no derivative of any portion of the
-Library, but is designed to work with the Library by being compiled or
-linked with it, is called a "work that uses the Library".  Such a
-work, in isolation, is not a derivative work of the Library, and
-therefore falls outside the scope of this License.
-
-  However, linking a "work that uses the Library" with the Library
-creates an executable that is a derivative of the Library (because it
-contains portions of the Library), rather than a "work that uses the
-library".  The executable is therefore covered by this License.
-Section 6 states terms for distribution of such executables.
-
-  When a "work that uses the Library" uses material from a header file
-that is part of the Library, the object code for the work may be a
-derivative work of the Library even though the source code is not.
-Whether this is true is especially significant if the work can be
-linked without the Library, or if the work is itself a library.  The
-threshold for this to be true is not precisely defined by law.
-
-  If such an object file uses only numerical parameters, data
-structure layouts and accessors, and small macros and small inline
-functions (ten lines or less in length), then the use of the object
-file is unrestricted, regardless of whether it is legally a derivative
-work.  (Executables containing this object code plus portions of the
-Library will still fall under Section 6.)
-
-  Otherwise, if the work is a derivative of the Library, you may
-distribute the object code for the work under the terms of Section 6.
-Any executables containing that work also fall under Section 6,
-whether or not they are linked directly with the Library itself.
-
-  6. As an exception to the Sections above, you may also combine or
-link a "work that uses the Library" with the Library to produce a
-work containing portions of the Library, and distribute that work
-under terms of your choice, provided that the terms permit
-modification of the work for the customer's own use and reverse
-engineering for debugging such modifications.
-
-  You must give prominent notice with each copy of the work that the
-Library is used in it and that the Library and its use are covered by
-this License.  You must supply a copy of this License.  If the work
-during execution displays copyright notices, you must include the
-copyright notice for the Library among them, as well as a reference
-directing the user to the copy of this License.  Also, you must do one
-of these things:
-
-    a) Accompany the work with the complete corresponding
-    machine-readable source code for the Library including whatever
-    changes were used in the work (which must be distributed under
-    Sections 1 and 2 above); and, if the work is an executable linked
-    with the Library, with the complete machine-readable "work that
-    uses the Library", as object code and/or source code, so that the
-    user can modify the Library and then relink to produce a modified
-    executable containing the modified Library.  (It is understood
-    that the user who changes the contents of definitions files in the
-    Library will not necessarily be able to recompile the application
-    to use the modified definitions.)
-
-    b) Use a suitable shared library mechanism for linking with the
-    Library.  A suitable mechanism is one that (1) uses at run time a
-    copy of the library already present on the user's computer system,
-    rather than copying library functions into the executable, and (2)
-    will operate properly with a modified version of the library, if
-    the user installs one, as long as the modified version is
-    interface-compatible with the version that the work was made with.
-
-    c) Accompany the work with a written offer, valid for at
-    least three years, to give the same user the materials
-    specified in Subsection 6a, above, for a charge no more
-    than the cost of performing this distribution.
-
-    d) If distribution of the work is made by offering access to copy
-    from a designated place, offer equivalent access to copy the above
-    specified materials from the same place.
-
-    e) Verify that the user has already received a copy of these
-    materials or that you have already sent this user a copy.
-
-  For an executable, the required form of the "work that uses the
-Library" must include any data and utility programs needed for
-reproducing the executable from it.  However, as a special exception,
-the materials to be distributed need not include anything that is
-normally distributed (in either source or binary form) with the major
-components (compiler, kernel, and so on) of the operating system on
-which the executable runs, unless that component itself accompanies
-the executable.
-
-  It may happen that this requirement contradicts the license
-restrictions of other proprietary libraries that do not normally
-accompany the operating system.  Such a contradiction means you cannot
-use both them and the Library together in an executable that you
-distribute.
-
-  7. You may place library facilities that are a work based on the
-Library side-by-side in a single library together with other library
-facilities not covered by this License, and distribute such a combined
-library, provided that the separate distribution of the work based on
-the Library and of the other library facilities is otherwise
-permitted, and provided that you do these two things:
-
-    a) Accompany the combined library with a copy of the same work
-    based on the Library, uncombined with any other library
-    facilities.  This must be distributed under the terms of the
-    Sections above.
-
-    b) Give prominent notice with the combined library of the fact
-    that part of it is a work based on the Library, and explaining
-    where to find the accompanying uncombined form of the same work.
-
-  8. You may not copy, modify, sublicense, link with, or distribute
-the Library except as expressly provided under this License.  Any
-attempt otherwise to copy, modify, sublicense, link with, or
-distribute the Library is void, and will automatically terminate your
-rights under this License.  However, parties who have received copies,
-or rights, from you under this License will not have their licenses
-terminated so long as such parties remain in full compliance.
-
-  9. You are not required to accept this License, since you have not
-signed it.  However, nothing else grants you permission to modify or
-distribute the Library or its derivative works.  These actions are
-prohibited by law if you do not accept this License.  Therefore, by
-modifying or distributing the Library (or any work based on the
-Library), you indicate your acceptance of this License to do so, and
-all its terms and conditions for copying, distributing or modifying
-the Library or works based on it.
-
-  10. Each time you redistribute the Library (or any work based on the
-Library), the recipient automatically receives a license from the
-original licensor to copy, distribute, link with or modify the Library
-subject to these terms and conditions.  You may not impose any further
-restrictions on the recipients' exercise of the rights granted herein.
-You are not responsible for enforcing compliance by third parties with
-this License.
-
-  11. If, as a consequence of a court judgment or allegation of patent
-infringement or for any other reason (not limited to patent issues),
-conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot
-distribute so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you
-may not distribute the Library at all.  For example, if a patent
-license would not permit royalty-free redistribution of the Library by
-all those who receive copies directly or indirectly through you, then
-the only way you could satisfy both it and this License would be to
-refrain entirely from distribution of the Library.
-
-If any portion of this section is held invalid or unenforceable under any
-particular circumstance, the balance of the section is intended to apply,
-and the section as a whole is intended to apply in other circumstances.
-
-It is not the purpose of this section to induce you to infringe any
-patents or other property right claims or to contest validity of any
-such claims; this section has the sole purpose of protecting the
-integrity of the free software distribution system which is
-implemented by public license practices.  Many people have made
-generous contributions to the wide range of software distributed
-through that system in reliance on consistent application of that
-system; it is up to the author/donor to decide if he or she is willing
-to distribute software through any other system and a licensee cannot
-impose that choice.
-
-This section is intended to make thoroughly clear what is believed to
-be a consequence of the rest of this License.
-
-  12. If the distribution and/or use of the Library is restricted in
-certain countries either by patents or by copyrighted interfaces, the
-original copyright holder who places the Library under this License may add
-an explicit geographical distribution limitation excluding those countries,
-so that distribution is permitted only in or among countries not thus
-excluded.  In such case, this License incorporates the limitation as if
-written in the body of this License.
-
-  13. The Free Software Foundation may publish revised and/or new
-versions of the Lesser General Public License from time to time.
-Such new versions will be similar in spirit to the present version,
-but may differ in detail to address new problems or concerns.
-
-Each version is given a distinguishing version number.  If the Library
-specifies a version number of this License which applies to it and
-"any later version", you have the option of following the terms and
-conditions either of that version or of any later version published by
-the Free Software Foundation.  If the Library does not specify a
-license version number, you may choose any version ever published by
-the Free Software Foundation.
-
-  14. If you wish to incorporate parts of the Library into other free
-programs whose distribution conditions are incompatible with these,
-write to the author to ask for permission.  For software which is
-copyrighted by the Free Software Foundation, write to the Free
-Software Foundation; we sometimes make exceptions for this.  Our
-decision will be guided by the two goals of preserving the free status
-of all derivatives of our free software and of promoting the sharing
-and reuse of software generally.
-
-			    NO WARRANTY
-
-  15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
-WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
-EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
-OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
-KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
-LIBRARY IS WITH YOU.  SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
-THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
-  16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
-WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
-AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
-FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
-CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
-LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
-RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
-FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
-SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
-DAMAGES.
-
-		     END OF TERMS AND CONDITIONS
-
-           How to Apply These Terms to Your New Libraries
-
-  If you develop a new library, and you want it to be of the greatest
-possible use to the public, we recommend making it free software that
-everyone can redistribute and change.  You can do so by permitting
-redistribution under these terms (or, alternatively, under the terms of the
-ordinary General Public License).
-
-  To apply these terms, attach the following notices to the library.  It is
-safest to attach them to the start of each source file to most effectively
-convey the exclusion of warranty; and each file should have at least the
-"copyright" line and a pointer to where the full notice is found.
-
-    <one line to give the library's name and a brief idea of what it does.>
-    Copyright (C) <year>  <name of author>
-
-    This library is free software; you can redistribute it and/or
-    modify it under the terms of the GNU Lesser General Public
-    License as published by the Free Software Foundation; either
-    version 2.1 of the License, or (at your option) any later version.
-
-    This library is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-    Lesser General Public License for more details.
-
-    You should have received a copy of the GNU Lesser General Public
-    License along with this library; if not, write to the Free Software
-    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
-
-Also add information on how to contact you by electronic and paper mail.
-
-You should also get your employer (if you work as a programmer) or your
-school, if any, to sign a "copyright disclaimer" for the library, if
-necessary.  Here is a sample; alter the names:
-
-  Yoyodyne, Inc., hereby disclaims all copyright interest in the
-  library `Frob' (a library for tweaking knobs) written by James Random Hacker.
-
-  <signature of Ty Coon>, 1 April 1990
-  Ty Coon, President of Vice
-
-That's all there is to it!
-
-

include/markdownify/TODO

@@ -1,29 +0,0 @@
-Markdownify
-===========
-* handle non-markdownifiable lists (i.e. `<ul><li id="foobar">asdf</li></ul>`)
-* organize methods better (i.e. flushlinebreaks & setlinebreaks close to each other)
-* take a look at function names etc.
-* is the new (in rev. 93) lastclosedtag property needed?
-* word wrapping (some work is done but it's still very buggy)
-
-
-Markdownify Extra
-=================
-
-* handle table alignment with KEEP_HTML=false
-* handle tables without headings when KEEP_HTML=false is set
-* handle Markdown inside non-markdownable tags
-
-
-Implementation Thoughts
-=======================
-* non-markdownifiable lists and markdown inside non-markdownable tags as well as the current
-  table implementation could be rewritten by using a rollback mechanism.
-  
-  example:
-  
-      <ul><li>asdf</li><li id="foobar">asdf</li></ul>
-  
-  we come to `<ul>`, know that this might fail and create a snapshot of our current parser
-  we keep on parsing and when we reach `<li id="foobar">` we gotta rollback and keep this
-  list in HTML format.

include/markdownify/example.php

@@ -1,51 +0,0 @@
-<?php
-	error_reporting(E_ALL);
-	if (!empty($_POST['input'])) {
-		include 'markdownify_extra.php';
-		if (!isset($_POST['leap'])) {
-			$leap = MDFY_LINKS_EACH_PARAGRAPH;
-		} else {
-			$leap = $_POST['leap'];
-		}
-		
-		if (!isset($_POST['keepHTML'])) {
-			$keephtml = MDFY_KEEPHTML;
-		} else {
-			$keephtml = $_POST['keepHTML'];
-		}
-		if (!empty($_POST['extra'])) {
-			$md = new Markdownify_Extra($leap, MDFY_BODYWIDTH, $keephtml);
-		} else {
-			$md = new Markdownify($leap, MDFY_BODYWIDTH, $keephtml);
-		}
-		if (ini_get('magic_quotes_gpc')) {
-			$_POST['input'] = stripslashes($_POST['input']);
-		}
-		$output = $md->parseString($_POST['input']);
-	} else {
-		$_POST['input'] = '';
-	}
-?><!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en-US" lang="en-US">
-	<head>
-		<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
-		<title>HTML to Markdown Converter</title>
-	</head>
-	<body>
-		<?php if (empty($_POST['input'])): ?>
-		<form action="<?php echo $_SERVER['PHP_SELF']; ?>" method="post">
-			<fieldset>
-				<legend>HTML Input</legend>
-				<textarea style="width:100%;" cols="85" rows="40" name="input"><?php echo htmlspecialchars($_POST['input'], ENT_NOQUOTES, 'UTF-8'); ?></textarea>
-			</fieldset>
-			<label for="extra">Markdownify Extra: <input name="extra" checked="checked" id="extra" type="checkbox" value="1"  /></label>
-			<label for="leap">Links after each block elem: <input name="leap" id="leap" type="checkbox" value="1"  /></label>
-			<label for="keepHTML">keep HTML: <input name="keepHTML" id="keepHTML" type="checkbox" value="1" checked="checked" /></label>
-			<input type="submit" name="submit" value="submit" />
-		</form>
-		<?php else: ?>
-		<h1 style="text-align:right;"><a href="<?php echo $_SERVER['PHP_SELF']; ?>">BACK</a></h1>
-		<pre><?php echo htmlspecialchars($output, ENT_NOQUOTES, 'UTF-8'); ?></pre>
-		<?php endif; ?>
-	</body>
-</html>

include/markdownify/markdownify_cli.php

@@ -1,33 +0,0 @@
-#!/usr/bin/php
-<?php
-require dirname(__FILE__) .'/markdownify_extra.php';
-
-function param($name, $default = false) {
-  if (!in_array('--'.$name, $_SERVER['argv']))
-    return $default;
-  reset($_SERVER['argv']);
-  while (each($_SERVER['argv'])) {
-    if (current($_SERVER['argv']) == '--'.$name)
-      break;
-  }
-  $value = next($_SERVER['argv']);
-  if ($value === false || substr($value, 0, 2) == '--')
-    return true;
-  else
-    return $value;
-}
-
-
-$input = stream_get_contents(STDIN);
-
-$linksAfterEachParagraph = param('links');
-$bodyWidth = param('width');
-$keepHTML = param('html', true);
-
-if (param('no_extra')) {
-  $parser = new Markdownify($linksAfterEachParagraph, $bodyWidth, $keepHTML);
-} else {
-  $parser = new Markdownify_Extra($linksAfterEachParagraph, $bodyWidth, $keepHTML);
-}
-
-echo $parser->parseString($input) ."\n";

include/markdownify/markdownify_extra.php

@@ -1,489 +0,0 @@
-<?php
-/**
- * Class to convert HTML to Markdown with PHP Markdown Extra syntax support.
- *
- * @version 1.0.0 alpha
- * @author Milian Wolff (<mail@milianw.de>, <http://milianw.de>)
- * @license LGPL, see LICENSE_LGPL.txt and the summary below
- * @copyright (C) 2007  Milian Wolff
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
- */
-
-/**
- * standard Markdownify class
- */
-require_once dirname(__FILE__).'/markdownify.php';
-
-class Markdownify_Extra extends Markdownify {
-  /**
-   * table data, including rows with content and the maximum width of each col
-   *
-   * @var array
-   */
-  var $table = array();
-  /**
-   * current col
-   *
-   * @var int
-   */
-  var $col = -1;
-  /**
-   * current row
-   *
-   * @var int
-   */
-  var $row = 0;
-  /**
-   * constructor, see Markdownify::Markdownify() for more information
-   */
-  function Markdownify_Extra($linksAfterEachParagraph = MDFY_LINKS_EACH_PARAGRAPH, $bodyWidth = MDFY_BODYWIDTH, $keepHTML = MDFY_KEEPHTML) {
-    parent::Markdownify($linksAfterEachParagraph, $bodyWidth, $keepHTML);
-
-    ### new markdownable tags & attributes
-    # header ids: # foo {bar}
-    $this->isMarkdownable['h1']['id'] = 'optional';
-    $this->isMarkdownable['h2']['id'] = 'optional';
-    $this->isMarkdownable['h3']['id'] = 'optional';
-    $this->isMarkdownable['h4']['id'] = 'optional';
-    $this->isMarkdownable['h5']['id'] = 'optional';
-    $this->isMarkdownable['h6']['id'] = 'optional';
-    # tables
-    $this->isMarkdownable['table'] = array();
-    $this->isMarkdownable['th'] = array(
-      'align' => 'optional',
-    );
-    $this->isMarkdownable['td'] = array(
-      'align' => 'optional',
-    );
-    $this->isMarkdownable['tr'] = array();
-    array_push($this->ignore, 'thead');
-    array_push($this->ignore, 'tbody');
-    array_push($this->ignore, 'tfoot');
-    # definition lists
-    $this->isMarkdownable['dl'] = array();
-    $this->isMarkdownable['dd'] = array();
-    $this->isMarkdownable['dt'] = array();
-    # footnotes
-    $this->isMarkdownable['fnref'] = array(
-      'target' => 'required',
-    );
-    $this->isMarkdownable['footnotes'] = array();
-    $this->isMarkdownable['fn'] = array(
-      'name' => 'required',
-    );
-    $this->parser->blockElements['fnref'] = false;
-    $this->parser->blockElements['fn'] = true;
-    $this->parser->blockElements['footnotes'] = true;
-    # abbr
-    $this->isMarkdownable['abbr'] = array(
-      'title' => 'required',
-    );
-    # build RegEx lookahead to decide wether table can pe parsed or not
-    $inlineTags = array_keys($this->parser->blockElements, false);
-    $colContents = '(?:[^<]|<(?:'.implode('|', $inlineTags).'|[^a-z]))+';
-    $this->tableLookaheadHeader = '{
-    ^\s*(?:<thead\s*>)?\s*                               # open optional thead
-      <tr\s*>\s*(?:                                    # start required row with headers
-        <th(?:\s+align=("|\')(?:left|center|right)\1)?\s*>   # header with optional align
-        \s*'.$colContents.'\s*                       # contents
-        </th>\s*                                     # close header
-      )+</tr>                                          # close row with headers
-    \s*(?:</thead>)?                                     # close optional thead
-    }sxi';
-    $this->tdSubstitute = '\s*'.$colContents.'\s*        # contents
-          </td>\s*';
-    $this->tableLookaheadBody = '{
-      \s*(?:<tbody\s*>)?\s*                            # open optional tbody
-        (?:<tr\s*>\s*                                # start row
-          %s                                       # cols to be substituted
-        </tr>)+                                      # close row
-      \s*(?:</tbody>)?                                 # close optional tbody
-    \s*</table>                                          # close table
-    }sxi';
-  }
-  /**
-   * handle header tags (<h1> - <h6>)
-   *
-   * @param int $level 1-6
-   * @return void
-   */
-  function handleHeader($level) {
-    static $id = null;
-    if ($this->parser->isStartTag) {
-      if (isset($this->parser->tagAttributes['id'])) {
-        $id = $this->parser->tagAttributes['id'];
-      }
-    } else {
-      if (!is_null($id)) {
-        $this->out(' {#'.$id.'}');
-        $id = null;
-      }
-    }
-    parent::handleHeader($level);
-  }
-  /**
-   * handle <abbr> tags
-   *
-   * @param void
-   * @return void
-   */
-  function handleTag_abbr() {
-    if ($this->parser->isStartTag) {
-      $this->stack();
-      $this->buffer();
-    } else {
-      $tag = $this->unstack();
-      $tag['text'] = $this->unbuffer();
-      $add = true;
-      foreach ($this->stack['abbr'] as $stacked) {
-        if ($stacked['text'] == $tag['text']) {
-          /** TODO: differing abbr definitions, i.e. different titles for same text **/
-          $add = false;
-          break;
-        }
-      }
-      $this->out($tag['text']);
-      if ($add) {
-        array_push($this->stack['abbr'], $tag);
-      }
-    }
-  }
-  /**
-   * flush stacked abbr tags
-   *
-   * @param void
-   * @return void
-   */
-  function flushStacked_abbr() {
-    $out = array();
-    foreach ($this->stack['abbr'] as $k => $tag) {
-      if (!isset($tag['unstacked'])) {
-        array_push($out, ' *['.$tag['text'].']: '.$tag['title']);
-        $tag['unstacked'] = true;
-        $this->stack['abbr'][$k] = $tag;
-      }
-    }
-    if (!empty($out)) {
-      $this->out("\n\n".implode("\n", $out));
-    }
-  }
-  /**
-   * handle <table> tags
-   *
-   * @param void
-   * @return void
-   */
-  function handleTag_table() {
-    if ($this->parser->isStartTag) {
-      # check if upcoming table can be converted
-      if ($this->keepHTML) {
-        if (preg_match($this->tableLookaheadHeader, $this->parser->html, $matches)) {
-          # header seems good, now check body
-          # get align & number of cols
-          preg_match_all('#<th(?:\s+align=("|\')(left|right|center)\1)?\s*>#si', $matches[0], $cols);
-          $regEx = '';
-          $i = 1;
-          $aligns = array();
-          foreach ($cols[2] as $align) {
-            $align = strtolower($align);
-            array_push($aligns, $align);
-            if (empty($align)) {
-              $align = 'left'; # default value
-            }
-            $td = '\s+align=("|\')'.$align.'\\'.$i;
-            $i++;
-            if ($align == 'left') {
-              # look for empty align or left
-              $td = '(?:'.$td.')?';
-            }
-            $td = '<td'.$td.'\s*>';
-            $regEx .= $td.$this->tdSubstitute;
-          }
-          $regEx = sprintf($this->tableLookaheadBody, $regEx);
-          if (preg_match($regEx, $this->parser->html, $matches, null, strlen($matches[0]))) {
-            # this is a markdownable table tag!
-            $this->table = array(
-              'rows' => array(),
-              'col_widths' => array(),
-              'aligns' => $aligns,
-            );
-            $this->row = 0;
-          } else {
-            # non markdownable table
-            $this->handleTagToText();
-          }
-        } else {
-          # non markdownable table
-          $this->handleTagToText();
-        }
-      } else {
-        $this->table = array(
-          'rows' => array(),
-          'col_widths' => array(),
-          'aligns' => array(),
-        );
-        $this->row = 0;
-      }
-    } else {
-      # finally build the table in Markdown Extra syntax
-      $separator = array();
-      # seperator with correct align identifikators
-      foreach($this->table['aligns'] as $col => $align) {
-        if (!$this->keepHTML && !isset($this->table['col_widths'][$col])) {
-          break;
-        }
-        $left = ' ';
-        $right = ' ';
-        switch ($align) {
-          case 'left':
-            $left = ':';
-            break;
-          case 'center':
-            $right = ':';
-            $left = ':';
-          case 'right':
-            $right = ':';
-            break;
-        }
-        array_push($separator, $left.str_repeat('-', $this->table['col_widths'][$col]).$right);
-      }
-      $separator = '|'.implode('|', $separator).'|';
-
-      $rows = array();
-      # add padding
-      array_walk_recursive($this->table['rows'], array(&$this, 'alignTdContent'));
-      $header = array_shift($this->table['rows']);
-      array_push($rows, '| '.implode(' | ', $header).' |');
-      array_push($rows, $separator);
-      foreach ($this->table['rows'] as $row) {
-        array_push($rows, '| '.implode(' | ', $row).' |');
-      }
-      $this->out(implode("\n".$this->indent, $rows));
-      $this->table = array();
-      $this->setLineBreaks(2);
-    }
-  }
-  /**
-   * properly pad content so it is aligned as whished
-   * should be used with array_walk_recursive on $this->table['rows']
-   *
-   * @param string &$content
-   * @param int $col
-   * @return void
-   */
-  function alignTdContent(&$content, $col) {
-    switch ($this->table['aligns'][$col]) {
-      default:
-      case 'left':
-        $content .= str_repeat(' ', $this->table['col_widths'][$col] - $this->strlen($content));
-        break;
-      case 'right':
-        $content = str_repeat(' ', $this->table['col_widths'][$col] - $this->strlen($content)).$content;
-        break;
-      case 'center':
-        $paddingNeeded = $this->table['col_widths'][$col] - $this->strlen($content);
-        $left = floor($paddingNeeded / 2);
-        $right = $paddingNeeded - $left;
-        $content = str_repeat(' ', $left).$content.str_repeat(' ', $right);
-        break;
-    }
-  }
-  /**
-   * handle <tr> tags
-   *
-   * @param void
-   * @return void
-   */
-  function handleTag_tr() {
-    if ($this->parser->isStartTag) {
-      $this->col = -1;
-    } else {
-      $this->row++;
-    }
-  }
-  /**
-   * handle <td> tags
-   *
-   * @param void
-   * @return void
-   */
-  function handleTag_td() {
-    if ($this->parser->isStartTag) {
-      $this->col++;
-      if (!isset($this->table['col_widths'][$this->col])) {
-        $this->table['col_widths'][$this->col] = 0;
-      }
-      $this->buffer();
-    } else {
-      $buffer = trim($this->unbuffer());
-      $this->table['col_widths'][$this->col] = max($this->table['col_widths'][$this->col], $this->strlen($buffer));
-      $this->table['rows'][$this->row][$this->col] = $buffer;
-    }
-  }
-  /**
-   * handle <th> tags
-   *
-   * @param void
-   * @return void
-   */
-  function handleTag_th() {
-    if (!$this->keepHTML && !isset($this->table['rows'][1]) && !isset($this->table['aligns'][$this->col+1])) {
-      if (isset($this->parser->tagAttributes['align'])) {
-        $this->table['aligns'][$this->col+1] = $this->parser->tagAttributes['align'];
-      } else {
-        $this->table['aligns'][$this->col+1] = '';
-      }
-    }
-    $this->handleTag_td();
-  }
-  /**
-   * handle <dl> tags
-   *
-   * @param void
-   * @return void
-   */
-  function handleTag_dl() {
-    if (!$this->parser->isStartTag) {
-      $this->setLineBreaks(2);
-    }
-  }
-  /**
-   * handle <dt> tags
-   *
-   * @param void
-   * @return void
-   **/
-  function handleTag_dt() {
-    if (!$this->parser->isStartTag) {
-      $this->setLineBreaks(1);
-    }
-  }
-  /**
-   * handle <dd> tags
-   *
-   * @param void
-   * @return void
-   */
-  function handleTag_dd() {
-    if ($this->parser->isStartTag) {
-      if (substr(ltrim($this->parser->html), 0, 3) == '<p>') {
-        # next comes a paragraph, so we'll need an extra line
-        $this->out("\n".$this->indent);
-      } elseif (substr($this->output, -2) == "\n\n") {
-        $this->output = substr($this->output, 0, -1);
-      }
-      $this->out(':   ');
-      $this->indent('    ', false);
-    } else {
-      # lookahead for next dt
-      if (substr(ltrim($this->parser->html), 0, 4) == '<dt>') {
-        $this->setLineBreaks(2);
-      } else {
-        $this->setLineBreaks(1);
-      }
-      $this->indent('    ');
-    }
-  }
-  /**
-   * handle <fnref /> tags (custom footnote references, see markdownify_extra::parseString())
-   *
-   * @param void
-   * @return void
-   */
-  function handleTag_fnref() {
-    $this->out('[^'.$this->parser->tagAttributes['target'].']');
-  }
-  /**
-   * handle <fn> tags (custom footnotes, see markdownify_extra::parseString()
-   * and markdownify_extra::_makeFootnotes())
-   *
-   * @param void
-   * @return void
-   */
-  function handleTag_fn() {
-    if ($this->parser->isStartTag) {
-      $this->out('[^'.$this->parser->tagAttributes['name'].']:');
-      $this->setLineBreaks(1);
-    } else {
-      $this->setLineBreaks(2);
-    }
-    $this->indent('    ');
-  }
-  /**
-   * handle <footnotes> tag (custom footnotes, see markdownify_extra::parseString()
-   *  and markdownify_extra::_makeFootnotes())
-   *
-   *  @param void
-   *  @return void
-   */
-  function handleTag_footnotes() {
-    if (!$this->parser->isStartTag) {
-      $this->setLineBreaks(2);
-    }
-  }
-  /**
-   * parse a HTML string, clean up footnotes prior
-   *
-   * @param string $HTML input
-   * @return string Markdown formatted output
-   */
-  function parseString($html) {
-    /** TODO: custom markdown-extra options, e.g. titles & classes **/
-    # <sup id="fnref:..."><a href"#fn..." rel="footnote">...</a></sup>
-    # => <fnref target="..." />
-    $html = preg_replace('@<sup id="fnref:([^"]+)">\s*<a href="#fn:\1" rel="footnote">\s*\d+\s*</a>\s*</sup>@Us', '<fnref target="$1" />', $html);
-    # <div class="footnotes">
-    # <hr />
-    # <ol>
-    #
-    # <li id="fn:...">...</li>
-    # ...
-    #
-    # </ol>
-    # </div>
-    # =>
-    # <footnotes>
-    #   <fn name="...">...</fn>
-    #   ...
-    # </footnotes>
-    $html = preg_replace_callback('#<div class="footnotes">\s*<hr />\s*<ol>\s*(.+)\s*</ol>\s*</div>#Us', array(&$this, '_makeFootnotes'), $html);
-    return parent::parseString($html);
-  }
-  /**
-   * replace HTML representation of footnotes with something more easily parsable
-   *
-   * @note this is a callback to be used in parseString()
-   *
-   * @param array $matches
-   * @return string
-   */
-  function _makeFootnotes($matches) {
-    # <li id="fn:1">
-    #   ...
-    #   <a href="#fnref:block" rev="footnote">&#8617;</a></p>
-    # </li>
-    # => <fn name="1">...</fn>
-    # remove footnote link
-    $fns = preg_replace('@\s*(&#160;\s*)?<a href="#fnref:[^"]+" rev="footnote"[^>]*>&#8617;</a>\s*@s', '', $matches[1]);
-    # remove empty paragraph
-    $fns = preg_replace('@<p>\s*</p>@s', '', $fns);
-    # <li id="fn:1">...</li> -> <footnote nr="1">...</footnote>
-    $fns = str_replace('<li id="fn:', '<fn name="', $fns);
-
-    $fns = '<footnotes>'.$fns.'</footnotes>';
-    return preg_replace('#</li>\s*(?=(?:<fn|</footnotes>))#s', '</fn>$1', $fns);
-  }
-}

include/markdownify/parsehtml/parsehtml.php

@@ -1,618 +0,0 @@
-<?php
-/**
- * parseHTML is a HTML parser which works with PHP 4 and above.
- * It tries to handle invalid HTML to some degree.
- *
- * @version 1.0 beta
- * @author Milian Wolff (mail@milianw.de, http://milianw.de)
- * @license LGPL, see LICENSE_LGPL.txt and the summary below
- * @copyright (C) 2007  Milian Wolff
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
- */
-class parseHTML {
-  /**
-   * tags which are always empty (<br /> etc.)
-   *
-   * @var array<string>
-   */
-  var $emptyTags = array(
-    'br',
-    'hr',
-    'input',
-    'img',
-    'area',
-    'link',
-    'meta',
-    'param',
-  );
-  /**
-   * tags with preformatted text
-   * whitespaces wont be touched in them
-   *
-   * @var array<string>
-   */
-  var $preformattedTags = array(
-    'script',
-    'style',
-    'pre',
-    'code',
-  );
-  /**
-   * supress HTML tags inside preformatted tags (see above)
-   *
-   * @var bool
-   */
-  var $noTagsInCode = false;
-  /**
-   * html to be parsed
-   *
-   * @var string
-   */
-  var $html = '';
-  /**
-   * node type:
-   *
-   * - tag (see isStartTag)
-   * - text (includes cdata)
-   * - comment
-   * - doctype
-   * - pi (processing instruction)
-   *
-   * @var string
-   */
-  var $nodeType = '';
-  /**
-   * current node content, i.e. either a
-   * simple string (text node), or something like
-   * <tag attrib="value"...>
-   *
-   * @var string
-   */
-  var $node = '';
-  /**
-   * wether current node is an opening tag (<a>) or not (</a>)
-   * set to NULL if current node is not a tag
-   * NOTE: empty tags (<br />) set this to true as well!
-   *
-   * @var bool | null
-   */
-  var $isStartTag = null;
-  /**
-   * wether current node is an empty tag (<br />) or not (<a></a>)
-   *
-   * @var bool | null
-   */
-  var $isEmptyTag = null;
-  /**
-   * tag name
-   *
-   * @var string | null
-   */
-  var $tagName = '';
-  /**
-   * attributes of current tag
-   *
-   * @var array (attribName=>value) | null
-   */
-  var $tagAttributes = null;
-  /**
-   * wether the current tag is a block element
-   *
-   * @var bool | null
-   */
-  var $isBlockElement = null;
-
-  /**
-   * keep whitespace
-   *
-   * @var int
-   */
-  var $keepWhitespace = 0;
-  /**
-   * list of open tags
-   * count this to get current depth
-   *
-   * @var array
-   */
-  var $openTags = array();
-  /**
-   * list of block elements
-   *
-   * @var array
-   * TODO: what shall we do with <del> and <ins> ?!
-   */
-  var $blockElements = array (
-    # tag name => <bool> is block
-    # block elements
-    'address' => true,
-    'blockquote' => true,
-    'center' => true,
-    'del' => true,
-    'dir' => true,
-    'div' => true,
-    'dl' => true,
-    'fieldset' => true,
-    'form' => true,
-    'h1' => true,
-    'h2' => true,
-    'h3' => true,
-    'h4' => true,
-    'h5' => true,
-    'h6' => true,
-    'hr' => true,
-    'ins' => true,
-    'isindex' => true,
-    'menu' => true,
-    'noframes' => true,
-    'noscript' => true,
-    'ol' => true,
-    'p' => true,
-    'pre' => true,
-    'table' => true,
-    'ul' => true,
-    # set table elements and list items to block as well
-    'thead' => true,
-    'tbody' => true,
-    'tfoot' => true,
-    'td' => true,
-    'tr' => true,
-    'th' => true,
-    'li' => true,
-    'dd' => true,
-    'dt' => true,
-    # header items and html / body as well
-    'html' => true,
-    'body' => true,
-    'head' => true,
-    'meta' => true,
-    'link' => true,
-    'style' => true,
-    'title' => true,
-    # unfancy media tags, when indented should be rendered as block
-    'map' => true,
-    'object' => true,
-    'param' => true,
-    'embed' => true,
-    'area' => true,
-    # inline elements
-    'a' => false,
-    'abbr' => false,
-    'acronym' => false,
-    'applet' => false,
-    'b' => false,
-    'basefont' => false,
-    'bdo' => false,
-    'big' => false,
-    'br' => false,
-    'button' => false,
-    'cite' => false,
-    'code' => false,
-    'del' => false,
-    'dfn' => false,
-    'em' => false,
-    'font' => false,
-    'i' => false,
-    'img' => false,
-    'ins' => false,
-    'input' => false,
-    'iframe' => false,
-    'kbd' => false,
-    'label' => false,
-    'q' => false,
-    'samp' => false,
-    'script' => false,
-    'select' => false,
-    'small' => false,
-    'span' => false,
-    'strong' => false,
-    'sub' => false,
-    'sup' => false,
-    'textarea' => false,
-    'tt' => false,
-    'var' => false,
-  );
-  /**
-   * get next node, set $this->html prior!
-   *
-   * @param void
-   * @return bool
-   */
-  function nextNode() {
-    if (empty($this->html)) {
-      # we are done with parsing the html string
-      return false;
-    }
-    static $skipWhitespace = true;
-    if ($this->isStartTag && !$this->isEmptyTag) {
-      array_push($this->openTags, $this->tagName);
-      if (in_array($this->tagName, $this->preformattedTags)) {
-        # dont truncate whitespaces for <code> or <pre> contents
-        $this->keepWhitespace++;
-      }
-    }
-
-    if ($this->html[0] == '<') {
-      $token = substr($this->html, 0, 9);
-      if (substr($token, 0, 2) == '<?') {
-        # xml prolog or other pi's
-        /** TODO **/
-        #trigger_error('this might need some work', E_USER_NOTICE);
-        $pos = strpos($this->html, '>');
-        $this->setNode('pi', $pos + 1);
-        return true;
-      }
-      if (substr($token, 0, 4) == '<!--') {
-        # comment
-        $pos = strpos($this->html, '-->');
-        if ($pos === false) {
-          # could not find a closing -->, use next gt instead
-          # this is firefox' behaviour
-          $pos = strpos($this->html, '>') + 1;
-        } else {
-          $pos += 3;
-        }
-        $this->setNode('comment', $pos);
-
-        $skipWhitespace = true;
-        return true;
-      }
-      if ($token == '<!DOCTYPE') {
-        # doctype
-        $this->setNode('doctype', strpos($this->html, '>')+1);
-
-        $skipWhitespace = true;
-        return true;
-      }
-      if ($token == '<![CDATA[') {
-        # cdata, use text node
-
-        # remove leading <![CDATA[
-        $this->html = substr($this->html, 9);
-
-        $this->setNode('text', strpos($this->html, ']]>')+3);
-
-        # remove trailing ]]> and trim
-        $this->node = substr($this->node, 0, -3);
-        $this->handleWhitespaces();
-
-        $skipWhitespace = true;
-        return true;
-      }
-      if ($this->parseTag()) {
-        # seems to be a tag
-        # handle whitespaces
-        if ($this->isBlockElement) {
-          $skipWhitespace = true;
-        } else {
-          $skipWhitespace = false;
-        }
-        return true;
-      }
-    }
-    if ($this->keepWhitespace) {
-      $skipWhitespace = false;
-    }
-    # when we get here it seems to be a text node
-    $pos = strpos($this->html, '<');
-    if ($pos === false) {
-      $pos = strlen($this->html);
-    }
-    $this->setNode('text', $pos);
-    $this->handleWhitespaces();
-    if ($skipWhitespace && $this->node == ' ') {
-      return $this->nextNode();
-    }
-    $skipWhitespace = false;
-    return true;
-  }
-  /**
-   * parse tag, set tag name and attributes, see if it's a closing tag and so forth...
-   *
-   * @param void
-   * @return bool
-   */
-  function parseTag() {
-    static $a_ord, $z_ord, $special_ords;
-    if (!isset($a_ord)) {
-      $a_ord = ord('a');
-      $z_ord = ord('z');
-      $special_ords = array(
-        ord(':'), // for xml:lang
-        ord('-'), // for http-equiv
-      );
-    }
-
-    $tagName = '';
-
-    $pos = 1;
-    $isStartTag = $this->html[$pos] != '/';
-    if (!$isStartTag) {
-      $pos++;
-    }
-    # get tagName
-    while (isset($this->html[$pos])) {
-      $pos_ord = ord(strtolower($this->html[$pos]));
-      if (($pos_ord >= $a_ord && $pos_ord <= $z_ord) || (!empty($tagName) && is_numeric($this->html[$pos]))) {
-        $tagName .= $this->html[$pos];
-        $pos++;
-      } else {
-        $pos--;
-        break;
-      }
-    }
-
-    $tagName = strtolower($tagName);
-    if (empty($tagName) || !isset($this->blockElements[$tagName])) {
-      # something went wrong => invalid tag
-      $this->invalidTag();
-      return false;
-    }
-    if ($this->noTagsInCode && end($this->openTags) == 'code' && !($tagName == 'code' && !$isStartTag)) {
-      # we supress all HTML tags inside code tags
-      $this->invalidTag();
-      return false;
-    }
-
-    # get tag attributes
-    /** TODO: in html 4 attributes do not need to be quoted **/
-    $isEmptyTag = false;
-    $attributes = array();
-    $currAttrib = '';
-    while (isset($this->html[$pos+1])) {
-      $pos++;
-      # close tag
-      if ($this->html[$pos] == '>' || $this->html[$pos].$this->html[$pos+1] == '/>') {
-        if ($this->html[$pos] == '/') {
-          $isEmptyTag = true;
-          $pos++;
-        }
-        break;
-      }
-
-      $pos_ord = ord(strtolower($this->html[$pos]));
-      if ( ($pos_ord >= $a_ord && $pos_ord <= $z_ord) || in_array($pos_ord, $special_ords)) {
-        # attribute name
-        $currAttrib .= $this->html[$pos];
-      } elseif (in_array($this->html[$pos], array(' ', "\t", "\n"))) {
-        # drop whitespace
-      } elseif (in_array($this->html[$pos].$this->html[$pos+1], array('="', "='"))) {
-        # get attribute value
-        $pos++;
-        $await = $this->html[$pos]; # single or double quote
-        $pos++;
-        $value = '';
-        while (isset($this->html[$pos]) && $this->html[$pos] != $await) {
-          $value .= $this->html[$pos];
-          $pos++;
-        }
-        $attributes[$currAttrib] = $value;
-        $currAttrib = '';
-      } else {
-        $this->invalidTag();
-        return false;
-      }
-    }
-    if ($this->html[$pos] != '>') {
-      $this->invalidTag();
-      return false;
-    }
-
-    if (!empty($currAttrib)) {
-      # html 4 allows something like <option selected> instead of <option selected="selected">
-      $attributes[$currAttrib] = $currAttrib;
-    }
-    if (!$isStartTag) {
-      if (!empty($attributes) || $tagName != end($this->openTags)) {
-        # end tags must not contain any attributes
-        # or maybe we did not expect a different tag to be closed
-        $this->invalidTag();
-        return false;
-      }
-      array_pop($this->openTags);
-      if (in_array($tagName, $this->preformattedTags)) {
-        $this->keepWhitespace--;
-      }
-    }
-    $pos++;
-    $this->node = substr($this->html, 0, $pos);
-    $this->html = substr($this->html, $pos);
-    $this->tagName = $tagName;
-    $this->tagAttributes = $attributes;
-    $this->isStartTag = $isStartTag;
-    $this->isEmptyTag = $isEmptyTag || in_array($tagName, $this->emptyTags);
-    if ($this->isEmptyTag) {
-      # might be not well formed
-      $this->node = preg_replace('# */? *>$#', ' />', $this->node);
-    }
-    $this->nodeType = 'tag';
-    $this->isBlockElement = $this->blockElements[$tagName];
-    return true;
-  }
-  /**
-   * handle invalid tags
-   *
-   * @param void
-   * @return void
-   */
-  function invalidTag() {
-    $this->html = substr_replace($this->html, '&lt;', 0, 1);
-  }
-  /**
-   * update all vars and make $this->html shorter
-   *
-   * @param string $type see description for $this->nodeType
-   * @param int $pos to which position shall we cut?
-   * @return void
-   */
-  function setNode($type, $pos) {
-    if ($this->nodeType == 'tag') {
-      # set tag specific vars to null
-      # $type == tag should not be called here
-      # see this::parseTag() for more
-      $this->tagName = null;
-      $this->tagAttributes = null;
-      $this->isStartTag = null;
-      $this->isEmptyTag = null;
-      $this->isBlockElement = null;
-
-    }
-    $this->nodeType = $type;
-    $this->node = substr($this->html, 0, $pos);
-    $this->html = substr($this->html, $pos);
-  }
-  /**
-   * check if $this->html begins with $str
-   *
-   * @param string $str
-   * @return bool
-   */
-  function match($str) {
-    return substr($this->html, 0, strlen($str)) == $str;
-  }
-  /**
-   * truncate whitespaces
-   *
-   * @param void
-   * @return void
-   */
-  function handleWhitespaces() {
-    if ($this->keepWhitespace) {
-      # <pre> or <code> before...
-      return;
-    }
-    # truncate multiple whitespaces to a single one
-    $this->node = preg_replace('#\s+#s', ' ', $this->node);
-  }
-  /**
-   * normalize self::node
-   *
-   * @param void
-   * @return void
-   */
-  function normalizeNode() {
-    $this->node = '<';
-    if (!$this->isStartTag) {
-      $this->node .= '/'.$this->tagName.'>';
-      return;
-    }
-    $this->node .= $this->tagName;
-    foreach ($this->tagAttributes as $name => $value) {
-      $this->node .= ' '.$name.'="'.str_replace('"', '&quot;', $value).'"';
-    }
-    if ($this->isEmptyTag) {
-      $this->node .= ' /';
-    }
-    $this->node .= '>';
-  }
-}
-
-/**
- * indent a HTML string properly
- *
- * @param string $html
- * @param string $indent optional
- * @return string
- */
-function indentHTML($html, $indent = "  ", $noTagsInCode = false) {
-  $parser = new parseHTML;
-  $parser->noTagsInCode = $noTagsInCode;
-  $parser->html = $html;
-  $html = '';
-  $last = true; # last tag was block elem
-  $indent_a = array();
-  while($parser->nextNode()) {
-    if ($parser->nodeType == 'tag') {
-      $parser->normalizeNode();
-    }
-    if ($parser->nodeType == 'tag' && $parser->isBlockElement) {
-      $isPreOrCode = in_array($parser->tagName, array('code', 'pre'));
-      if (!$parser->keepWhitespace && !$last && !$isPreOrCode) {
-        $html = rtrim($html)."\n";
-      }
-      if ($parser->isStartTag) {
-        $html .= implode($indent_a);
-        if (!$parser->isEmptyTag) {
-          array_push($indent_a, $indent);
-        }
-      } else {
-        array_pop($indent_a);
-        if (!$isPreOrCode) {
-          $html .= implode($indent_a);
-        }
-      }
-      $html .= $parser->node;
-      if (!$parser->keepWhitespace && !($isPreOrCode && $parser->isStartTag)) {
-        $html .= "\n";
-      }
-      $last = true;
-    } else {
-      if ($parser->nodeType == 'tag' && $parser->tagName == 'br') {
-        $html .= $parser->node."\n";
-        $last = true;
-        continue;
-      } elseif ($last && !$parser->keepWhitespace) {
-        $html .= implode($indent_a);
-        $parser->node = ltrim($parser->node);
-      }
-      $html .= $parser->node;
-
-      if (in_array($parser->nodeType, array('comment', 'pi', 'doctype'))) {
-        $html .= "\n";
-      } else {
-        $last = false;
-      }
-    }
-  }
-  return $html;
-}
-/*
-# testcase / example
-error_reporting(E_ALL);
-
-$html = '<p>Simple block on one line:</p>
-
-<div>foo</div>
-
-<p>And nested without indentation:</p>
-
-<div>
-<div>
-<div>
-foo
-</div>
-<div style=">"/>
-</div>
-<div>bar</div>
-</div>
-
-<p>And with attributes:</p>
-
-<div>
-    <div id="foo">
-    </div>
-</div>
-
-<p>This was broken in 1.0.2b7:</p>
-
-<div class="inlinepage">
-<div class="toggleableend">
-foo
-</div>
-</div>';
-#$html = '<a href="asdfasdf"       title=\'asdf\' foo="bar">asdf</a>';
-echo indentHTML($html);
-die();
-*/

include/nav.php

@@ -125,8 +125,10 @@ function nav_info(&$a) {
 		if(strlen($gdir))
 			$gdirpath = $gdir;
 	}
-	elseif(! get_config('system','no_community_page'))
+	elseif(get_config('system','community_page_style') == CP_USERS_ON_SERVER)
 		$nav['community'] = array('community', t('Community'), "", t('Conversations on this site'));
+	elseif(get_config('system','community_page_style') == CP_GLOBAL_COMMUNITY)
+		$nav['community'] = array('community', t('Community'), "", t('Conversations on the network'));
 
 	$nav['directory'] = array($gdirpath, t('Directory'), "", t('People directory'));
 

include/network.php

@@ -50,6 +50,7 @@ function fetch_url($url,$binary = false, &$redirects = 0, $timeout = 0, $accept_
 
 	$check_cert = get_config('system','verifyssl');
 	@curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, (($check_cert) ? true : false));
+	@curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, (($check_cert) ? 2 : false));
 
 	$prx = get_config('system','proxy');
 	if(strlen($prx)) {
@@ -71,6 +72,7 @@ function fetch_url($url,$binary = false, &$redirects = 0, $timeout = 0, $accept_
 
 	$base = $s;
 	$curl_info = @curl_getinfo($ch);
+	@curl_close($ch);
 	$http_code = $curl_info['http_code'];
 	logger('fetch_url '.$url.': '.$http_code." ".$s, LOGGER_DATA);
 	$header = '';
@@ -94,7 +96,7 @@ function fetch_url($url,$binary = false, &$redirects = 0, $timeout = 0, $accept_
 			$newurl = $new_location_info["scheme"]."://".$new_location_info["host"].$old_location_info["path"];
 
 		$matches = array();
-		if (preg_match('/(Location:|URI:)(.*?)\n/', $header, $matches)) {
+		if (preg_match('/(Location:|URI:)(.*?)\n/i', $header, $matches)) {
 			$newurl = trim(array_pop($matches));
 		}
 		if(strpos($newurl,'/') === 0)
@@ -110,7 +112,6 @@ function fetch_url($url,$binary = false, &$redirects = 0, $timeout = 0, $accept_
 
 	$body = substr($s,strlen($header));
 	$a->set_curl_headers($header);
-	@curl_close($ch);
 
 	$a->save_timestamp($stamp1, "network");
 
@@ -158,6 +159,7 @@ function post_url($url,$params, $headers = null, &$redirects = 0, $timeout = 0)
 
 	$check_cert = get_config('system','verifyssl');
 	curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, (($check_cert) ? true : false));
+	curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, (($check_cert) ? 2 : false));
 	$prx = get_config('system','proxy');
 	if(strlen($prx)) {
 		curl_setopt($ch, CURLOPT_HTTPPROXYTUNNEL, 1);
@@ -259,43 +261,43 @@ function http_status_exit($val) {
 if(! function_exists('convert_xml_element_to_array')) {
 function convert_xml_element_to_array($xml_element, &$recursion_depth=0) {
 
-        // If we're getting too deep, bail out
+	// If we're getting too deep, bail out
-        if ($recursion_depth > 512) {
+	if ($recursion_depth > 512) {
-                return(null);
+		return(null);
-        }
+	}
 
-        if (!is_string($xml_element) &&
+	if (!is_string($xml_element) &&
-        !is_array($xml_element) &&
+	!is_array($xml_element) &&
-        (get_class($xml_element) == 'SimpleXMLElement')) {
+	(get_class($xml_element) == 'SimpleXMLElement')) {
-                $xml_element_copy = $xml_element;
+		$xml_element_copy = $xml_element;
-                $xml_element = get_object_vars($xml_element);
+		$xml_element = get_object_vars($xml_element);
-        }
+	}
 
-        if (is_array($xml_element)) {
+	if (is_array($xml_element)) {
-                $result_array = array();
+		$result_array = array();
-                if (count($xml_element) <= 0) {
+		if (count($xml_element) <= 0) {
-                        return (trim(strval($xml_element_copy)));
+			return (trim(strval($xml_element_copy)));
-                }
+		}
 
-                foreach($xml_element as $key=>$value) {
+		foreach($xml_element as $key=>$value) {
 
-                        $recursion_depth++;
+			$recursion_depth++;
-                        $result_array[strtolower($key)] =
+			$result_array[strtolower($key)] =
-                convert_xml_element_to_array($value, $recursion_depth);
+		convert_xml_element_to_array($value, $recursion_depth);
-                        $recursion_depth--;
+			$recursion_depth--;
-                }
+		}
-                if ($recursion_depth == 0) {
+		if ($recursion_depth == 0) {
-                        $temp_array = $result_array;
+			$temp_array = $result_array;
-                        $result_array = array(
+			$result_array = array(
-                                strtolower($xml_element_copy->getName()) => $temp_array,
+				strtolower($xml_element_copy->getName()) => $temp_array,
-                        );
+			);
-                }
+		}
 
-                return ($result_array);
+		return ($result_array);
 
-        } else {
+	} else {
-                return (trim(strval($xml_element)));
+		return (trim(strval($xml_element)));
-        }
+	}
 }}
 
 // Given an email style address, perform webfinger lookup and
@@ -651,7 +653,7 @@ function validate_email($addr) {
 		return false;
 	$h = substr($addr,strpos($addr,'@') + 1);
 
-	if(($h) && (dns_get_record($h, DNS_A + DNS_CNAME + DNS_PTR + DNS_MX) || filter_var($h['host'], FILTER_VALIDATE_IP) )) {
+	if(($h) && (dns_get_record($h, DNS_A + DNS_CNAME + DNS_PTR + DNS_MX) || filter_var($h, FILTER_VALIDATE_IP) )) {
 		return true;
 	}
 	return false;
@@ -868,13 +870,6 @@ function scale_external_images($srctext, $include_link = true, $scale_replace =
 			if(! $i)
 				return $srctext;
 
-			$cachefile = get_cachefile(hash("md5", $scaled));
-			if ($cachefile != '') {
-				$stamp1 = microtime(true);
-				file_put_contents($cachefile, $i);
-				$a->save_timestamp($stamp1, "file");
-			}
-
 			// guess mimetype from headers or filename
 			$type = guess_image_type($mtch[1],true);
 
@@ -969,8 +964,8 @@ function xml2array($contents, $namespaces = true, $get_attributes=1, $priority =
     if(!$contents) return array();
 
     if(!function_exists('xml_parser_create')) {
-        logger('xml2array: parser function missing');
+	logger('xml2array: parser function missing');
-        return array();
+	return array();
     }
 
 
@@ -1013,29 +1008,29 @@ function xml2array($contents, $namespaces = true, $get_attributes=1, $priority =
     // Go through the tags.
     $repeated_tag_index = array(); // Multiple tags with same name will be turned into an array
     foreach($xml_values as $data) {
-        unset($attributes,$value); // Remove existing values, or there will be trouble
+	unset($attributes,$value); // Remove existing values, or there will be trouble
 
-        // This command will extract these variables into the foreach scope
+	// This command will extract these variables into the foreach scope
-        // tag(string), type(string), level(int), attributes(array).
+	// tag(string), type(string), level(int), attributes(array).
-        extract($data); // We could use the array by itself, but this cooler.
+	extract($data); // We could use the array by itself, but this cooler.
 
-        $result = array();
+	$result = array();
-        $attributes_data = array();
+	$attributes_data = array();
 
-        if(isset($value)) {
+	if(isset($value)) {
-            if($priority == 'tag') $result = $value;
+	    if($priority == 'tag') $result = $value;
-            else $result['value'] = $value; // Put the value in a assoc array if we are in the 'Attribute' mode
+	    else $result['value'] = $value; // Put the value in a assoc array if we are in the 'Attribute' mode
-        }
+	}
 
-        //Set the attributes too.
+	//Set the attributes too.
-        if(isset($attributes) and $get_attributes) {
+	if(isset($attributes) and $get_attributes) {
-            foreach($attributes as $attr => $val) {
+	    foreach($attributes as $attr => $val) {
-                if($priority == 'tag') $attributes_data[$attr] = $val;
+		if($priority == 'tag') $attributes_data[$attr] = $val;
-                else $result['@attributes'][$attr] = $val; // Set all the attributes in a array called 'attr'
+		else $result['@attributes'][$attr] = $val; // Set all the attributes in a array called 'attr'
-            }
+	    }
-        }
+	}
 
-        // See tag status and do the needed.
+	// See tag status and do the needed.
 		if($namespaces && strpos($tag,':')) {
 			$namespc = substr($tag,0,strrpos($tag,':'));
 			$tag = strtolower(substr($tag,strlen($namespc)+1));
@@ -1044,72 +1039,72 @@ function xml2array($contents, $namespaces = true, $get_attributes=1, $priority =
 		$tag = strtolower($tag);
 
 		if($type == "open") {   // The starting of the tag '<tag>'
-            $parent[$level-1] = &$current;
+	    $parent[$level-1] = &$current;
-            if(!is_array($current) or (!in_array($tag, array_keys($current)))) { // Insert New tag
+	    if(!is_array($current) or (!in_array($tag, array_keys($current)))) { // Insert New tag
-                $current[$tag] = $result;
+		$current[$tag] = $result;
-                if($attributes_data) $current[$tag. '_attr'] = $attributes_data;
+		if($attributes_data) $current[$tag. '_attr'] = $attributes_data;
-                $repeated_tag_index[$tag.'_'.$level] = 1;
+		$repeated_tag_index[$tag.'_'.$level] = 1;
 
-                $current = &$current[$tag];
+		$current = &$current[$tag];
 
-            } else { // There was another element with the same tag name
+	    } else { // There was another element with the same tag name
 
-                if(isset($current[$tag][0])) { // If there is a 0th element it is already an array
+		if(isset($current[$tag][0])) { // If there is a 0th element it is already an array
-                    $current[$tag][$repeated_tag_index[$tag.'_'.$level]] = $result;
+		    $current[$tag][$repeated_tag_index[$tag.'_'.$level]] = $result;
-                    $repeated_tag_index[$tag.'_'.$level]++;
+		    $repeated_tag_index[$tag.'_'.$level]++;
-                } else { // This section will make the value an array if multiple tags with the same name appear together
+		} else { // This section will make the value an array if multiple tags with the same name appear together
-                    $current[$tag] = array($current[$tag],$result); // This will combine the existing item and the new item together to make an array
+		    $current[$tag] = array($current[$tag],$result); // This will combine the existing item and the new item together to make an array
-                    $repeated_tag_index[$tag.'_'.$level] = 2;
+		    $repeated_tag_index[$tag.'_'.$level] = 2;
 
-                    if(isset($current[$tag.'_attr'])) { // The attribute of the last(0th) tag must be moved as well
+		    if(isset($current[$tag.'_attr'])) { // The attribute of the last(0th) tag must be moved as well
-                        $current[$tag]['0_attr'] = $current[$tag.'_attr'];
+			$current[$tag]['0_attr'] = $current[$tag.'_attr'];
-                        unset($current[$tag.'_attr']);
+			unset($current[$tag.'_attr']);
-                    }
+		    }
 
-                }
+		}
-                $last_item_index = $repeated_tag_index[$tag.'_'.$level]-1;
+		$last_item_index = $repeated_tag_index[$tag.'_'.$level]-1;
-                $current = &$current[$tag][$last_item_index];
+		$current = &$current[$tag][$last_item_index];
-            }
+	    }
 
-        } elseif($type == "complete") { // Tags that ends in 1 line '<tag />'
+	} elseif($type == "complete") { // Tags that ends in 1 line '<tag />'
-            //See if the key is already taken.
+	    //See if the key is already taken.
-            if(!isset($current[$tag])) { //New Key
+	    if(!isset($current[$tag])) { //New Key
-                $current[$tag] = $result;
+		$current[$tag] = $result;
-                $repeated_tag_index[$tag.'_'.$level] = 1;
+		$repeated_tag_index[$tag.'_'.$level] = 1;
-                if($priority == 'tag' and $attributes_data) $current[$tag. '_attr'] = $attributes_data;
+		if($priority == 'tag' and $attributes_data) $current[$tag. '_attr'] = $attributes_data;
 
-            } else { // If taken, put all things inside a list(array)
+	    } else { // If taken, put all things inside a list(array)
-                if(isset($current[$tag][0]) and is_array($current[$tag])) { // If it is already an array...
+		if(isset($current[$tag][0]) and is_array($current[$tag])) { // If it is already an array...
 
-                    // ...push the new element into that array.
+		    // ...push the new element into that array.
-                    $current[$tag][$repeated_tag_index[$tag.'_'.$level]] = $result;
+		    $current[$tag][$repeated_tag_index[$tag.'_'.$level]] = $result;
 
-                    if($priority == 'tag' and $get_attributes and $attributes_data) {
+		    if($priority == 'tag' and $get_attributes and $attributes_data) {
-                        $current[$tag][$repeated_tag_index[$tag.'_'.$level] . '_attr'] = $attributes_data;
+			$current[$tag][$repeated_tag_index[$tag.'_'.$level] . '_attr'] = $attributes_data;
-                    }
+		    }
-                    $repeated_tag_index[$tag.'_'.$level]++;
+		    $repeated_tag_index[$tag.'_'.$level]++;
 
-                } else { // If it is not an array...
+		} else { // If it is not an array...
-                    $current[$tag] = array($current[$tag],$result); //...Make it an array using using the existing value and the new value
+		    $current[$tag] = array($current[$tag],$result); //...Make it an array using using the existing value and the new value
-                    $repeated_tag_index[$tag.'_'.$level] = 1;
+		    $repeated_tag_index[$tag.'_'.$level] = 1;
-                    if($priority == 'tag' and $get_attributes) {
+		    if($priority == 'tag' and $get_attributes) {
-                        if(isset($current[$tag.'_attr'])) { // The attribute of the last(0th) tag must be moved as well
+			if(isset($current[$tag.'_attr'])) { // The attribute of the last(0th) tag must be moved as well
 
-                            $current[$tag]['0_attr'] = $current[$tag.'_attr'];
+			    $current[$tag]['0_attr'] = $current[$tag.'_attr'];
-                            unset($current[$tag.'_attr']);
+			    unset($current[$tag.'_attr']);
-                        }
+			}
 
-                        if($attributes_data) {
+			if($attributes_data) {
-                            $current[$tag][$repeated_tag_index[$tag.'_'.$level] . '_attr'] = $attributes_data;
+			    $current[$tag][$repeated_tag_index[$tag.'_'.$level] . '_attr'] = $attributes_data;
-                        }
+			}
-                    }
+		    }
-                    $repeated_tag_index[$tag.'_'.$level]++; // 0 and 1 indexes are already taken
+		    $repeated_tag_index[$tag.'_'.$level]++; // 0 and 1 indexes are already taken
-                }
+		}
-            }
+	    }
 
-        } elseif($type == 'close') { // End of tag '</tag>'
+	} elseif($type == 'close') { // End of tag '</tag>'
-            $current = &$parent[$level-1];
+	    $current = &$parent[$level-1];
-        }
+	}
     }
 
     return($xml_array);
@@ -1151,71 +1146,91 @@ function original_url($url, $depth=1, $fetchbody = false) {
 			$url = substr($url, 0, -1);
 	}
 
-        if ($depth > 10)
+	if ($depth > 10)
-        	return($url);
+		return($url);
 
-        $url = trim($url, "'");
+	$url = trim($url, "'");
 
-        $siteinfo = array();
+	$stamp1 = microtime(true);
-        $ch = curl_init();
-        curl_setopt($ch, CURLOPT_URL, $url);
-        curl_setopt($ch, CURLOPT_HEADER, 1);
 
-        if ($fetchbody)
+	$siteinfo = array();
-                curl_setopt($ch, CURLOPT_NOBODY, 0);
+	$ch = curl_init();
-        else
+	curl_setopt($ch, CURLOPT_URL, $url);
-                curl_setopt($ch, CURLOPT_NOBODY, 1);
+	curl_setopt($ch, CURLOPT_HEADER, 1);
-
+	curl_setopt($ch, CURLOPT_NOBODY, 1);
-        curl_setopt($ch, CURLOPT_TIMEOUT, 10);
+	curl_setopt($ch, CURLOPT_TIMEOUT, 10);
-        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
+	curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
 	curl_setopt($ch, CURLOPT_USERAGENT, $a->get_useragent());
 
-        $header = curl_exec($ch);
+	$header = curl_exec($ch);
-        $curl_info = @curl_getinfo($ch);
+	$curl_info = @curl_getinfo($ch);
-        $http_code = $curl_info['http_code'];
+	$http_code = $curl_info['http_code'];
-        curl_close($ch);
+	curl_close($ch);
 
-        if ((($curl_info['http_code'] == "301") OR ($curl_info['http_code'] == "302"))
+	$a->save_timestamp($stamp1, "network");
-                AND (($curl_info['redirect_url'] != "") OR ($curl_info['location'] != ""))) {
-                if ($curl_info['redirect_url'] != "")
-                        return(original_url($curl_info['redirect_url'], ++$depth, $fetchbody));
-                else
-                        return(original_url($curl_info['location'], ++$depth, $fetchbody));
-        }
 
-        $pos = strpos($header, "\r\n\r\n");
+	if ((($curl_info['http_code'] == "301") OR ($curl_info['http_code'] == "302"))
+		AND (($curl_info['redirect_url'] != "") OR ($curl_info['location'] != ""))) {
+		if ($curl_info['redirect_url'] != "")
+			return(original_url($curl_info['redirect_url'], ++$depth, $fetchbody));
+		else
+			return(original_url($curl_info['location'], ++$depth, $fetchbody));
+	}
 
-        if ($pos)
+	// Check for redirects in the meta elements of the body if there are no redirects in the header.
-                $body = trim(substr($header, $pos));
+	if (!$fetchbody)
-        else
+		return(original_url($url, ++$depth, true));
-                $body = $header;
 
-        if (trim($body) == "")
+	// if the file is too large then exit
-                return(original_url($url, ++$depth, true));
+	if ($curl_info["download_content_length"] > 1000000)
+		return($url);
 
-        $doc = new DOMDocument();
+	// if it isn't a HTML file then exit
-        @$doc->loadHTML($body);
+	if (($curl_info["content_type"] != "") AND !strstr(strtolower($curl_info["content_type"]),"html"))
+		return($url);
 
-        $xpath = new DomXPath($doc);
+	$stamp1 = microtime(true);
 
-        $list = $xpath->query("//meta[@content]");
+	$ch = curl_init();
-        foreach ($list as $node) {
+	curl_setopt($ch, CURLOPT_URL, $url);
-                $attr = array();
+	curl_setopt($ch, CURLOPT_HEADER, 0);
-                if ($node->attributes->length)
+	curl_setopt($ch, CURLOPT_NOBODY, 0);
-                        foreach ($node->attributes as $attribute)
+	curl_setopt($ch, CURLOPT_TIMEOUT, 10);
-                                $attr[$attribute->name] = $attribute->value;
+	curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
+	curl_setopt($ch, CURLOPT_USERAGENT, $a->get_useragent());
 
-                if (@$attr["http-equiv"] == 'refresh') {
+	$body = curl_exec($ch);
-                        $path = $attr["content"];
+	curl_close($ch);
-                        $pathinfo = explode(";", $path);
-                        $content = "";
-                        foreach ($pathinfo AS $value)
-                                if (substr(strtolower($value), 0, 4) == "url=")
-                                        return(original_url(substr($value, 4), ++$depth));
-                }
-        }
 
-        return($url);
+	$a->save_timestamp($stamp1, "network");
+
+	if (trim($body) == "")
+		return($url);
+
+	// Check for redirect in meta elements
+	$doc = new DOMDocument();
+	@$doc->loadHTML($body);
+
+	$xpath = new DomXPath($doc);
+
+	$list = $xpath->query("//meta[@content]");
+	foreach ($list as $node) {
+		$attr = array();
+		if ($node->attributes->length)
+			foreach ($node->attributes as $attribute)
+				$attr[$attribute->name] = $attribute->value;
+
+		if (@$attr["http-equiv"] == 'refresh') {
+			$path = $attr["content"];
+			$pathinfo = explode(";", $path);
+			$content = "";
+			foreach ($pathinfo AS $value)
+				if (substr(strtolower($value), 0, 4) == "url=")
+					return(original_url(substr($value, 4), ++$depth));
+		}
+	}
+
+	return($url);
 }
 
 if (!function_exists('short_link')) {

include/notifier.php

@@ -5,16 +5,16 @@ require_once('include/html2plain.php');
 
 /*
  * This file was at one time responsible for doing all deliveries, but this caused
- * big problems on shared hosting systems, where the process might get killed by the 
+ * big problems on shared hosting systems, where the process might get killed by the
- * hosting provider and nothing would get delivered. 
+ * hosting provider and nothing would get delivered.
  * It now only delivers one message under certain cases, and invokes a queued
- * delivery mechanism (include/deliver.php) to deliver individual contacts at 
+ * delivery mechanism (include/deliver.php) to deliver individual contacts at
  * controlled intervals.
  * This has a much better chance of surviving random processes getting killed
- * by the hosting provider. 
+ * by the hosting provider.
  * A lot of this code is duplicated in include/deliver.php until we have time to go back
- * and re-structure the delivery procedure based on the obstacles that have been thrown at 
+ * and re-structure the delivery procedure based on the obstacles that have been thrown at
- * us by hosting providers. 
+ * us by hosting providers.
  */
 
 /*
@@ -50,12 +50,12 @@ function notifier_run(&$argv, &$argc){
 	if(is_null($a)){
 		$a = new App;
 	}
-  
+
 	if(is_null($db)) {
 		@include(".htconfig.php");
 		require_once("include/dba.php");
 		$db = new dba($db_host, $db_user, $db_pass, $db_data);
-		        unset($db_host, $db_user, $db_pass, $db_data);
+			unset($db_host, $db_user, $db_pass, $db_data);
 	}
 
 	require_once("include/session.php");
@@ -90,7 +90,7 @@ function notifier_run(&$argv, &$argc){
 	$expire = false;
 	$mail = false;
 	$fsuggest = false;
-    $relocate = false;
+	$relocate = false;
 	$top_level = false;
 	$recipients = array();
 	$url_recipients = array();
@@ -135,8 +135,7 @@ function notifier_run(&$argv, &$argc){
 		$uid = $suggest[0]['uid'];
 		$recipients[] = $suggest[0]['cid'];
 		$item = $suggest[0];
-	}
+	} elseif($cmd === 'removeme') {
-	elseif($cmd === 'removeme') {
 		$r = q("SELECT * FROM `user` WHERE `uid` = %d LIMIT 1", intval($item_id));
 		if (! $r)
 			return;
@@ -156,13 +155,11 @@ function notifier_run(&$argv, &$argc){
 			terminate_friendship($user, $self, $contact);
 		}
 		return;
-	}
+	} elseif($cmd === 'relocate') {
-    elseif($cmd === 'relocate') {
+		$normal_mode = false;
-        $normal_mode = false;
 		$relocate = true;
-        $uid = $item_id;
+		$uid = $item_id;
-    }
+	} else {
-	else {
 		// find ancestors
 		$r = q("SELECT * FROM `item` WHERE `id` = %d and visible = 1 and moderated = 0 LIMIT 1",
 			intval($item_id)
@@ -204,10 +201,10 @@ function notifier_run(&$argv, &$argc){
 
 	}
 
-	$r = q("SELECT `contact`.*, `user`.`pubkey` AS `upubkey`, `user`.`prvkey` AS `uprvkey`, 
+	$r = q("SELECT `contact`.*, `user`.`pubkey` AS `upubkey`, `user`.`prvkey` AS `uprvkey`,
-		`user`.`timezone`, `user`.`nickname`, `user`.`sprvkey`, `user`.`spubkey`, 
+		`user`.`timezone`, `user`.`nickname`, `user`.`sprvkey`, `user`.`spubkey`,
 		`user`.`page-flags`, `user`.`prvnets`
-		FROM `contact` INNER JOIN `user` ON `user`.`uid` = `contact`.`uid` 
+		FROM `contact` INNER JOIN `user` ON `user`.`uid` = `contact`.`uid`
 		WHERE `contact`.`uid` = %d AND `contact`.`self` = 1 LIMIT 1",
 		intval($uid)
 	);
@@ -295,8 +292,28 @@ function notifier_run(&$argv, &$argc){
 			$followup = true;
 			$public_message = false; // not public
 			$conversant_str = dbesc($parent['contact-id']);
-		}
+			$recipients = array($parent['contact-id']);
-		else {
+
+			if ($parent['network'] == NETWORK_OSTATUS) {
+
+				// Check if the recipient isn't in your contact list
+				$r = q("SELECT `url` FROM `contact` WHERE `id` = %d", $parent['contact-id']);
+				if (count($r)) {
+					$url_recipients = array();
+
+					$thrparent = q("SELECT `author-link` FROM `item` WHERE `uri` = '%s'", dbesc($target_item["thr-parent"]));
+					if (count($thrparent) AND (normalise_link($r[0]["url"]) != normalise_link($thrparent[0]["author-link"]))) {
+						require_once("include/Scrape.php");
+						$probed_contact = probe_url($thrparent[0]["author-link"]);
+						if ($probed_contact["notify"] != "") {
+							logger('scrape data for slapper: '.print_r($probed_contact, true));
+							$url_recipients[$probed_contact["notify"]] = $probed_contact["notify"];
+						}
+					}
+				}
+				logger("url_recipients".print_r($url_recipients,true));
+			}
+		} else {
 			$followup = false;
 
 			// don't send deletions onward for other people's stuff
@@ -306,9 +323,9 @@ function notifier_run(&$argv, &$argc){
 				return;
 			}
 
-			if((strlen($parent['allow_cid'])) 
+			if((strlen($parent['allow_cid']))
-				|| (strlen($parent['allow_gid'])) 
+				|| (strlen($parent['allow_gid']))
-				|| (strlen($parent['deny_cid'])) 
+				|| (strlen($parent['deny_cid']))
 				|| (strlen($parent['deny_gid']))) {
 				$public_message = false; // private recipients, not public
 			}
@@ -410,8 +427,7 @@ function notifier_run(&$argv, &$argc){
 			'$content'      => xmlify($body),
 			'$parent_id'    => xmlify($item['parent-uri'])
 		));
-	}
+	} elseif($fsuggest) {
-	elseif($fsuggest) {
 		$public_message = false;  // suggestions are not public
 
 		$sugg_template = get_markup_template('atom_suggest.tpl');
@@ -430,9 +446,8 @@ function notifier_run(&$argv, &$argc){
 			intval($item['id'])
 		);
 
-	}
+	} elseif($relocate) {
-    elseif($relocate) {
+		$public_message = false;  // suggestions are not public
-        $public_message = false;  // suggestions are not public
 
 		$sugg_template = get_markup_template('atom_relocate.tpl');
 
@@ -453,24 +468,23 @@ function notifier_run(&$argv, &$argc){
 		}
 		unset($rp, $ext);
 
-        $atom .= replace_macros($sugg_template, array(
+		$atom .= replace_macros($sugg_template, array(
-            '$name' => xmlify($owner['name']),
+					'$name' => xmlify($owner['name']),
-            '$photo' => xmlify($photos[4]),
+					'$photo' => xmlify($photos[4]),
-            '$thumb' => xmlify($photos[5]),
+					'$thumb' => xmlify($photos[5]),
-            '$micro' => xmlify($photos[6]),
+					'$micro' => xmlify($photos[6]),
-            '$url' => xmlify($owner['url']),
+					'$url' => xmlify($owner['url']),
-            '$request' => xmlify($owner['request']),
+					'$request' => xmlify($owner['request']),
-            '$confirm' => xmlify($owner['confirm']),
+					'$confirm' => xmlify($owner['confirm']),
-            '$notify' => xmlify($owner['notify']),
+					'$notify' => xmlify($owner['notify']),
-            '$poll' => xmlify($owner['poll']),
+					'$poll' => xmlify($owner['poll']),
-            '$sitepubkey' => xmlify(get_config('system','site_pubkey')),
+					'$sitepubkey' => xmlify(get_config('system','site_pubkey')),
-            //'$pubkey' => xmlify($owner['pubkey']),
+					//'$pubkey' => xmlify($owner['pubkey']),
-            //'$prvkey' => xmlify($owner['prvkey']),
+					//'$prvkey' => xmlify($owner['prvkey']),
-		)); 
+			));
-        $recipients_relocate = q("SELECT * FROM contact WHERE uid = %d  AND self = 0 AND network = '%s'" , intval($uid), NETWORK_DFRN);
+		$recipients_relocate = q("SELECT * FROM contact WHERE uid = %d  AND self = 0 AND network = '%s'" , intval($uid), NETWORK_DFRN);
 		unset($photos);
-    }
+	} else {
-	else {
 		if($followup) {
 			foreach($items as $item) {  // there is only one item
 				if(! $item['parent'])
@@ -481,8 +495,7 @@ function notifier_run(&$argv, &$argc){
 					$atom .= atom_entry($item,'text',null,$owner,false);
 				}
 			}
-		}
+		} else {
-		else {
 			foreach($items as $item) {
 
 				if(! $item['parent'])
@@ -506,11 +519,10 @@ function notifier_run(&$argv, &$argc){
 
 				    if($item_id == $item['id'] || $item['id'] == $item['parent'])
 						$atom .= atom_entry($item,'text',null,$owner,true);
-				}
+				} else
-				else
 					$atom .= atom_entry($item,'text',null,$owner,true);
 
-				if(($top_level) && ($public_message) && ($item['author-link'] === $item['owner-link']) && (! $expire)) 
+				if(($top_level) && ($public_message) && ($item['author-link'] === $item['owner-link']) && (! $expire))
 					$slaps[] = atom_entry($item,'html',null,$owner,true);
 			}
 		}
@@ -526,8 +538,8 @@ function notifier_run(&$argv, &$argc){
 	$mail_disabled = ((function_exists('imap_open') && (! get_config('system','imap_disabled'))) ? 0 : 1);
 
 	if(! $mail_disabled) {
-		if((! strlen($target_item['allow_cid'])) && (! strlen($target_item['allow_gid'])) 
+		if((! strlen($target_item['allow_cid'])) && (! strlen($target_item['allow_gid']))
-			&& (! strlen($target_item['deny_cid'])) && (! strlen($target_item['deny_gid'])) 
+			&& (! strlen($target_item['deny_cid'])) && (! strlen($target_item['deny_gid']))
 			&& (intval($target_item['pubmail']))) {
 			$r = q("SELECT * FROM `contact` WHERE `uid` = %d AND `network` = '%s'",
 				intval($uid),
@@ -545,12 +557,12 @@ function notifier_run(&$argv, &$argc){
 	else
 		$recip_str = implode(', ', $recipients);
 
-    if ($relocate)
+	if ($relocate)
-        $r = $recipients_relocate;
+		$r = $recipients_relocate;
-    else
+	else
-        $r = q("SELECT * FROM `contact` WHERE `id` IN ( %s ) AND `blocked` = 0 AND `pending` = 0 ",
+		$r = q("SELECT * FROM `contact` WHERE `id` IN ( %s ) AND `blocked` = 0 AND `pending` = 0 ",
-            dbesc($recip_str)
+			dbesc($recip_str)
-        );
+		);
 
 
 	require_once('include/salmon.php');
@@ -946,7 +958,7 @@ function notifier_run(&$argv, &$argc){
 				}
 
 				if((! $mail) && (! $fsuggest) && (! $followup)) {
-					logger('notifier: delivery agent: ' . $rr['name'] . ' ' . $rr['id']); 
+					logger('notifier: delivery agent: ' . $rr['name'] . ' ' . $rr['id']);
 					proc_run('php','include/delivery.php',$cmd,$item_id,$rr['id']);
 					if($interval)
 						@time_sleep_until(microtime(true) + (float) $interval);
@@ -966,7 +978,7 @@ function notifier_run(&$argv, &$argc){
 					if ($h === '[internal]') {
 						// Set push flag for PuSH subscribers to this topic,
 						// they will be notified in queue.php
-						q("UPDATE `push_subscriber` SET `push` = 1 " . 
+						q("UPDATE `push_subscriber` SET `push` = 1 " .
 						  "WHERE `nickname` = '%s'", dbesc($owner['nickname']));
 					} else {
 
@@ -1001,6 +1013,6 @@ function notifier_run(&$argv, &$argc){
 
 
 if (array_search(__file__,get_included_files())===0){
-  notifier_run($argv,$argc);
+	notifier_run($_SERVER["argv"],$_SERVER["argc"]);
-  killme();
+	killme();
 }

include/oembed.php

@@ -11,7 +11,6 @@ function oembed_replacecb($matches){
 
 
 function oembed_fetch_url($embedurl, $no_rich_type = false){
-
 	$embedurl = trim($embedurl, "'");
 	$embedurl = trim($embedurl, '"');
 
@@ -78,6 +77,11 @@ function oembed_fetch_url($embedurl, $no_rich_type = false){
 	if (!is_object($j))
 		return false;
 
+	// Always embed the SSL version
+	if (isset($j->html))
+		$j->html = str_replace(array("http://www.youtube.com/", "http://player.vimeo.com/"),
+			array("https://www.youtube.com/", "https://player.vimeo.com/"), $j->html);
+
 	$j->embedurl = $embedurl;
 
 	// If fetching information doesn't work, then improve via internal functions
@@ -105,6 +109,8 @@ function oembed_fetch_url($embedurl, $no_rich_type = false){
 		}
 	}
 
+	call_hooks('oembed_fetch_url', $embedurl, $j);
+
 	return $j;
 }
 
@@ -156,6 +162,7 @@ function oembed_format_object($j){
 
 	// add link to source if not present in "rich" type
 	if ($j->type!='rich' || !strpos($j->html,$embedurl) ){
+		$ret .= "<h4>";
 		if (isset($j->title)) {
 			if (isset($j->provider_name))
 				$ret .= $j->provider_name.": ";
@@ -182,11 +189,12 @@ function oembed_format_object($j){
 		}
 		//if (isset($j->author_name)) $ret.=" by ".$j->author_name;
 		//if (isset($j->provider_name)) $ret.=" on ".$j->provider_name;
+		$ret .= "</h4>";
 	} else {
 		// add <a> for html2bbcode conversion
 		$ret .= "<a href='$embedurl' rel='oembed'>$embedurl</a>";
+		$ret.="<br style='clear:left'></span>";
 	}
-	$ret.="<br style='clear:left'></span>";
 	return  mb_convert_encoding($ret, 'HTML-ENTITIES', mb_detect_encoding($ret));
 }
 

include/onepoll.php

@@ -15,7 +15,7 @@ function onepoll_run(&$argv, &$argc){
 	if(is_null($a)) {
 		$a = new App;
 	}
-  
+
 	if(is_null($db)) {
 	    @include(".htconfig.php");
     	require_once("include/dba.php");
@@ -57,28 +57,30 @@ function onepoll_run(&$argv, &$argc){
 		return;
 	}
 
-	// Test
 	$lockpath = get_lockpath();
 	if ($lockpath != '') {
 		$pidfile = new pidfile($lockpath, 'onepoll'.$contact_id);
-		if($pidfile->is_already_running()) {
+		if ($pidfile->is_already_running()) {
 			logger("onepoll: Already running for contact ".$contact_id);
+			if ($pidfile->running_time() > 9*60) {
+				$pidfile->kill();
+				logger("killed stale process");
+			}
 			exit;
 		}
 	}
 
-
 	$d = datetime_convert();
 
 	// Only poll from those with suitable relationships,
-	// and which have a polling address and ignore Diaspora since 
+	// and which have a polling address and ignore Diaspora since
 	// we are unable to match those posts with a Diaspora GUID and prevent duplicates.
 
-	$contacts = q("SELECT `contact`.* FROM `contact` 
+	$contacts = q("SELECT `contact`.* FROM `contact`
 		WHERE ( `rel` = %d OR `rel` = %d ) AND `poll` != ''
 		AND NOT `network` IN ( '%s', '%s', '%s' )
 		AND `contact`.`id` = %d
-		AND `self` = 0 AND `contact`.`blocked` = 0 AND `contact`.`readonly` = 0 
+		AND `self` = 0 AND `contact`.`blocked` = 0 AND `contact`.`readonly` = 0
 		AND `contact`.`archive` = 0 LIMIT 1",
 		intval(CONTACT_IS_SHARING),
 		intval(CONTACT_IS_FRIEND),
@@ -174,7 +176,7 @@ function onepoll_run(&$argv, &$argc){
 			return;
 		}
 
-		if(! strstr($handshake_xml,'<?xml')) {
+		if(! strstr($handshake_xml,'<')) {
 			logger('poller: response from ' . $url . ' did not contain XML.');
 
 			mark_for_death($contact);
@@ -282,13 +284,13 @@ function onepoll_run(&$argv, &$argc){
 	}
 	elseif($contact['network'] === NETWORK_MAIL || $contact['network'] === NETWORK_MAIL2) {
 
-		logger("onepoll: mail: Fetching", LOGGER_DEBUG);
+		logger("Mail: Fetching", LOGGER_DEBUG);
 
 		$mail_disabled = ((function_exists('imap_open') && (! get_config('system','imap_disabled'))) ? 0 : 1);
 		if($mail_disabled)
 			return;
 
-		logger("onepoll: Mail: Enabled", LOGGER_DEBUG);
+		logger("Mail: Enabled", LOGGER_DEBUG);
 
 		$mbox = null;
 		$x = q("SELECT `prvkey` FROM `user` WHERE `uid` = %d LIMIT 1",
@@ -310,7 +312,9 @@ function onepoll_run(&$argv, &$argc){
 					intval($mailconf[0]['id']),
 					intval($importer_uid)
 				);
-			}
+				logger("Mail: Connected to " . $mailconf[0]['user']);
+			} else
+				logger("Mail: Connection error ".$mailconf[0]['user']." ".print_r(imap_errors()));
 		}
 		if($mbox) {
 
@@ -521,7 +525,10 @@ function onepoll_run(&$argv, &$argc){
 						}
 					}
 				}
-			}
+			} else
+				logger("Mail: no mails for ".$mailconf[0]['user']);
+
+			logger("Mail: closing connection for ".$mailconf[0]['user']);
 			imap_close($mbox);
 		}
 	}
@@ -535,7 +542,7 @@ function onepoll_run(&$argv, &$argc){
 
 	if($xml) {
 		logger('poller: received xml : ' . $xml, LOGGER_DATA);
-		if((! strstr($xml,'<?xml')) && (! strstr($xml,'<rss'))) {
+		if(! strstr($xml,'<')) {
 			logger('poller: post_handshake: response from ' . $url . ' did not contain XML.');
 			$r = q("UPDATE `contact` SET `last-update` = '%s' WHERE `id` = %d",
 				dbesc(datetime_convert()),
@@ -584,8 +591,8 @@ function onepoll_run(&$argv, &$argc){
 
 	// load current friends if possible.
 
-	if($contact['poco']) {	
+	if($contact['poco']) {
-		$r = q("SELECT count(*) as total from glink 
+		$r = q("SELECT count(*) as total from glink
 			where `cid` = %d and updated > UTC_TIMESTAMP() - INTERVAL 1 DAY",
 			intval($contact['id'])
 		);
@@ -600,6 +607,6 @@ function onepoll_run(&$argv, &$argc){
 }
 
 if (array_search(__file__,get_included_files())===0){
-  onepoll_run($argv,$argc);
+  onepoll_run($_SERVER["argv"],$_SERVER["argc"]);
   killme();
 }

include/ostatus_conversation.php

@@ -20,12 +20,12 @@ function check_conversations() {
         if($last) {
                 $next = $last + ($poll_interval * 60);
                 if($next > time()) {
-                        logger('complete_conversation: poll interval not reached');
+                        logger('poll interval not reached');
                         return;
                 }
         }
 
-        logger('complete_conversation: cron_start');
+        logger('cron_start');
 
         $start = date("Y-m-d H:i:s", time() - ($poll_timeframe * 60));
         $conversations = q("SELECT * FROM `term` WHERE `type` = 7 AND `term` > '%s'",
@@ -36,7 +36,7 @@ function check_conversations() {
                 complete_conversation($id, $url);
         }
 
-        logger('complete_conversation: cron_end');
+        logger(' cron_end');
 
         set_config('system','ostatus_last_poll', time());
 }
@@ -52,9 +52,7 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio
 
 	$a->last_ostatus_conversation_url = $conversation_url;
 
-	//logger('complete_conversation: completing conversation url '.$conversation_url.' for id '.$itemid);
+	$messages = q("SELECT `uid`, `parent`, `created`, `received`, `guid` FROM `item` WHERE `id` = %d LIMIT 1", intval($itemid));
-
-	$messages = q("SELECT `uid`, `parent`, `created` FROM `item` WHERE `id` = %d LIMIT 1", intval($itemid));
 	if (!$messages)
 		return;
 	$message = $messages[0];
@@ -64,8 +62,9 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio
 		intval($message["uid"]), intval($itemid), intval(TERM_OBJ_POST), intval(TERM_CONVERSATION));
 
 	if (!$conversation) {
-		$r = q("INSERT INTO `term` (`uid`, `oid`, `otype`, `type`, `term`, `url`) VALUES (%d, %d, %d, %d, '%s', '%s')",
+		$r = q("INSERT INTO `term` (`uid`, `oid`, `otype`, `type`, `term`, `url`, `created`, `received`, `guid`) VALUES (%d, %d, %d, %d, '%s', '%s', '%s', '%s', '%s')",
-			intval($message["uid"]), intval($itemid), intval(TERM_OBJ_POST), intval(TERM_CONVERSATION), dbesc($message["created"]), dbesc($conversation_url));
+			intval($message["uid"]), intval($itemid), intval(TERM_OBJ_POST), intval(TERM_CONVERSATION),
+			dbesc($message["created"]), dbesc($conversation_url), dbesc($message["created"]), dbesc($message["received"]), dbesc($message["guid"]));
 		logger('complete_conversation: Storing conversation url '.$conversation_url.' for id '.$itemid);
 	}
 
@@ -90,8 +89,6 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio
 
 	do {
 		$conv_as = fetch_url($conv."?page=".$pageno);
-		//$conv_as = fetch_url($conv."?page=".$pageno, false, 0, 10);
-		//$conv_as = file_get_contents($conv."?page=".$pageno);
 		$conv_as = str_replace(',"statusnet:notice_info":', ',"statusnet_notice_info":', $conv_as);
 		$conv_as = json_decode($conv_as);
 
@@ -110,7 +107,7 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio
 	$items = array_reverse($items);
 
 	foreach ($items as $single_conv) {
-		// identi.ca just changed the format of the activity streams. This is a quick fix.
+		// status.net changed the format of the activity streams. This is a quick fix.
 		if (@is_string($single_conv->object->id))
 			$single_conv->id = $single_conv->object->id;
 
@@ -127,7 +124,7 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio
 				intval($message["uid"]), dbesc($first_id));
 			if ($new_parents) {
 				$parent = $new_parents[0];
-				logger('complete_conversation: adopting new parent '.$parent["id"].' for '.$itemid);
+				logger('adopting new parent '.$parent["id"].' for '.$itemid);
 			} else {
 				$parent["id"] = 0;
 				$parent["uri"] = $first_id;
@@ -144,6 +141,8 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio
 		if ($message_exists) {
 			if ($parent["id"] != 0) {
 				$existing_message = $message_exists[0];
+
+				// This is partly bad, since the entry in the thread table isn't updated
 				$r = q("UPDATE `item` SET `parent` = %d, `parent-uri` = '%s', `thr-parent` = '%s' WHERE `id` = %d",
 					intval($parent["id"]),
 					dbesc($parent["uri"]),
@@ -153,12 +152,23 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio
 			continue;
 		}
 
+		$contact = q("SELECT `id` FROM `contact` WHERE `uid` = %d AND `nurl` = '%s' AND `network` != '%s'",
+				$message["uid"], normalise_link($single_conv->actor->id), NETWORK_STATUSNET);
+
+		if (count($contact)) {
+			logger("Found contact for url ".$single_conv->actor->id, LOGGER_DEBUG);
+			$contact_id = $contact[0]["id"];
+		} else {
+			logger("No contact found for url ".$single_conv->actor->id, LOGGER_DEBUG);
+			$contact_id = $parent["contact-id"];
+		}
+
 		$arr = array();
 		$arr["network"] = NETWORK_OSTATUS;
 		$arr["uri"] = $single_conv->id;
 		$arr["plink"] = $single_conv->id;
 		$arr["uid"] = $message["uid"];
-		$arr["contact-id"] = $parent["contact-id"]; // To-Do
+		$arr["contact-id"] = $contact_id;
 		if ($parent["id"] != 0)
 			$arr["parent"] = $parent["id"];
 		$arr["parent-uri"] = $parent["uri"];
@@ -201,20 +211,13 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio
 
 		// If the newly created item is the top item then change the parent settings of the thread
 		if ($newitem AND ($arr["uri"] == $first_id)) {
-			logger('complete_conversation: setting new parent to id '.$newitem);
+			logger('setting new parent to id '.$newitem);
 			$new_parents = q("SELECT `id`, `uri`, `contact-id`, `type`, `verb`, `visible` FROM `item` WHERE `uid` = %d AND `id` = %d LIMIT 1",
 				intval($message["uid"]), intval($newitem));
 			if ($new_parents) {
 				$parent = $new_parents[0];
-				logger('complete_conversation: done changing parents to parent '.$newitem);
+				logger('done changing parents to parent '.$newitem);
 			}
-
-			/*logger('complete_conversation: changing parents to parent '.$newitem.' old parent: '.$parent["id"].' new uri: '.$arr["uri"]);
-			$r = q("UPDATE `item` SET `parent` = %d, `parent-uri` = '%s' WHERE `parent` = %d",
-				intval($newitem),
-				dbesc($arr["uri"]),
-				intval($parent["id"]));
-			logger('complete_conversation: done changing parents to parent '.$newitem.' '.print_r($r, true));*/
 		}
 	}
 }

include/poller.php

@@ -1,4 +1,14 @@
 <?php
+if (!file_exists("boot.php") AND (sizeof($_SERVER["argv"]) != 0)) {
+	$directory = dirname($_SERVER["argv"][0]);
+
+	if (substr($directory, 0, 1) != "/")
+		$directory = $_SERVER["PWD"]."/".$directory;
+
+	$directory = realpath($directory."/..");
+
+	chdir($directory);
+}
 
 require_once("boot.php");
 
@@ -74,8 +84,8 @@ function poller_run(&$argv, &$argc){
 
 	// expire any expired accounts
 
-	q("UPDATE user SET `account_expired` = 1 where `account_expired` = 0 
+	q("UPDATE user SET `account_expired` = 1 where `account_expired` = 0
-		AND `account_expires_on` != '0000-00-00 00:00:00' 
+		AND `account_expires_on` != '0000-00-00 00:00:00'
 		AND `account_expires_on` < UTC_TIMESTAMP() ");
 
 	// delete user and contact records for recently removed accounts
@@ -149,7 +159,6 @@ function poller_run(&$argv, &$argc){
 
 	$manual_id  = 0;
 	$generation = 0;
-	$hub_update = false;
 	$force      = false;
 	$restart    = false;
 
@@ -169,7 +178,7 @@ function poller_run(&$argv, &$argc){
 	}
 
 	$interval = intval(get_config('system','poll_interval'));
-	if(! $interval) 
+	if(! $interval)
 		$interval = ((get_config('system','delivery_interval') === false) ? 3 : intval(get_config('system','delivery_interval')));
 
 	$sql_extra = (($manual_id) ? " AND `id` = $manual_id " : "");
@@ -182,26 +191,27 @@ function poller_run(&$argv, &$argc){
 		proc_run('php','include/cronhooks.php');
 
 	// Only poll from those with suitable relationships,
-	// and which have a polling address and ignore Diaspora since 
+	// and which have a polling address and ignore Diaspora since
 	// we are unable to match those posts with a Diaspora GUID and prevent duplicates.
 
-	$abandon_sql = (($abandon_days) 
+	$abandon_sql = (($abandon_days)
-		? sprintf(" AND `user`.`login_date` > UTC_TIMESTAMP() - INTERVAL %d DAY ", intval($abandon_days)) 
+		? sprintf(" AND `user`.`login_date` > UTC_TIMESTAMP() - INTERVAL %d DAY ", intval($abandon_days))
-		: '' 
+		: ''
 	);
 
-	$contacts = q("SELECT `contact`.`id` FROM `contact` INNER JOIN `user` ON `user`.`uid` = `contact`.`uid` 
+	$contacts = q("SELECT `contact`.`id` FROM `contact` INNER JOIN `user` ON `user`.`uid` = `contact`.`uid`
-		WHERE ( `rel` = %d OR `rel` = %d ) AND `poll` != ''
+		WHERE `rel` IN (%d, %d) AND `poll` != '' AND `network` IN ('%s', '%s', '%s', '%s', '%s', '%s')
-		AND NOT `network` IN ( '%s', '%s', '%s' )
+		$sql_extra
-		$sql_extra 
+		AND NOT `self` AND NOT `contact`.`blocked` AND NOT `contact`.`readonly` AND NOT `contact`.`archive`
-		AND `self` = 0 AND `contact`.`blocked` = 0 AND `contact`.`readonly` = 0 
+		AND NOT `user`.`account_expired` AND NOT `user`.`account_removed` $abandon_sql ORDER BY RAND()",
-		AND `contact`.`archive` = 0 
-		AND `user`.`account_expired` = 0 AND `user`.`account_removed` = 0 $abandon_sql ORDER BY RAND()",
 		intval(CONTACT_IS_SHARING),
 		intval(CONTACT_IS_FRIEND),
-		dbesc(NETWORK_DIASPORA),
+		dbesc(NETWORK_DFRN),
-		dbesc(NETWORK_FACEBOOK),
+		dbesc(NETWORK_ZOT),
-		dbesc(NETWORK_PUMPIO)
+		dbesc(NETWORK_OSTATUS),
+		dbesc(NETWORK_FEED),
+		dbesc(NETWORK_MAIL),
+		dbesc(NETWORK_MAIL2)
 	);
 
 	if(! count($contacts)) {
@@ -224,45 +234,34 @@ function poller_run(&$argv, &$argc){
 			if($manual_id)
 				$contact['last-update'] = '0000-00-00 00:00:00';
 
-			if($contact['network'] === NETWORK_DFRN)
+			if(in_array($contact['network'], array(NETWORK_DFRN, NETWORK_ZOT, NETWORK_OSTATUS)))
 				$contact['priority'] = 2;
 
-			if(!get_config('system','ostatus_use_priority') and ($contact['network'] === NETWORK_OSTATUS))
+			if($contact['subhub'] AND in_array($contact['network'], array(NETWORK_DFRN, NETWORK_ZOT, NETWORK_OSTATUS))) {
-				$contact['priority'] = 2;
+				// We should be getting everything via a hub. But just to be sure, let's check once a day.
+				// (You can make this more or less frequent if desired by setting 'pushpoll_frequency' appropriately)
+				// This also lets us update our subscription to the hub, and add or replace hubs in case it
+				// changed. We will only update hubs once a day, regardless of 'pushpoll_frequency'.
 
-			if($contact['priority'] || $contact['subhub']) {
+				$poll_interval = get_config('system','pushpoll_frequency');
+				$contact['priority'] = (($poll_interval !== false) ? intval($poll_interval) : 3);
+			}
+
+			if($contact['priority'] AND !$force) {
 
-				$hub_update = true;
 				$update     = false;
 
 				$t = $contact['last-update'];
 
-				// We should be getting everything via a hub. But just to be sure, let's check once a day.
-				// (You can make this more or less frequent if desired by setting 'pushpoll_frequency' appropriately)
-				// This also lets us update our subscription to the hub, and add or replace hubs in case it
-				// changed. We will only update hubs once a day, regardless of 'pushpoll_frequency'. 
-
-
-				if($contact['subhub']) {
-					$poll_interval = get_config('system','pushpoll_frequency');
-					$contact['priority'] = (($poll_interval !== false) ? intval($poll_interval) : 3);
-					$hub_update = false;
-	
-					if((datetime_convert('UTC','UTC', 'now') > datetime_convert('UTC','UTC', $t . " + 1 day")) || $force)
-							$hub_update = true;
-				}
-				else
-					$hub_update = false;
-
 				/**
 				 * Based on $contact['priority'], should we poll this site now? Or later?
-				 */			
+				 */
 
 				switch ($contact['priority']) {
 					case 5:
 						if(datetime_convert('UTC','UTC', 'now') > datetime_convert('UTC','UTC', $t . " + 1 month"))
 							$update = true;
-						break;					
+						break;
 					case 4:
 						if(datetime_convert('UTC','UTC', 'now') > datetime_convert('UTC','UTC', $t . " + 1 week"))
 							$update = true;
@@ -281,20 +280,25 @@ function poller_run(&$argv, &$argc){
 							$update = true;
 						break;
 				}
-				if((! $update) && (! $force))
+				if(!$update)
 					continue;
 			}
 
+			logger("Polling ".$contact["network"]." ".$contact["id"]." ".$contact["nick"]." ".$contact["name"]);
+
 			proc_run('php','include/onepoll.php',$contact['id']);
+
 			if($interval)
 				@time_sleep_until(microtime(true) + (float) $interval);
 		}
 	}
 
+	logger('poller: end');
+
 	return;
 }
 
 if (array_search(__file__,get_included_files())===0){
-  poller_run($argv,$argc);
+  poller_run($_SERVER["argv"],$_SERVER["argc"]);
   killme();
 }

include/queue.php

@@ -265,6 +265,6 @@ function queue_run(&$argv, &$argc){
 }
 
 if (array_search(__file__,get_included_files())===0){
-  queue_run($argv,$argc);
+  queue_run($_SERVER["argv"],$_SERVER["argc"]);
   killme();
 }

include/shadowupdate.php

@@ -0,0 +1,22 @@
+<?php
+require_once("boot.php");
+require_once("include/threads.php");
+
+global $a, $db;
+
+if(is_null($a))
+	$a = new App;
+
+if(is_null($db)) {
+	@include(".htconfig.php");
+	require_once("include/dba.php");
+	$db = new dba($db_host, $db_user, $db_pass, $db_data);
+	unset($db_host, $db_user, $db_pass, $db_data);
+}
+
+load_config('config');
+load_config('system');
+
+update_shadow_copy();
+killme();
+?>

include/socgraph.php

@@ -12,14 +12,17 @@ require_once('include/datetime.php');
  *
  * Once the global contact is stored add (if necessary) the contact linkage which associates
  * the given uid, cid to the global contact entry. There can be many uid/cid combinations
- * pointing to the same global contact id. 
+ * pointing to the same global contact id.
  *
  */
- 
+
 
 
 
 function poco_load($cid,$uid = 0,$zcid = 0,$url = null) {
+
+	require_once("include/html2bbcode.php");
+
 	$a = get_app();
 
 	if($cid) {
@@ -39,7 +42,7 @@ function poco_load($cid,$uid = 0,$zcid = 0,$url = null) {
 	if(! $url)
 		return;
 
-	$url = $url . (($uid) ? '/@me/@all?fields=displayName,urls,photos' : '?fields=displayName,urls,photos') ;
+	$url = $url . (($uid) ? '/@me/@all?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation' : '?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation') ;
 
 	logger('poco_load: ' . $url, LOGGER_DEBUG);
 
@@ -67,6 +70,13 @@ function poco_load($cid,$uid = 0,$zcid = 0,$url = null) {
 		$profile_photo = '';
 		$connect_url = '';
 		$name = '';
+		$network = '';
+		$updated = '0000-00-00 00:00:00';
+		$location = '';
+		$about = '';
+		$keywords = '';
+		$gender = '';
+		$generation = 0;
 
 		$name = $entry->displayName;
 
@@ -82,7 +92,7 @@ function poco_load($cid,$uid = 0,$zcid = 0,$url = null) {
 				}
 			}
 		}
-		if(isset($entry->photos)) { 
+		if(isset($entry->photos)) {
 			foreach($entry->photos as $photo) {
 				if($photo->type == 'profile') {
 					$profile_photo = $photo->value;
@@ -91,74 +101,48 @@ function poco_load($cid,$uid = 0,$zcid = 0,$url = null) {
 			}
 		}
 
-		if((! $name) || (! $profile_url) || (! $profile_photo))
+		if(isset($entry->updated))
-			continue;
+			$updated = date("Y-m-d H:i:s", strtotime($entry->updated));
 
-		$x = q("select * from `gcontact` where `nurl` = '%s' limit 1",
+		if(isset($entry->network))
-			dbesc(normalise_link($profile_url))
+			$network = $entry->network;
-		);
 
-		if(count($x)) {
+		if(isset($entry->currentLocation))
-			$gcid = $x[0]['id'];
+			$location = $entry->currentLocation;
 
-			if($x[0]['name'] != $name || $x[0]['photo'] != $profile_photo) {
+		if(isset($entry->aboutMe))
-				q("update gcontact set `name` = '%s', `photo` = '%s', `connect` = '%s', `url` = '%s'
+			$about = html2bbcode($entry->aboutMe);
-					where `nurl` = '%s'",
+
-					dbesc($name),
+		if(isset($entry->gender))
-					dbesc($profile_photo),
+			$gender = $entry->gender;
-					dbesc($connect_url),
+
-					dbesc($profile_url),
+		if(isset($entry->generation) AND ($entry->generation > 0))
-					dbesc(normalise_link($profile_url))
+			$generation = ++$entry->generation;
-				);
+
-			}
+		if(isset($entry->tags))
-		}
+			foreach($entry->tags as $tag)
-		else {
+				$keywords = implode(", ", $tag);
-			q("insert into `gcontact` (`name`,`url`,`nurl`,`photo`,`connect`)
+
-				values ( '%s', '%s', '%s', '%s','%s') ",
+		// If you query a Friendica server for its profiles, the network has to be Friendica
-				dbesc($name),
+		if ($uid == 0)
-				dbesc($profile_url),
+			$network = NETWORK_DFRN;
+
+		poco_check($profile_url, $name, $network, $profile_photo, $about, $location, $gender, $keywords, $connect_url, $updated, $generation, $cid, $uid, $zcid);
+
+		// Update the Friendica contacts. Diaspora is doing it via a message. (See include/diaspora.php)
+		if (($location != "") OR ($about != "") OR ($keywords != "") OR ($gender != ""))
+			q("UPDATE `contact` SET `location` = '%s', `about` = '%s', `keywords` = '%s', `gender` = '%s'
+				WHERE `nurl` = '%s' AND NOT `self` AND `network` = '%s'",
+				dbesc($location),
+				dbesc($about),
+				dbesc($keywords),
+				dbesc($gender),
 				dbesc(normalise_link($profile_url)),
-				dbesc($profile_photo),
+				dbesc(NETWORK_DFRN));
-				dbesc($connect_url)
-			);
-			$x = q("select * from `gcontact` where `nurl` = '%s' limit 1",
-				dbesc(normalise_link($profile_url))
-			);
-			if(count($x))
-				$gcid = $x[0]['id'];
-		}
-		if(! $gcid)
-			return;
-
-		$r = q("select * from glink where `cid` = %d and `uid` = %d and `gcid` = %d and `zcid` = %d limit 1",
-			intval($cid),
-			intval($uid),
-			intval($gcid),
-			intval($zcid)
-		);
-		if(! count($r)) {
-			q("insert into glink ( `cid`,`uid`,`gcid`,`zcid`, `updated`) values (%d,%d,%d,%d, '%s') ",
-				intval($cid),
-				intval($uid),
-				intval($gcid),
-				intval($zcid),
-				dbesc(datetime_convert())
-			);
-		}
-		else {
-			q("update glink set updated = '%s' where `cid` = %d and `uid` = %d and `gcid` = %d and zcid = %d",
-				dbesc(datetime_convert()),
-				intval($cid),
-				intval($uid),
-				intval($gcid),
-				intval($zcid)
-			);
-		}
-
 	}
 	logger("poco_load: loaded $total entries",LOGGER_DEBUG);
 
-	q("delete from glink where `cid` = %d and `uid` = %d and `zcid` = %d and `updated` < UTC_TIMESTAMP - INTERVAL 2 DAY",
+	q("DELETE FROM `glink` WHERE `cid` = %d AND `uid` = %d AND `zcid` = %d AND `updated` < UTC_TIMESTAMP - INTERVAL 2 DAY",
 		intval($cid),
 		intval($uid),
 		intval($zcid)
@@ -166,6 +150,256 @@ function poco_load($cid,$uid = 0,$zcid = 0,$url = null) {
 
 }
 
+function poco_check($profile_url, $name, $network, $profile_photo, $about, $location, $gender, $keywords, $connect_url, $updated, $generation, $cid = 0, $uid = 0, $zcid = 0) {
+
+	$a = get_app();
+
+	// Generation:
+	//  0: No definition
+	//  1: Profiles on this server
+	//  2: Contacts of profiles on this server
+	//  3: Contacts of contacts of profiles on this server
+	//  4: ...
+
+	$gcid = "";
+
+	if ($profile_url == "")
+		return $gcid;
+
+	// Don't store the statusnet connector as network
+	// We can't simply set this to NETWORK_OSTATUS since the connector could have fetched posts from friendica as well
+	if ($network == NETWORK_STATUSNET)
+		$network = "";
+
+	// The global contacts should contain the original picture, not the cached one
+	if (($generation != 1) AND stristr(normalise_link($profile_photo), normalise_link($a->get_baseurl()."/photo/")))
+		$profile_photo = "";
+
+	$r = q("SELECT `network` FROM `contact` WHERE `nurl` = '%s' AND `network` != '' AND `network` != '%s' LIMIT 1",
+		dbesc(normalise_link($profile_url)), dbesc(NETWORK_STATUSNET)
+	);
+	if(count($r))
+		$network = $r[0]["network"];
+
+	if (($network == "") OR ($network == NETWORK_OSTATUS)) {
+		$r = q("SELECT `network`, `url` FROM `contact` WHERE `alias` IN ('%s', '%s') AND `network` != '' AND `network` != '%s' LIMIT 1",
+			dbesc($profile_url), dbesc(normalise_link($profile_url)), dbesc(NETWORK_STATUSNET)
+		);
+		if(count($r)) {
+			$network = $r[0]["network"];
+			$profile_url = $r[0]["url"];
+		}
+	}
+
+	$x = q("SELECT * FROM `gcontact` WHERE `nurl` = '%s' LIMIT 1",
+		dbesc(normalise_link($profile_url))
+	);
+	if(count($x) AND ($network == "") AND ($x[0]["network"] != NETWORK_STATUSNET))
+		$network = $x[0]["network"];
+
+	if (($network == "") OR ($name == "") OR ($profile_photo == "")) {
+		require_once("include/Scrape.php");
+
+		$data = probe_url($profile_url);
+		$network = $data["network"];
+		$name = $data["name"];
+		$profile_url = $data["url"];
+		$profile_photo = $data["photo"];
+	}
+
+	if (count($x) AND ($x[0]["network"] == "") AND ($network != "")) {
+		q("UPDATE `gcontact` SET `network` = '%s' WHERE `nurl` = '%s'",
+			dbesc($network),
+			dbesc(normalise_link($profile_url))
+		);
+	}
+
+	if (($name == "") OR ($profile_photo == ""))
+		return $gcid;
+
+	if (!in_array($network, array(NETWORK_DFRN, NETWORK_OSTATUS, NETWORK_DIASPORA)))
+		return $gcid;
+
+	logger("profile-check generation: ".$generation." Network: ".$network." URL: ".$profile_url." name: ".$name." avatar: ".$profile_photo, LOGGER_DEBUG);
+
+	if(count($x)) {
+		$gcid = $x[0]['id'];
+
+		if (($location == "") AND ($x[0]['location'] != ""))
+			$location = $x[0]['location'];
+
+		if (($about == "") AND ($x[0]['about'] != ""))
+			$about = $x[0]['about'];
+
+		if (($gender == "") AND ($x[0]['gender'] != ""))
+			$gender = $x[0]['gender'];
+
+		if (($keywords == "") AND ($x[0]['keywords'] != ""))
+			$keywords = $x[0]['keywords'];
+
+		if (($generation == 0) AND ($x[0]['generation'] > 0))
+			$generation = $x[0]['generation'];
+
+		if($x[0]['name'] != $name || $x[0]['photo'] != $profile_photo || $x[0]['updated'] < $updated) {
+			q("UPDATE `gcontact` SET `name` = '%s', `network` = '%s', `photo` = '%s', `connect` = '%s', `url` = '%s',
+				`updated` = '%s', `location` = '%s', `about` = '%s', `keywords` = '%s', `gender` = '%s', `generation` = %d
+				WHERE (`generation` >= %d OR `generation` = 0) AND `nurl` = '%s'",
+				dbesc($name),
+				dbesc($network),
+				dbesc($profile_photo),
+				dbesc($connect_url),
+				dbesc($profile_url),
+				dbesc($updated),
+				dbesc($location),
+				dbesc($about),
+				dbesc($keywords),
+				dbesc($gender),
+				intval($generation),
+				intval($generation),
+				dbesc(normalise_link($profile_url))
+			);
+		}
+	} else {
+		q("INSERT INTO `gcontact` (`name`,`network`, `url`,`nurl`,`photo`,`connect`, `updated`, `location`, `about`, `keywords`, `gender`, `generation`)
+			VALUES ('%s', '%s', '%s', '%s', '%s','%s', '%s', '%s', '%s', '%s', '%s', %d)",
+			dbesc($name),
+			dbesc($network),
+			dbesc($profile_url),
+			dbesc(normalise_link($profile_url)),
+			dbesc($profile_photo),
+			dbesc($connect_url),
+			dbesc($updated),
+			dbesc($location),
+			dbesc($about),
+			dbesc($keywords),
+			dbesc($gender),
+			intval($generation)
+		);
+		$x = q("SELECT * FROM `gcontact` WHERE `nurl` = '%s' LIMIT 1",
+			dbesc(normalise_link($profile_url))
+		);
+		if(count($x))
+			$gcid = $x[0]['id'];
+	}
+
+	if(! $gcid)
+		return $gcid;
+
+	$r = q("SELECT * FROM `glink` WHERE `cid` = %d AND `uid` = %d AND `gcid` = %d AND `zcid` = %d LIMIT 1",
+		intval($cid),
+		intval($uid),
+		intval($gcid),
+		intval($zcid)
+	);
+	if(! count($r)) {
+		q("INSERT INTO `glink` (`cid`,`uid`,`gcid`,`zcid`, `updated`) VALUES (%d,%d,%d,%d, '%s') ",
+			intval($cid),
+			intval($uid),
+			intval($gcid),
+			intval($zcid),
+			dbesc(datetime_convert())
+		);
+	} else {
+		q("UPDATE `glink` SET `updated` = '%s' WHERE `cid` = %d AND `uid` = %d AND `gcid` = %d AND `zcid` = %d",
+			dbesc(datetime_convert()),
+			intval($cid),
+			intval($uid),
+			intval($gcid),
+			intval($zcid)
+		);
+	}
+
+	// For unknown reasons there are sometimes duplicates
+	q("DELETE FROM `gcontact` WHERE `nurl` = '%s' AND `id` != %d AND
+		NOT EXISTS (SELECT `gcid` FROM `glink` WHERE `gcid` = `gcontact`.`id`)",
+		dbesc(normalise_link($profile_url)),
+		intval($gcid)
+	);
+
+	return $gcid;
+}
+
+function poco_contact_from_body($body, $created, $cid, $uid) {
+	preg_replace_callback("/\[share(.*?)\].*?\[\/share\]/ism",
+		function ($match) use ($created, $cid, $uid){
+			return(sub_poco_from_share($match, $created, $cid, $uid));
+		}, $body);
+}
+
+function sub_poco_from_share($share, $created, $cid, $uid) {
+        $profile = "";
+        preg_match("/profile='(.*?)'/ism", $share[1], $matches);
+        if ($matches[1] != "")
+                $profile = $matches[1];
+
+        preg_match('/profile="(.*?)"/ism', $share[1], $matches);
+        if ($matches[1] != "")
+                $profile = $matches[1];
+
+	if ($profile == "")
+		return;
+
+	logger("prepare poco_check for profile ".$profile, LOGGER_DEBUG);
+        poco_check($profile, "", "", "", "", "", "", "", "", $created, 3, $cid, $uid);
+}
+
+function poco_store($item) {
+
+	// Isn't it public?
+	if ($item['private'])
+		return;
+
+	// Or is it from a network where we don't store the global contacts?
+	if (!in_array($item["network"], array(NETWORK_DFRN, NETWORK_DIASPORA, NETWORK_OSTATUS, NETWORK_STATUSNET, "")))
+		return;
+
+	// Is it a global copy?
+	$store_gcontact = ($item["uid"] == 0);
+
+	// Is it a comment on a global copy?
+	if (!$store_gcontact AND ($item["uri"] != $item["parent-uri"])) {
+		$q = q("SELECT `id` FROM `item` WHERE `uri`='%s' AND `uid` = 0", $item["parent-uri"]);
+		$store_gcontact = count($q);
+	}
+
+	if (!$store_gcontact)
+		return;
+
+	// "3" means: We don't know this contact directly (Maybe a reshared item)
+	$generation = 3;
+	$network = "";
+	$profile_url = $item["author-link"];
+
+	// Is it a user from our server?
+	$q = q("SELECT `id` FROM `contact` WHERE `self` AND `nurl` = '%s' LIMIT 1",
+		dbesc(normalise_link($item["author-link"])));
+	if (count($q)) {
+		logger("Our user (generation 1): ".$item["author-link"], LOGGER_DEBUG);
+		$generation = 1;
+		$network = NETWORK_DFRN;
+	} else { // Is it a contact from a user on our server?
+		$q = q("SELECT `network`, `url` FROM `contact` WHERE `uid` != 0 AND `network` != ''
+			AND (`nurl` = '%s' OR `alias` IN ('%s', '%s')) AND `network` != '%s' LIMIT 1",
+			dbesc(normalise_link($item["author-link"])),
+			dbesc(normalise_link($item["author-link"])),
+			dbesc($item["author-link"]),
+			dbesc(NETWORK_STATUSNET));
+		if (count($q)) {
+			$generation = 2;
+			$network = $q[0]["network"];
+			$profile_url = $q[0]["url"];
+			logger("Known contact (generation 2): ".$profile_url, LOGGER_DEBUG);
+		}
+	}
+
+	if ($generation == 3)
+		logger("Unknown contact (generation 3): ".$item["author-link"], LOGGER_DEBUG);
+
+	poco_check($profile_url, $item["author-name"], $network, $item["author-avatar"], "", "", "", "", "", $item["received"], $generation, $item["contact-id"], $item["uid"]);
+
+	// Maybe its a body with a shared item? Then extract a global contact from it.
+	poco_contact_from_body($item["body"], $item["received"], $item["contact-id"], $item["uid"]);
+}
 
 function count_common_friends($uid,$cid) {
 
@@ -192,9 +426,9 @@ function common_friends($uid,$cid,$start = 0,$limit=9999,$shuffle = false) {
 	if($shuffle)
 		$sql_extra = " order by rand() ";
 	else
-		$sql_extra = " order by `gcontact`.`name` asc "; 
+		$sql_extra = " order by `gcontact`.`name` asc ";
 
-	$r = q("SELECT `gcontact`.* 
+	$r = q("SELECT `gcontact`.*
 		FROM `glink` INNER JOIN `gcontact` on `glink`.`gcid` = `gcontact`.`id`
 		where `glink`.`cid` = %d and `glink`.`uid` = %d
 		and `gcontact`.`nurl` in (select nurl from contact where uid = %d and self = 0 and blocked = 0 and hidden = 0 and id != %d ) 
@@ -214,7 +448,7 @@ function common_friends($uid,$cid,$start = 0,$limit=9999,$shuffle = false) {
 
 function count_common_friends_zcid($uid,$zcid) {
 
-	$r = q("SELECT count(*) as `total` 
+	$r = q("SELECT count(*) as `total`
 		FROM `glink` INNER JOIN `gcontact` on `glink`.`gcid` = `gcontact`.`id`
 		where `glink`.`zcid` = %d
 		and `gcontact`.`nurl` in (select nurl from contact where uid = %d and self = 0 and blocked = 0 and hidden = 0 ) ",
@@ -233,9 +467,9 @@ function common_friends_zcid($uid,$zcid,$start = 0, $limit = 9999,$shuffle = fal
 	if($shuffle)
 		$sql_extra = " order by rand() ";
 	else
-		$sql_extra = " order by `gcontact`.`name` asc "; 
+		$sql_extra = " order by `gcontact`.`name` asc ";
 
-	$r = q("SELECT `gcontact`.* 
+	$r = q("SELECT `gcontact`.*
 		FROM `glink` INNER JOIN `gcontact` on `glink`.`gcid` = `gcontact`.`id`
 		where `glink`.`zcid` = %d
 		and `gcontact`.`nurl` in (select nurl from contact where uid = %d and self = 0 and blocked = 0 and hidden = 0 ) 
@@ -269,9 +503,9 @@ function count_all_friends($uid,$cid) {
 
 function all_friends($uid,$cid,$start = 0, $limit = 80) {
 
-	$r = q("SELECT `gcontact`.* 
+	$r = q("SELECT `gcontact`.*
 		FROM `glink` INNER JOIN `gcontact` on `glink`.`gcid` = `gcontact`.`id`
-		where `glink`.`cid` = %d and `glink`.`uid` = %d 
+		where `glink`.`cid` = %d and `glink`.`uid` = %d
 		order by `gcontact`.`name` asc LIMIT %d, %d ",
 		intval($cid),
 		intval($uid),
@@ -289,16 +523,31 @@ function suggestion_query($uid, $start = 0, $limit = 80) {
 	if(! $uid)
 		return array();
 
-	$r = q("SELECT count(glink.gcid) as `total`, gcontact.* from gcontact 
+	$network = array(NETWORK_DFRN);
-		INNER JOIN glink on glink.gcid = gcontact.id 
+
+	if (get_config('system','diaspora_enabled'))
+		$network[] = NETWORK_DIASPORA;
+
+	if (!get_config('system','ostatus_disabled'))
+		$network[] = NETWORK_OSTATUS;
+
+	$sql_network = implode("', '", $network);
+	//$sql_network = "'".$sql_network."', ''";
+	$sql_network = "'".$sql_network."'";
+
+	$r = q("SELECT count(glink.gcid) as `total`, gcontact.* from gcontact
+		INNER JOIN glink on glink.gcid = gcontact.id
 		where uid = %d and not gcontact.nurl in ( select nurl from contact where uid = %d )
 		and not gcontact.name in ( select name from contact where uid = %d )
 		and not gcontact.id in ( select gcid from gcign where uid = %d )
-		group by glink.gcid order by total desc limit %d, %d ",
+		AND `gcontact`.`updated` != '0000-00-00 00:00:00'
+		AND `gcontact`.`network` IN (%s)
+		group by glink.gcid order by gcontact.updated desc,total desc limit %d, %d ",
 		intval($uid),
 		intval($uid),
 		intval($uid),
 		intval($uid),
+		$sql_network,
 		intval($start),
 		intval($limit)
 	);
@@ -306,22 +555,30 @@ function suggestion_query($uid, $start = 0, $limit = 80) {
 	if(count($r) && count($r) >= ($limit -1))
 		return $r;
 
-	$r2 = q("SELECT gcontact.* from gcontact 
+	$r2 = q("SELECT gcontact.* from gcontact
-		INNER JOIN glink on glink.gcid = gcontact.id 
+		INNER JOIN glink on glink.gcid = gcontact.id
 		where glink.uid = 0 and glink.cid = 0 and glink.zcid = 0 and not gcontact.nurl in ( select nurl from contact where uid = %d )
 		and not gcontact.name in ( select name from contact where uid = %d )
 		and not gcontact.id in ( select gcid from gcign where uid = %d )
+		AND `gcontact`.`updated` != '0000-00-00 00:00:00'
+		AND `gcontact`.`network` IN (%s)
 		order by rand() limit %d, %d ",
 		intval($uid),
 		intval($uid),
 		intval($uid),
+		$sql_network,
 		intval($start),
 		intval($limit)
 	);
 
+	$list = array();
+	foreach ($r2 AS $suggestion)
+		$list[$suggestion["nurl"]] = $suggestion;
 
-	return array_merge($r,$r2);
+	foreach ($r AS $suggestion)
+		$list[$suggestion["nurl"]] = $suggestion;
 
+	return $list;
 }
 
 function update_suggestions() {

include/tags.php

@@ -9,7 +9,7 @@ function create_tags_from_item($itemid) {
 
 	$searchpath = $a->get_baseurl()."/search?tag=";
 
-	$messages = q("SELECT `guid`, `uid`, `id`, `edited`, `deleted`, `title`, `body`, `tag`, `parent` FROM `item` WHERE `id` = %d LIMIT 1", intval($itemid));
+	$messages = q("SELECT `guid`, `uid`, `id`, `edited`, `deleted`, `created`, `received`, `title`, `body`, `tag`, `parent` FROM `item` WHERE `id` = %d LIMIT 1", intval($itemid));
 
 	if (!$messages)
 		return;
@@ -26,16 +26,6 @@ function create_tags_from_item($itemid) {
 	if ($message["deleted"])
 		return;
 
-	$cachefile = get_cachefile(urlencode($message["guid"])."-".hash("md5", $message['body']));
-
-	if (($cachefile != '') AND !file_exists($cachefile)) {
-		$s = prepare_text($message['body']);
-		$stamp1 = microtime(true);
-		file_put_contents($cachefile, $s);
-		$a->save_timestamp($stamp1, "file");
-		logger('create_tags_from_item: put item '.$message["id"].' into cachefile '.$cachefile);
-	}
-
 	$taglist = explode(",", $message["tag"]);
 
 	$tags = "";
@@ -47,6 +37,9 @@ function create_tags_from_item($itemid) {
 
 	$data = " ".$message["title"]." ".$message["body"]." ".$tags." ";
 
+	// ignore anything in a code block
+	$data = preg_replace('/\[code\](.*?)\[\/code\]/sm','',$data);
+
 	$tags = array();
 
 	$pattern = "/\W\#([^\[].*?)[\s'\".,:;\?!\[\]\/]/ism";
@@ -79,8 +72,22 @@ function create_tags_from_item($itemid) {
 			$term = $tag;
 		}
 
-		$r = q("INSERT INTO `term` (`uid`, `oid`, `otype`, `type`, `term`, `url`) VALUES (%d, %d, %d, %d, '%s', '%s')",
+		if ($message["uid"] == 0) {
-			intval($message["uid"]), intval($itemid), intval(TERM_OBJ_POST), intval($type), dbesc($term), dbesc($link));
+			$global = true;
+
+			q("UPDATE `term` SET `global` = 1 WHERE `otype` = %d AND `guid` = '%s'",
+				intval(TERM_OBJ_POST), dbesc($message["guid"]));
+		} else {
+			$isglobal = q("SELECT `global` FROM `term` WHERE `uid` = 0 AND `otype` = %d AND `guid` = '%s'",
+				intval(TERM_OBJ_POST), dbesc($message["guid"]));
+
+			$global = (count($isglobal) > 0);
+		}
+
+		$r = q("INSERT INTO `term` (`uid`, `oid`, `otype`, `type`, `term`, `url`, `guid`, `created`, `received`, `global`)
+				VALUES (%d, %d, %d, %d, '%s', '%s', '%s', '%s', '%s', %d)",
+			intval($message["uid"]), intval($itemid), intval(TERM_OBJ_POST), intval($type), dbesc($term),
+			dbesc($link), dbesc($message["guid"]), dbesc($message["created"]), dbesc($message["received"]), intval($global));
 
 		// Search for mentions
 		if ((substr($tag, 0, 1) == '@') AND (strpos($link, $profile_base_friendica) OR strpos($link, $profile_base_diaspora))) {
@@ -106,10 +113,39 @@ function create_tags_from_itemuri($itemuri, $uid) {
 }
 
 function update_items() {
-	//$messages = q("SELECT `id` FROM `item` where tag !='' ORDER BY `created` DESC limit 10");
+	global $db;
-	$messages = q("SELECT `id` FROM `item` where tag !=''");
 
-	foreach ($messages as $message)
+        $messages = $db->q("SELECT `oid`,`item`.`guid`, `item`.`created`, `item`.`received` FROM `term` INNER JOIN `item` ON `item`.`id`=`term`.`oid` WHERE `term`.`otype` = 1 AND `term`.`guid` = ''", true);
-		create_tags_from_item($message["id"]);
+
+        logger("fetched messages: ".count($messages));
+        while ($message = $db->qfetch()) {
+
+		if ($message["uid"] == 0) {
+			$global = true;
+
+			q("UPDATE `term` SET `global` = 1 WHERE `otype` = %d AND `guid` = '%s'",
+				intval(TERM_OBJ_POST), dbesc($message["guid"]));
+		} else {
+			$isglobal = q("SELECT `global` FROM `term` WHERE `uid` = 0 AND `otype` = %d AND `guid` = '%s'",
+				intval(TERM_OBJ_POST), dbesc($message["guid"]));
+
+			$global = (count($isglobal) > 0);
+		}
+
+		q("UPDATE `term` SET `guid` = '%s', `created` = '%s', `received` = '%s', `global` = %d WHERE `otype` = %d AND `oid` = %d",
+			dbesc($message["guid"]), dbesc($message["created"]), dbesc($message["received"]),
+			intval($global), intval(TERM_OBJ_POST), intval($message["oid"]));
+	}
+
+        $db->qclose();
+
+	$messages = $db->q("SELECT `guid` FROM `item` WHERE `uid` = 0", true);
+
+	logger("fetched messages: ".count($messages));
+	while ($message = $db->qfetch()) {
+		q("UPDATE `item` SET `global` = 1 WHERE `guid` = '%s'", dbesc($message["guid"]));
+	}
+
+	$db->qclose();
 }
 ?>

include/tagupdate.php

@@ -0,0 +1,22 @@
+<?php
+require_once("boot.php");
+require_once("include/tags.php");
+
+global $a, $db;
+
+if(is_null($a))
+	$a = new App;
+
+if(is_null($db)) {
+	@include(".htconfig.php");
+	require_once("include/dba.php");
+	$db = new dba($db_host, $db_user, $db_pass, $db_data);
+	unset($db_host, $db_user, $db_pass, $db_data);
+}
+
+load_config('config');
+load_config('system');
+
+update_items();
+killme();
+?>

include/text.php

@@ -2,6 +2,7 @@
 
 require_once("include/template_processor.php");
 require_once("include/friendica_smarty.php");
+require_once("mod/proxy.php");
 
 if(! function_exists('replace_macros')) {
 /**
@@ -269,23 +270,23 @@ if(! function_exists('paginate_data')) {
  * @return Array data for pagination template
  */
 function paginate_data(&$a, $count=null) {
-	$stripped = preg_replace('/(&page=[0-9]*)/','',$a->query_string);
+	$stripped = preg_replace('/([&?]page=[0-9]*)/','',$a->query_string);
 
 	$stripped = str_replace('q=','',$stripped);
 	$stripped = trim($stripped,'/');
 	$pagenum = $a->pager['page'];
 
-	if (($a->page_offset != "") AND !strstr($stripped, "&offset="))
+	if (($a->page_offset != "") AND !preg_match('/[?&].offset=/', $stripped))
 		$stripped .= "&offset=".urlencode($a->page_offset);
-	if (!strpos($stripped, "?")) {
-		if ($pos = strpos($stripped, "&"))
-			$stripped = substr($stripped, 0, $pos)."?".substr($stripped, $pos + 1);
-	}
 
 	$url = $a->get_baseurl() . '/' . $stripped;
 
 	$data = array();
 	function _l(&$d, $name, $url, $text, $class="") {
+		if (!strpos($url, "?")) {
+			if ($pos = strpos($url, "&"))
+				$url = substr($url, 0, $pos)."?".substr($url, $pos + 1);
+		}
 
 		$d[$name] = array('url'=>$url, 'text'=>$text, 'class'=>$class);
 	}
@@ -384,6 +385,18 @@ function alt_pager(&$a, $i) {
 
 }}
 
+if(! function_exists('scroll_loader')) {
+/**
+ * Loader for infinite scrolling
+ * @return string html for loader
+ */
+function scroll_loader() {
+	$tpl = get_markup_template("scroll_loader.tpl");
+	return replace_macros($tpl, array(
+		'wait' => t('Loading more entries...'),
+		'end' => t('The end')
+	));
+}}
 
 if(! function_exists('expand_acl')) {
 /**
@@ -739,6 +752,9 @@ if(! function_exists('get_tags')) {
 function get_tags($s) {
 	$ret = array();
 
+	// Convert hashtag links to hashtags
+	$s = preg_replace("/#\[url\=([^\[\]]*)\](.*?)\[\/url\]/ism", "#$2", $s);
+
 	// ignore anything in a code block
 	$s = preg_replace('/\[code\](.*?)\[\/code\]/sm','',$s);
 
@@ -928,7 +944,7 @@ function micropro($contact, $redirect = false, $class = '', $textmode = false) {
 			. (($click) ? ' fakelink' : '') . '" '
 			. (($redir) ? ' target="redir" ' : '')
 			. (($url) ? ' href="' . $url . '"' : '') . $click . ' ><img class="contact-block-img' . $class . $sparkle . '" src="'
-			. $contact['micro'] . '" title="' . $contact['name'] . ' [' . $contact['url'] . ']" alt="' . $contact['name']
+			. proxy_url($contact['micro']) . '" title="' . $contact['name'] . ' [' . $contact['url'] . ']" alt="' . $contact['name']
 			. '" /></a></div>' . "\r\n";
 	}
 }}
@@ -1109,7 +1125,8 @@ function smilies($s, $sample = false) {
 		':like',
 		':dislike',
                 '~friendica',
-                'red#'
+                'red#',
+		'red#matrix'
 
 	);
 
@@ -1147,7 +1164,8 @@ function smilies($s, $sample = false) {
 		'<img class="smiley" src="' . $a->get_baseurl() . '/images/like.gif" alt=":like" />',
 		'<img class="smiley" src="' . $a->get_baseurl() . '/images/dislike.gif" alt=":dislike" />',
 		'<a href="http://friendica.com">~friendica <img class="smiley" src="' . $a->get_baseurl() . '/images/friendica-16.png" alt="~friendica" /></a>',
-		'<a href="http://redmatrix.me/">red <img class="smiley" src="' . $a->get_baseurl() . '/images/rhash-16.png" alt="red" /></a>'
+		'<a href="http://redmatrix.me/">red<img class="smiley" src="' . $a->get_baseurl() . '/images/rm-16.png" alt="red" />matrix</a>',
+		'<a href="http://redmatrix.me/">red<img class="smiley" src="' . $a->get_baseurl() . '/images/rm-16.png" alt="red" />matrix</a>'
 	);
 
 	$params = array('texts' => $texts, 'icons' => $icons, 'string' => $s);
@@ -1278,6 +1296,28 @@ function redir_private_images($a, &$item) {
 
 }}
 
+function put_item_in_cache(&$item, $update = false) {
+
+	if (($item["rendered-hash"] != hash("md5", $item["body"])) OR ($item["rendered-hash"] == "") OR
+		($item["rendered-html"] == "") OR get_config("system", "ignore_cache")) {
+
+		// The function "redir_private_images" changes the body.
+		// I'm not sure if we should store it permanently, so we save the old value.
+		$body = $item["body"];
+
+		$a = get_app();
+		redir_private_images($a, $item);
+
+		$item["rendered-html"] = prepare_text($item["body"]);
+		$item["rendered-hash"] = hash("md5", $item["body"]);
+		$item["body"] = $body;
+
+		if ($update AND ($item["id"] != 0)) {
+			q("UPDATE `item` SET `rendered-html` = '%s', `rendered-hash` = '%s' WHERE `id` = %d",
+				dbesc($item["rendered-html"]), dbesc($item["rendered-hash"]), intval($item["id"]));
+		}
+	}
+}
 
 // Given an item array, convert the body element from bbcode to html and add smilie icons.
 // If attach is true, also add icons for item attachments
@@ -1329,28 +1369,8 @@ function prepare_body(&$item,$attach = false, $preview = false) {
 	$item['hashtags'] = $hashtags;
 	$item['mentions'] = $mentions;
 
-
+	put_item_in_cache($item, true);
-	$cachefile = get_cachefile(urlencode($item["guid"])."-".hash("md5", $item['body']));
+	$s = $item["rendered-html"];
-
-	if (($cachefile != '')) {
-		if (file_exists($cachefile)) {
-			$stamp1 = microtime(true);
-			$s = file_get_contents($cachefile);
-			$a->save_timestamp($stamp1, "file");
-		} else {
-			redir_private_images($a, $item);
-			$s = prepare_text($item['body']);
-
-			$stamp1 = microtime(true);
-			file_put_contents($cachefile, $s);
-			$a->save_timestamp($stamp1, "file");
-
-			logger('prepare_body: put item '.$item["id"].' into cachefile '.$cachefile);
-		}
-	} else {
-		redir_private_images($a, $item);
-		$s = prepare_text($item['body']);
-	}
 
 	require_once("mod/proxy.php");
 	$s = proxy_parse_html($s);

include/threads.php

@@ -1,5 +1,5 @@
 <?php
-function add_thread($itemid) {
+function add_thread($itemid, $onlyshadow = false) {
 	$items = q("SELECT `uid`, `created`, `edited`, `commented`, `received`, `changed`, `wall`, `private`, `pubmail`, `moderated`, `visible`, `spam`, `starred`, `bookmark`, `contact-id`,
 			`deleted`, `origin`, `forum_mode`, `mention`, `network`  FROM `item` WHERE `id` = %d AND (`parent` = %d OR `parent` = 0) LIMIT 1", intval($itemid), intval($itemid));
 
@@ -9,13 +9,98 @@ function add_thread($itemid) {
 	$item = $items[0];
 	$item['iid'] = $itemid;
 
-	$result = dbq("INSERT INTO `thread` (`"
+	if (!$onlyshadow) {
-			.implode("`, `", array_keys($item))
+		$result = dbq("INSERT INTO `thread` (`"
-			."`) VALUES ('"
+				.implode("`, `", array_keys($item))
-			.implode("', '", array_values($item))
+				."`) VALUES ('"
-			."')" );
+				.implode("', '", array_values($item))
+				."')");
 
-	logger("add_thread: Add thread for item ".$itemid." - ".print_r($result, true), LOGGER_DEBUG);
+		logger("add_thread: Add thread for item ".$itemid." - ".print_r($result, true), LOGGER_DEBUG);
+	}
+
+	// is it already a copy?
+	if (($itemid == 0) OR ($item['uid'] == 0))
+		return;
+
+	// Is it a visible public post?
+	if (!$item["visible"] OR $item["deleted"] OR $item["moderated"] OR $item["private"])
+		return;
+
+	// is it an entry from a connector? Only add an entry for natively connected networks
+	if (!in_array($item["network"], array(NETWORK_DFRN, NETWORK_DIASPORA, NETWORK_OSTATUS, "")))
+		return;
+
+	// Only do these checks if the post isn't a wall post
+	if (!$item["wall"]) {
+		// Check, if hide-friends is activated - then don't do a shadow entry
+		$r = q("SELECT `hide-friends` FROM `profile` WHERE `is-default` AND `uid` = %d AND NOT `hide-friends`",
+			$item['uid']);
+		if (!count($r))
+			return;
+		// Check if the contact is hidden or blocked
+		$r = q("SELECT `id` FROM `contact` WHERE NOT `hidden` AND NOT `blocked` AND `id` = %d",
+			$item['contact-id']);
+		if (!count($r))
+			return;
+	}
+
+	// Only add a shadow, if the profile isn't hidden
+	$r = q("SELECT `uid` FROM `user` where `uid` = %d AND NOT `hidewall`", $item['uid']);
+	if (!count($r))
+		return;
+
+	$item = q("SELECT * FROM `item` WHERE `id` = %d",
+		intval($itemid));
+
+	if (count($item) AND ($item[0]["allow_cid"] == '')  AND ($item[0]["allow_gid"] == '') AND
+		($item[0]["deny_cid"] == '') AND ($item[0]["deny_gid"] == '')) {
+
+		$r = q("SELECT `id` FROM `item` WHERE `uri` = '%s' AND `uid` = 0 LIMIT 1",
+			dbesc($item['uri']));
+
+		if (!$r) {
+			// Preparing public shadow (removing user specific data)
+			require_once("include/items.php");
+			require_once("include/Contact.php");
+
+			unset($item[0]['id']);
+			$item[0]['uid'] = 0;
+			$item[0]['contact-id'] = get_contact($item[0]['author-link'], 0);
+			$public_shadow = item_store($item[0], false, false, true);
+
+			logger("add_thread: Stored public shadow for post ".$itemid." under id ".$public_shadow, LOGGER_DEBUG);
+		}
+	}
+}
+
+function add_shadow_entry($item) {
+
+	// Is this a shadow entry?
+	if ($item['uid'] == 0)
+		return;
+
+	// Is there a shadow parent?
+	$r = q("SELECT `id` FROM `item` WHERE `uri` = '%s' AND `uid` = 0 LIMIT 1", dbesc($item['parent-uri']));
+	if (!count($r))
+		return;
+
+	// Is there already a shadow entry?
+	$r = q("SELECT `id` FROM `item` WHERE `uri` = '%s' AND `uid` = 0 LIMIT 1", dbesc($item['uri']));
+
+	if (count($r))
+		return;
+
+	// Preparing public shadow (removing user specific data)
+	require_once("include/items.php");
+	require_once("include/Contact.php");
+
+	unset($item['id']);
+	$item['uid'] = 0;
+	$item['contact-id'] = get_contact($item['author-link'], 0);
+	$public_shadow = item_store($item, false, false, true);
+
+	logger("Stored public shadow for comment ".$item['uri']." under id ".$public_shadow, LOGGER_DEBUG);
 }
 
 function update_thread_uri($itemuri, $uid) {
@@ -27,7 +112,7 @@ function update_thread_uri($itemuri, $uid) {
 }
 
 function update_thread($itemid, $setmention = false) {
-	$items = q("SELECT `uid`, `created`, `edited`, `commented`, `received`, `changed`, `wall`, `private`, `pubmail`, `moderated`, `visible`, `spam`, `starred`, `bookmark`, `contact-id`,
+	$items = q("SELECT `uid`, `uri`, `created`, `edited`, `commented`, `received`, `changed`, `wall`, `private`, `pubmail`, `moderated`, `visible`, `spam`, `starred`, `bookmark`, `contact-id`,
 			`deleted`, `origin`, `forum_mode`, `network`  FROM `item` WHERE `id` = %d AND (`parent` = %d OR `parent` = 0) LIMIT 1", intval($itemid), intval($itemid));
 
 	if (!$items)
@@ -40,16 +125,50 @@ function update_thread($itemid, $setmention = false) {
 
 	$sql = "";
 
-	foreach ($item AS $field => $data) {
+	foreach ($item AS $field => $data)
-		if ($sql != "")
+		if ($field != "uri") {
-			$sql .= ", ";
+			if ($sql != "")
+				$sql .= ", ";
 
-		$sql .= "`".$field."` = '".$data."'";
+			$sql .= "`".$field."` = '".dbesc($data)."'";
-	}
+		}
 
-	$result = q("UPDATE `thread` SET ".$sql." WHERE `iid` = %d", $itemid);
+	$result = q("UPDATE `thread` SET ".$sql." WHERE `iid` = %d", intval($itemid));
 
 	logger("update_thread: Update thread for item ".$itemid." - ".print_r($result, true)." ".print_r($item, true), LOGGER_DEBUG);
+
+	// Updating a shadow item entry
+	$items = q("SELECT `id`, `title`, `body`, `created`, `edited`, `commented`, `received`, `changed`, `wall`, `private`, `pubmail`,
+			`moderated`, `visible`, `spam`, `starred`, `bookmark`, `deleted`, `origin`, `forum_mode`, `network`
+			FROM `item` WHERE `uri` = '%s' AND `uid` = 0 LIMIT 1", dbesc($item["uri"]));
+
+	if (!$items)
+		return;
+
+	$item = $items[0];
+
+	$result = q("UPDATE `item` SET `title` = '%s', `body` = '%s', `network` = '%s' WHERE `id` = %d",
+			dbesc($item["title"]),
+			dbesc($item["body"]),
+			dbesc($item["network"]),
+			intval($item["id"])
+		);
+	logger("update_thread: Updating public shadow for post ".$item["id"]." Result: ".print_r($result, true), LOGGER_DEBUG);
+
+	/*
+	$sql = "";
+
+	foreach ($item AS $field => $data)
+		if ($field != "id") {
+			if ($sql != "")
+				$sql .= ", ";
+
+			$sql .= "`".$field."` = '".dbesc($data)."'";
+		}
+	//logger("update_thread: Updating public shadow for post ".$item["id"]." SQL: ".$sql, LOGGER_DEBUG);
+	$result = q("UPDATE `item` SET ".$sql." WHERE `id` = %d", intval($item["id"]));
+	logger("update_thread: Updating public shadow for post ".$item["id"]." Result: ".print_r($result, true), LOGGER_DEBUG);
+	*/
 }
 
 function delete_thread_uri($itemuri, $uid) {
@@ -57,17 +176,32 @@ function delete_thread_uri($itemuri, $uid) {
 
 	if(count($messages))
 		foreach ($messages as $message)
-			delete_thread($message["id"]);
+			delete_thread($message["id"], $itemuri);
 }
 
-function delete_thread($itemid) {
+function delete_thread($itemid, $itemuri = "") {
+	$item = q("SELECT `uid` FROM `thread` WHERE `iid` = %d", intval($itemid));
+
 	$result = q("DELETE FROM `thread` WHERE `iid` = %d", intval($itemid));
 
 	logger("delete_thread: Deleted thread for item ".$itemid." - ".print_r($result, true), LOGGER_DEBUG);
+
+	if ($itemuri != "") {
+		$r = q("SELECT `id` FROM `item` WHERE `uri` = '%s' AND NOT (`uid` IN (%d, 0))",
+				dbesc($itemuri),
+				intval($item["uid"])
+			);
+		if (!count($r)) {
+			$r = q("DELETE FROM `item` WHERE `uri` = '%s' AND `uid` = 0)",
+				dbesc($itemuri)
+			);
+			logger("delete_thread: Deleted shadow for item ".$itemuri." - ".print_r($result, true), LOGGER_DEBUG);
+		}
+	}
 }
 
 function update_threads() {
-        global $db;
+	global $db;
 
 	logger("update_threads: start");
 
@@ -96,4 +230,21 @@ function update_threads_mention() {
 			q("UPDATE `thread` SET `mention` = 1 WHERE `iid` = %d", $parent["parent"]);
 	}
 }
+
+
+function update_shadow_copy() {
+	global $db;
+
+	logger("start");
+
+	$messages = $db->q(sprintf("SELECT `iid` FROM `thread` WHERE `uid` != 0 AND `network` IN ('', '%s', '%s', '%s')
+					AND `visible` AND NOT `deleted` AND NOT `moderated` AND NOT `private` ORDER BY `created`",
+				NETWORK_DFRN, NETWORK_DIASPORA, NETWORK_OSTATUS), true);
+
+	logger("fetched messages: ".count($messages));
+	while ($message = $db->qfetch())
+		add_thread($message["iid"], true);
+
+	$db->qclose();
+}
 ?>

index.php

@@ -115,7 +115,6 @@ if((x($_GET,'zrl')) && (!$install && !$maintenance)) {
  *
  * What we really need to do is output the raw headers ourselves so we can keep them separate.
  *
-
  */
 
 // header('Link: <' . $a->get_baseurl() . '/amcd>; rel="acct-mgmt";');
@@ -498,30 +497,38 @@ $(document).ready(function() {
 });
 
 function loadcontent() {
-	//$("div.loader").show();
+	if (lockLoadContent) return;
+	lockLoadContent = true;
+
+	$("#scroll-loader").fadeIn('normal');
 
 	num+=1;
 
 	console.log('Loading page ' + num);
 
 	$.get('/network?mode=raw$reload_uri&page=' + num, function(data) {
-		$(data).insertBefore('#conversation-end');
+		$("#scroll-loader").hide();
+		if ($(data).length > 0) {
+			$(data).insertBefore('#conversation-end');
+			lockLoadContent = false;
+		} else {
+			$("#scroll-end").fadeIn('normal');
+		}
 	});
-
-	//$("div.loader").fadeOut('normal');
 }
 
 var num = $pageno;
+var lockLoadContent = false;
 
 $(window).scroll(function(e){
 
 	if ($(document).height() != $(window).height()) {
 		// First method that is expected to work - but has problems with Chrome
-		if ($(window).scrollTop() == $(document).height() - $(window).height())
+		if ($(window).scrollTop() > ($(document).height() - $(window).height() * 1.5))
 			loadcontent();
 	} else {
 		// This method works with Chrome - but seems to be much slower in Firefox
-		if ($(window).scrollTop() > (($("section").height() + $("header").height() + $("footer").height()) - $(window).height()))
+		if ($(window).scrollTop() > (($("section").height() + $("header").height() + $("footer").height()) - $(window).height() * 1.5))
 			loadcontent();
 	}
 });

js/main.js

@@ -210,7 +210,7 @@
 			});
 			eSysmsg.children("info").each(function(){
 				text = $(this).text();
-				$.jGrowl(text, { sticky: false, theme: 'info', life: 10000 });
+				$.jGrowl(text, { sticky: false, theme: 'info', life: 5000 });
 			});
 			
 		});
@@ -290,7 +290,7 @@
 			if(livetime) {
 				clearTimeout(livetime);
 			}
-			livetime = setTimeout(liveUpdate, 10000);
+			livetime = setTimeout(liveUpdate, 5000);
 			return;
 		}
 		if(livetime != null)

library/HTML5/Parser.php

@@ -20,7 +20,12 @@ class HTML5_Parser
 
 	// Cleanup invalid HTML
 	$doc = new DOMDocument();
-	@$doc->loadHTML($text);
+
+	if (mb_detect_encoding($text, "UTF-8", true) == "UTF-8")
+		@$doc->loadHTML('<?xml encoding="UTF-8" ?>'.$text);
+	else
+		@$doc->loadHTML($text);
+
 	$text = $doc->saveHTML();
 
         $tokenizer = new HTML5_Tokenizer($text, $builder);

library/Mobile_Detect/.gitignore

@@ -0,0 +1,11 @@
+vendor/
+nbproject/
+/*.buildpath
+/*.project
+/.settings
+/error.log
+/export/nicejson
+.idea/
+*.iml
+/coverage
+/phpunit.phar

library/Mobile_Detect/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "export/nicejson"]
+	path = export/nicejson
+	url = https://github.com/GerHobbelt/nicejson-php.git

library/Mobile_Detect/.php_cs

@@ -0,0 +1,24 @@
+<?php
+
+use Symfony\CS\FixerInterface;
+
+$finder = Symfony\CS\Finder\DefaultFinder::create()
+    ->notName('LICENSE')
+    ->notName('README.md')
+    ->notName('.php_cs')
+    ->notName('composer.*')
+    ->notName('phpunit.xml*')
+    ->notName('*.phar')
+    ->exclude('vendor')
+    ->exclude('examples')
+    ->exclude('Symfony/CS/Tests/Fixer')
+    ->notName('ElseifFixer.php')
+    ->exclude('data')
+    ->in(__DIR__)
+;
+
+return Symfony\CS\Config\Config::create()
+    ->finder($finder)
+;
+
+

library/Mobile_Detect/.travis.yml

@@ -0,0 +1,17 @@
+language: php
+php:
+  - "5.2"
+  - "5.3"
+  - "5.4"
+  - "5.5"
+  - "5.6"
+
+branches:
+  only:
+    - devel
+
+script:
+  - phpunit -v -c tests/phpunit.xml
+
+notifications:
+  email: false

library/Mobile_Detect/LICENSE.txt

@@ -0,0 +1,48 @@
+MIT License
+
+Copyright (c) <2011-2014> <Serban Ghita> <serbanghita@gmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+Developer’s Certificate of Origin 1.1
+
+By making a contribution to this project, I certify that:
+
+(a) The contribution was created in whole or in part by me and I
+    have the right to submit it under the open source license
+    indicated in the file; or
+
+(b) The contribution is based upon previous work that, to the best
+    of my knowledge, is covered under an appropriate open source
+    license and I have the right under that license to submit that
+    work with modifications, whether created in whole or in part
+    by me, under the same open source license (unless I am
+    permitted to submit under a different license), as indicated
+    in the file; or
+
+(c) The contribution was provided directly to me by some other
+    person who certified (a), (b) or (c) and I have not modified
+    it.
+
+(d) I understand and agree that this project and the contribution
+    are public and that a record of the contribution (including all
+    personal information I submit with it, including my sign-off) is
+    maintained indefinitely and may be redistributed consistent with
+    this project or the open source license(s) involved.

library/Mobile_Detect/README.md

@@ -0,0 +1,219 @@
+[![Build Status](https://travis-ci.org/serbanghita/Mobile-Detect.svg?branch=devel)](https://travis-ci.org/serbanghita/Mobile-Detect) [![Latest Stable Version](https://poser.pugx.org/mobiledetect/mobiledetectlib/v/stable.svg)](https://packagist.org/packages/mobiledetect/mobiledetectlib) [![Total Downloads](https://poser.pugx.org/mobiledetect/mobiledetectlib/downloads.svg)](https://packagist.org/packages/mobiledetect/mobiledetectlib) [![Daily Downloads](https://poser.pugx.org/mobiledetect/mobiledetectlib/d/daily.png)](https://packagist.org/packages/mobiledetect/mobiledetectlib) [![License](https://poser.pugx.org/mobiledetect/mobiledetectlib/license.svg)](https://packagist.org/packages/mobiledetect/mobiledetectlib)
+[![Gitter](https://badges.gitter.im/Join Chat.svg)](https://gitter.im/serbanghita/Mobile-Detect?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
+
+![Mobile Detect](http://demo.mobiledetect.net/logo-github.png)
+
+> Motto: "Every business should have a mobile detection script to detect mobile readers."
+
+<i>Mobile_Detect is a lightweight PHP class for detecting mobile devices (including tablets).
+It uses the User-Agent string combined with specific HTTP headers to detect the mobile environment.</i>
+
+> We're commited to make Mobile_Detect the best open-source mobile detection resource and this is why before each release we're running [unit tests](./tests), we also research and update the detection rules on <b>daily</b> and <b>weekly</b> basis.
+
+Your website's _content strategy_ is important! You need a complete toolkit to deliver an experience that is  _optimized_, _fast_ and _relevant_ to your users. Mobile_Detect class is a [server-side detection](http://www.w3.org/TR/mwabp/#bp-devcap-detection) tool that can help you with your RWD strategy, it is not a replacement for CSS3 media queries or other forms of client-side feature detection.
+
+##### This month updates
+
+First of all a **BIG THANK YOU** to our growing community for your continuous support and for all the feedback received! I'm still working my way with the current issues and all the emails. 
+
+Nick is almost done with all the code for the upcoming `3.0.0` so that I only have to integrate the new parsing engine. We will release minor `2.8.xx` versions until a feature freeze where we will switch to the new branch. You will all be announced before this and hopefully we can make the transition smooth for everyone.
+
+<a href="http://trycatch.us/"><img align="left" src="http://assets.mobiledetect.net/img/try_catch_logo_80px.jpg" hspace="20"></a> Last but not least, special thanks for supporting us to our friends from [TryCatch.us](http://trycatch.us/) who _are set to carefully curate the most talented developers in Europe_!
+
+Thank you all and we're excited for the new release!
+
+##### Download and demo
+
+|Download|Docs|Examples|
+|-------------|-------------|-------------|
+|[Go to releases](../../tags)|[Become a contributor](../../wiki/Become-a-contributor)|[Code examples](../../wiki/Code-examples)
+|[Mobile_Detect.php](./Mobile_Detect.php)|[History](../../wiki/History)|[:iphone: Live demo!](http://is.gd/mobiletest)
+|[Composer package](https://packagist.org/packages/mobiledetect/mobiledetectlib)|
+
+##### Help
+
+|Pledgie|Paypal|
+|-------|------|
+|[Donate :+1:](http://pledgie.com/campaigns/21856)|[Donate :beer:](https://www.paypal.com/cgi-bin/webscr?cmd=_donations&business=mobiledetectlib%40gmail%2ecom&lc=US&item_name=Mobile%20Detect&currency_code=USD&bn=PP%2dDonationsBF%3abtn_donate_SM%2egif%3aNonHosted)|
+
+
+I'm currently paying for hosting and spend a lot of my family time to maintain the project and planning the future releases.
+I would highly appreciate any money donations that will keep the research going.
+
+Special thanks to the community :+1: for donations, [BrowserStack](http://browserstack.com) - for providing access to their great platform, [Zend](http://zend.com) - for donating licenses, [Dragos Gavrila](https://twitter.com/grafician) who contributed with the logo.
+
+##### 3rd party modules / [Submit new](../../issues/new?title=New%203rd%20party%20module&body=Name, Link and Description of the module.)
+
+:point_right: Keep `Mobile_Detect.php` class in a separate `module` and do NOT include it in your script core because of the high frequency of updates.
+:point_right: When including the class into you `web application` or `module` always use `include_once '../path/to/Mobile_Detect.php` to prevent conflicts.
+
+<table>
+
+<tr>
+  <td>Varnish Cache</td>
+  <td>
+    <p><a href="https://github.com/willemk/varnish-mobiletranslate">Varnish Mobile Detect</a> - Drop-in varnish solution to mobile user detection based on the Mobile-Detect library. Made by <a href="https://github.com/willemk">willemk</a></p>
+    <p><a href="https://github.com/carlosabalde/mobiledetect2vcl">mobiledetect2vcl</a> - Python script to transform the Mobile Detect JSON database into an UA-based mobile detection VCL subroutine easily integrable in any Varnish Cache configuration. Made by <a href="https://github.com/carlosabalde">Carlos Abalde</a></p>
+  </td>
+</tr>
+
+<tr>
+  <td>WordPress</td>
+  <td>
+    <p><a href="http://wordpress.org/extend/plugins/wp-mobile-detect/">WordPress Mobile Detect</a> - Gives you the ability to wrap that infographic in a `[notdevice][/notdevice]` shortcode so at the server level <code>WordPress</code> will decide to show that content only if the user is NOT on a phone or tablet. Made by <a href="http://profiles.wordpress.org/professor44/">Jesse Friedman</a>.</p>
+
+    <p><a href="http://wordpress.org/plugins/mobble/">mobble</a> - provides mobile related conditional functions for your site. e.g. is_iphone(), is_mobile() and is_tablet(). Made by Scott Evans.</p>
+
+    <p><a href="https://github.com/iamspacehead/responsage">WordPress Responsage</a> - A small <code>WordPress</code> theme plugin that allows you to make your images responsive. Made by <a href="https://github.com/iamspacehead">Adrian Ciaschetti</a>.</p>
+
+    <p><a href="http://wordpress.org/plugins/social-popup/">Social PopUP</a> - This plugin will display a popup or splash screen when a new user visit your site showing a Google+, Twitter and Facebook follow links. It uses Mobile_Detect to detect mobile devices.</p>
+  </td>
+</tr>
+
+<tr>
+  <td>Drupal</td>
+  <td>
+    <p><a href="http://drupal.org/project/mobile_switch">Drupal Mobile Switch</a> - The Mobile Switch <code>Drupal</code> module provides a automatic theme switch functionality for mobile devices,
+    detected by Browscap or Mobile Detect. Made by <a href="http://drupal.org/user/45267">Siegfried Neumann</a>.</p>
+
+   <p><a href="http://drupal.org/project/context_mobile_detect">Drupal Context Mobile Detect</a> - This is a <code>Drupal context</code> module which integrates Context and PHP Mobile Detect library.
+      Created by <a href="http://drupal.org/user/432492">Artem Shymko</a>.</p>
+
+   <p><a href="http://drupal.org/project/mobile_detect">Drupal Mobile Detect</a> - Lightweight mobile detect module for <code>Drupal</code> created by <a href="http://drupal.org/user/325244">Matthew Donadio</a></p>
+  </td>
+</tr>
+
+  <tr>
+    <td>Joomla</td>
+    <td><p><a href="http://www.yagendoo.com/en/blog/free-mobile-detection-plugin-for-joomla.html">yagendoo Joomla! Mobile Detection Plugin</a> - Lightweight PHP plugin for Joomla! that detects a mobile browser using the Mobile Detect class. Made by <a href="http://www.yagendoo.com/">yagendoo media</a>.</p></td>
+ </tr>
+
+ <tr>
+     <td>Magento</td>
+     <td><p><a href="http://www.magentocommerce.com/magento-connect/catalog/product/view/id/16835/">Magento</a> - This <code>Magento helper</code> from Optimise Web enables the use of all functions provided by MobileDetect.net.
+     Made by <a href="http://www.kathirvel.com">Kathir Vel</a>.</p></td>
+ </tr>
+
+ <tr>
+  <td>PrestaShop</td>
+  <td><p><a href="http://www.prestashop.com/">PrestaShop</a> is a free, secure and open source shopping cart platform. Mobile_Detect is included in the default package since 1.5.x.</p></td>
+ </tr>
+
+ <tr>
+  <td>Zend Framework</td>
+  <td>
+    <p><a href="https://github.com/neilime/zf2-mobile-detect.git">ZF2 Mobile-Detect</a> - Zend Framework 2 module that provides Mobile-Detect features (Mobile_Detect class as a service, helper for views and plugin controllers). Made by <a href="https://github.com/neilime">neilime</a></p>
+
+   <p><a href="https://github.com/nikolaposa/MobileDetectModule">ZF2 MobileDetectModule</a> - Facilitates integration of a PHP MobileDetect class with some ZF2-based application. Has similar idea like the existing ZF2 Mobile-Detect module, but differs in initialization and provision routine of the actual Mobile_Detect class. Appropriate view helper and controller plugin also have different conceptions. Made by <a href="https://github.com/nikolaposa">Nikola Posa</a></p>
+  </td>
+ </tr>
+
+  <tr>
+    <td>Symfony</td>
+    <td><p><a href="https://github.com/suncat2000/MobileDetectBundle">Symfony2 Mobile Detect Bundle</a> - The bundle for detecting mobile devices, manage mobile view and redirect to the mobile and tablet version.
+        Made by <a href="https://github.com/suncat2000">Nikolay Ivlev</a>.</p>
+        <p><a href="https://github.com/jbinfo/MobileDetectServiceProvider">Silex Mobile Detect Service Provider</a> - <code>Silex</code> service provider to interact with Mobile detect class methods. Made by <a href="https://github.com/jbinfo">Lhassan Baazzi</a>.</p>
+     </td>
+  </tr>
+
+  <tr>
+    <td>Laravel</td>
+    <td>
+    <p><a href="https://github.com/jenssegers/Laravel-Agent">Laravel-Agent</a> a user agent class for Laravel, based on Mobile Detect with some additional functionality. Made by <a href="https://github.com/jenssegers">Jens Segers</a>.</p>
+    <p><a href="https://github.com/hisorange/browser-detect">BrowserDetect</a> is a browser &amp; mobile detection package, collects and wrap together the best user-agent identifiers for Laravel. Created by <a href="https://github.com/hisorange">Varga Zsolt</a>.</p>
+    </td>
+  </tr>
+
+  <tr>
+    <td>ExpressionEngine</td>
+    <td><p><a href="https://github.com/garethtdavies/detect-mobile">EE2 Detect Mobile</a> - Lightweight PHP plugin for <code>EE2</code> that detects a mobile browser using the Mobile Detect class. Made by <a href="https://github.com/garethtdavies">Gareth Davies</a>.</p></td>
+ </tr>
+
+ <tr>
+  <td>Yii Framework</td>
+  <td><p><a href="https://github.com/iamsalnikov/MobileDetect">Yii Extension</a> - Mobile detect plugin for Yii framework. Made by <a href="https://github.com/iamsalnikov">Alexey Salnikov</a>.</p></td>
+ </tr>
+
+<tr>
+    <td>CakePHP</td>
+    <td><p><a href="https://github.com/chronon/CakePHP-MobileDetectComponent-Plugin">CakePHP MobileDetect</a> - <code>plugin</code> component for <code>CakePHP</code> 2.x. Made by <a href="https://github.com/chronon">Gregory Gaskill</a></p></td>
+</tr>
+
+<tr>
+    <td>FuelPHP</td>
+    <td><a href="https://github.com/rob-bar/special_agent">Special Agent</a> is a FuelPHP package which uses php-mobile-detect to determine whether a device is mobile or not.
+It overrides the Fuelphp Agent class its methods. Made by <a href="https://github.com/rob-bar">Robbie Bardjin</a>.</td>
+</tr>
+
+<tr>
+  <td>Typo3</td>
+  <td><a href="http://docs.typo3.org/typo3cms/extensions/px_mobiledetect/1.0.2/">px_mobiledetect</a> is an extension that helps to detect visitor's mobile device class (if that’s tablet or mobile device like smartphone). Made by Alexander Tretyak.</td>
+</tr>
+
+<tr>
+  <td>Statamic</td>
+  <td><p><a href="https://github.com/sergeifilippov/statamic-mobile-detect">Statamic CMS Mobile Detect</a> - <code>plugin</code>. Made by <a href="https://github.com/haikulab/statamic-mobile-detect">Sergei Filippov of Haiku Lab</a>.</p></td>
+</tr>
+
+<tr>
+  <td>Kohana</td>
+  <td><p><a href="https://github.com/madeinnordeste/kohana-mobile-detect">Kohana Mobile Detect</a> - an example of implementation of <code>Mobile_Detect</code> class with Kohana framework. Written by <a href="https://github.com/madeinnordeste">Luiz Alberto S. Ribeiro</a>.</p></td>
+</tr>
+
+<tr>
+  <td>mobile-detect.js</td>
+  <td><p>A <a href="https://github.com/hgoebl/mobile-detect.js">JavaScript port</a> of Mobile-Detect class. Made by <a href="https://github.com/hgoebl">Heinrich Goebl</a></p></td>
+  </tr>
+
+ <tr>
+      <td>Perl</td>
+      <td><p><a href="http://www.buzzerstar.com/development/">MobileDetect.pm</a> - <code>Perl module</code> for  Mobile Detect. Made by <a href="http://www.buzzerstar.com/">Sebastian Enger</a>.</p></td>
+ </tr>
+
+ <tr>
+      <td>python</td>
+      <td><p><a href="http://pypi.python.org/pypi/pymobiledetect">pymobiledetect</a> - Mobile detect <code>python package</code>. Made by Bas van Oostveen.</p></td>
+ </tr>
+ 
+ <tr>
+	 <td>Ruby</td>
+	 <td><p><a href="https://github.com/ktaragorn/mobile_detect">mobile_detect.rb</a> - A <code>Ruby gem</code> using the JSON data exposed by the php project and implementing a basic subset of the API (as much as can be done by the exposed data). Made by <a href="https://github.com/ktaragorn">Karthik T</a>.</p></td>
+ </tr>
+
+<tr>
+	<td>GoMobileDetect</td>
+	<td><p><a href="https://github.com/Shaked/gomobiledetect">GoMobileDetect</a> - Go port of Mobile Detect class. Made by <a href="https://github.com/Shaked">Shaked</a>.</p></td>
+</tr>
+
+ <tr>
+    <td>MemHT</td>
+    <td><p><a href="http://www.memht.com/">MemHT</a> is a Free PHP CMS and Blog that permit the creation and the management online of websites with few and easy steps. Has the class included in the core.</p></td>
+ </tr>
+
+ <tr>
+  <td>concrete5</td>
+  <td><p><a href="https://www.concrete5.org/">concrete5</a> is a CMS that is free and open source. The library is included in the core.</p></td>
+ </tr>
+
+ <tr>
+  <td>engine7</td>
+  <td><p><a href="https://github.com/gchiappe/exengine7">ExEngine 7</a> PHP Open Source Framework. The Mobile_Detect class is included in the engine.</p></td>
+ </tr>
+
+ <tr>
+  <td>Zikula</td>
+  <td><p><a href="http://zikula.org/">Zikula</a> is a free and open-source Content Management Framework, which allows you to run impressive websites and build powerful online applications. The core uses Mobile-Detect to switch to a special Mobile theme, using jQueryMobile</p></td>
+ </tr>
+
+<tr>
+    <td>UserAgentInfo</td>
+    <td><p><a href="https://github.com/quentin389/UserAgentInfo">UserAgentInfo</a> is a PHP class for parsing user agent strings (HTTP_USER_AGENT). Includes mobile checks, bot checks, browser types/versions and more.
+Based on browscap, Mobile_Detect and ua-parser. Created for high traffic websites and fast batch processing. Made by <a href="https://github.com/quentin389">quentin389</a></p></td>
+</tr>
+
+<tr>
+  <td>Craft CMS</td>
+  <td><p><a href="https://github.com/lewisjenkins/craft-lj-mobiledetect">LJ Mobile Detect</a> is a simple implementation of Mobile Detect for Craft CMS. Made by <a href="https://github.com/lewisjenkins">Lewis Jenkins</a></p></td>
+</tr>
+
+</table>

library/Mobile_Detect/composer.json

@@ -0,0 +1,30 @@
+{
+    "name": "mobiledetect/mobiledetectlib",
+    "type": "library",
+    "description": "Mobile_Detect is a lightweight PHP class for detecting mobile devices. It uses the User-Agent string combined with specific HTTP headers to detect the mobile environment.",
+    "keywords": ["mobile", "mobile detect", "mobile detector", "php mobile detect", "detect mobile devices"],
+    "homepage": "https://github.com/serbanghita/Mobile-Detect",
+    "license": "MIT",
+    "authors": [
+        {
+            "name": "Serban Ghita",
+            "email": "serbanghita@gmail.com",
+            "homepage": "http://mobiledetect.net",
+            "role": "Developer"
+        }
+    ],
+    "require": {
+        "php": ">=5.0.0"
+    },
+    "require-dev": {
+        "phpunit/phpunit": "*",
+        "johnkary/phpunit-speedtrap": "~1.0@dev",
+        "codeclimate/php-test-reporter": "dev-master"
+    },
+    "autoload": {
+        "classmap": ["Mobile_Detect.php"],
+        "psr-0": {
+            "Detection": "namespaced/"
+        }
+    }
+}

library/Mobile_Detect/namespaced/Detection/MobileDetect.php

@@ -0,0 +1,22 @@
+<?php
+/**
+ * Little piece of PHP to make Mobile_Detect auto-loadable in PSR-0 compatible PHP autoloaders like
+ * the Symfony Universal ClassLoader by Fabien Potencier. Since PSR-0 handles an underscore in
+ * classnames (on the filesystem) as a slash, "Mobile_Detect.php" autoloaders will try to convert
+ * the classname and path to "Mobile\Detect.php". This script will ensure autoloading with:
+ *  - Namespace:	   Detection
+ *  - Classname:       MobileDetect
+ *  - Namespased:      \Detection\MobileDetect
+ *  - Autoload path:   ./namespaced
+ *  - Converted path:  ./namespaced/Detection/MobileDetect.php
+ *
+ * Don't forget to use MobileDetect (instead of Mobile_Detect) as class in code when autoloading.
+ *
+ * Thanks to @WietseWind.
+ * For details please check: https://github.com/serbanghita/Mobile-Detect/pull/120
+ */
+
+namespace Detection;
+require_once dirname(__FILE__) . DIRECTORY_SEPARATOR . '..' . DIRECTORY_SEPARATOR . '..' . DIRECTORY_SEPARATOR . 'Mobile_Detect.php';
+
+class MobileDetect extends \Mobile_Detect {}

library/OAuth1.php

@@ -293,8 +293,8 @@ class OAuthRequest {
     }
     // fix for friendica redirect system
     
-    $http_url =  substr($http_url, 0, strpos($http_url,$parameters['q'])+strlen($parameters['q']));
+    $http_url =  substr($http_url, 0, strpos($http_url,$parameters['pagename'])+strlen($parameters['pagename']));
-    unset( $parameters['q'] );
+    unset( $parameters['pagename'] );
     
 	//echo "<pre>".__function__."\n"; var_dump($http_method, $http_url, $parameters, $_SERVER['REQUEST_URI']); killme();
     return new OAuthRequest($http_method, $http_url, $parameters);

library/html-to-markdown/.gitignore

@@ -0,0 +1,3 @@
+~*
+vendor
+composer.lock

library/html-to-markdown/.travis.yml

@@ -0,0 +1,6 @@
+language: php
+php:
+  - "5.5"
+  - "5.4"
+  - "5.3"
+script: phpunit --no-configuration HTML_To_MarkdownTest ./tests/HTML_To_MarkdownTest.php

library/html-to-markdown/HTML_To_Markdown.php

@@ -0,0 +1,598 @@
+<?php
+/**
+ * Class HTML_To_Markdown
+ *
+ * A helper class to convert HTML to Markdown.
+ *
+ * @version 2.2.1
+ * @author Nick Cernis <nick@cern.is>
+ * @link https://github.com/nickcernis/html2markdown/ Latest version on GitHub.
+ * @link http://twitter.com/nickcernis Nick on twitter.
+ * @license http://www.opensource.org/licenses/mit-license.php MIT
+ */
+class HTML_To_Markdown
+{
+    /**
+     * @var DOMDocument The root of the document tree that holds our HTML.
+     */
+    private $document;
+
+    /**
+     * @var string|boolean The Markdown version of the original HTML, or false if conversion failed
+     */
+    private $output;
+
+    /**
+     * @var array Class-wide options users can override.
+     */
+    private $options = array(
+        'header_style'    => 'setext', // Set to "atx" to output H1 and H2 headers as # Header1 and ## Header2
+        'suppress_errors' => true, // Set to false to show warnings when loading malformed HTML
+        'strip_tags'      => false, // Set to true to strip tags that don't have markdown equivalents. N.B. Strips tags, not their content. Useful to clean MS Word HTML output.
+        'bold_style'      => '**', // Set to '__' if you prefer the underlined style
+        'italic_style'    => '*', // Set to '_' if you prefer the underlined style
+        'remove_nodes'    => '', // space-separated list of dom nodes that should be removed. example: "meta style script"
+    );
+
+
+    /**
+     * Constructor
+     *
+     * Set up a new DOMDocument from the supplied HTML, convert it to Markdown, and store it in $this->$output.
+     *
+     * @param string $html The HTML to convert to Markdown.
+     * @param array $overrides [optional] List of style and error display overrides.
+     */
+    public function __construct($html = null, $overrides = null)
+    {
+        if ($overrides)
+            $this->options = array_merge($this->options, $overrides);
+
+        if ($html)
+            $this->convert($html);
+    }
+
+
+    /**
+     * Setter for conversion options
+     *
+     * @param $name
+     * @param $value
+     */
+    public function set_option($name, $value)
+    {
+        $this->options[$name] = $value;
+    }
+
+
+    /**
+     * Convert
+     *
+     * Loads HTML and passes to get_markdown()
+     *
+     * @param $html
+     * @return string The Markdown version of the html
+     */
+    public function convert($html)
+    {
+        $html = preg_replace('~>\s+<~', '><', $html); // Strip white space between tags to prevent creation of empty #text nodes
+
+        $this->document = new DOMDocument();
+
+        if ($this->options['suppress_errors'])
+            libxml_use_internal_errors(true); // Suppress conversion errors (from http://bit.ly/pCCRSX )
+
+        $this->document->loadHTML('<?xml encoding="UTF-8">' . $html); // Hack to load utf-8 HTML (from http://bit.ly/pVDyCt )
+        $this->document->encoding = 'UTF-8';
+
+        if ($this->options['suppress_errors'])
+            libxml_clear_errors();
+
+        return $this->get_markdown($html);
+    }
+
+
+    /**
+     * Is Child Of?
+     *
+     * Is the node a child of the given parent tag?
+     *
+     * @param $parent_name string|array The name of the parent node(s) to search for e.g. 'code' or array('pre', 'code')
+     * @param $node
+     * @return bool
+     */
+    private static function is_child_of($parent_name, $node)
+    {
+        for ($p = $node->parentNode; $p != false; $p = $p->parentNode) {
+            if (is_null($p))
+                return false;
+
+            if ( is_array($parent_name) && in_array($p->nodeName, $parent_name) )
+                return true;
+            
+            if ($p->nodeName == $parent_name)
+                return true;
+        }
+        return false;
+    }
+
+
+    /**
+     * Convert Children
+     *
+     * Recursive function to drill into the DOM and convert each node into Markdown from the inside out.
+     *
+     * Finds children of each node and convert those to #text nodes containing their Markdown equivalent,
+     * starting with the innermost element and working up to the outermost element.
+     *
+     * @param $node
+     */
+    private function convert_children($node)
+    {
+        // Don't convert HTML code inside <code> and <pre> blocks to Markdown - that should stay as HTML
+        if (self::is_child_of(array('pre', 'code'), $node))
+            return;
+
+        // If the node has children, convert those to Markdown first
+        if ($node->hasChildNodes()) {
+            $length = $node->childNodes->length;
+
+            for ($i = 0; $i < $length; $i++) {
+                $child = $node->childNodes->item($i);
+                $this->convert_children($child);
+            }
+        }
+
+        // Now that child nodes have been converted, convert the original node
+        $markdown = $this->convert_to_markdown($node);
+
+        // Create a DOM text node containing the Markdown equivalent of the original node
+        $markdown_node = $this->document->createTextNode($markdown);
+
+        // Replace the old $node e.g. "<h3>Title</h3>" with the new $markdown_node e.g. "### Title"
+        $node->parentNode->replaceChild($markdown_node, $node);
+    }
+
+
+    /**
+     * Get Markdown
+     *
+     * Sends the body node to convert_children() to change inner nodes to Markdown #text nodes, then saves and
+     * returns the resulting converted document as a string in Markdown format.
+     *
+     * @return string|boolean The converted HTML as Markdown, or false if conversion failed
+     */
+    private function get_markdown()
+    {
+        // Work on the entire DOM tree (including head and body)
+        $input = $this->document->getElementsByTagName("html")->item(0);
+
+        if (!$input)
+            return false;
+
+        // Convert all children of this root element. The DOMDocument stored in $this->doc will
+        // then consist of #text nodes, each containing a Markdown version of the original node
+        // that it replaced.
+        $this->convert_children($input);
+
+        // Sanitize and return the body contents as a string.
+        $markdown = $this->document->saveHTML(); // stores the DOMDocument as a string
+        $markdown = html_entity_decode($markdown, ENT_QUOTES, 'UTF-8');
+        $markdown = html_entity_decode($markdown, ENT_QUOTES, 'UTF-8'); // Double decode to cover cases like &amp;nbsp; http://www.php.net/manual/en/function.htmlentities.php#99984
+        $markdown = preg_replace("/<!DOCTYPE [^>]+>/", "", $markdown); // Strip doctype declaration
+        $unwanted = array('<html>', '</html>', '<body>', '</body>', '<head>', '</head>', '<?xml encoding="UTF-8">', '&#xD;');
+        $markdown = str_replace($unwanted, '', $markdown); // Strip unwanted tags
+        $markdown = trim($markdown, "\n\r\0\x0B");
+
+        $this->output = $markdown;
+
+        return $markdown;
+    }
+
+
+    /**
+     * Convert to Markdown
+     *
+     * Converts an individual node into a #text node containing a string of its Markdown equivalent.
+     *
+     * Example: An <h3> node with text content of "Title" becomes a text node with content of "### Title"
+     *
+     * @param $node
+     * @return string The converted HTML as Markdown
+     */
+    private function convert_to_markdown($node)
+    {
+        $tag = $node->nodeName; // the type of element, e.g. h1
+        $value = $node->nodeValue; // the value of that element, e.g. The Title
+        
+        // Strip nodes named in remove_nodes
+        $tags_to_remove = explode(' ', $this->options['remove_nodes']);
+        if ( in_array($tag, $tags_to_remove) )
+            return false;
+
+        switch ($tag) {
+            case "p":
+                $markdown = (trim($value)) ? rtrim($value) . PHP_EOL . PHP_EOL : '';
+                break;
+            case "pre":
+                $markdown = PHP_EOL . $this->convert_code($node) . PHP_EOL;
+                break;
+            case "h1":
+            case "h2":
+                $markdown = $this->convert_header($tag, $node);
+                break;
+            case "h3":
+                $markdown = "### " . $value . PHP_EOL . PHP_EOL;
+                break;
+            case "h4":
+                $markdown = "#### " . $value . PHP_EOL . PHP_EOL;
+                break;
+            case "h5":
+                $markdown = "##### " . $value . PHP_EOL . PHP_EOL;
+                break;
+            case "h6":
+                $markdown = "###### " . $value . PHP_EOL . PHP_EOL;
+                break;
+            case "em":
+            case "i":
+            case "strong":
+            case "b":
+                $markdown = $this->convert_emphasis($tag, $value);
+                break;
+            case "hr":
+                $markdown = "- - - - - -" . PHP_EOL . PHP_EOL;
+                break;
+            case "br":
+                $markdown = "  " . PHP_EOL;
+                break;
+            case "blockquote":
+                $markdown = $this->convert_blockquote($node);
+                break;
+            case "code":
+                $markdown = $this->convert_code($node);
+                break;
+            case "ol":
+            case "ul":
+                $markdown = $value . PHP_EOL;
+                break;
+            case "li":
+                $markdown = $this->convert_list($node);
+                break;
+            case "img":
+                $markdown = $this->convert_image($node);
+                break;
+            case "a":
+                $markdown = $this->convert_anchor($node);
+                break;
+            case "#text":
+                $markdown = preg_replace('~\s+~', ' ', $value);
+                $markdown = preg_replace('~^#~', '\\\\#', $markdown);
+                break;
+            case "#comment":
+                $markdown = '';
+                break;
+            case "div":
+                $markdown = ($this->options['strip_tags']) ? $value . PHP_EOL . PHP_EOL : html_entity_decode($node->C14N());
+                break;
+            default:
+                // If strip_tags is false (the default), preserve tags that don't have Markdown equivalents,
+                // such as <span> nodes on their own. C14N() canonicalizes the node to a string.
+                // See: http://www.php.net/manual/en/domnode.c14n.php
+                $markdown = ($this->options['strip_tags']) ? $value : html_entity_decode($node->C14N());
+        }
+
+        return $markdown;
+    }
+
+
+    /**
+     * Convert Header
+     *
+     * Converts h1 and h2 headers to Markdown-style headers in setext style,
+     * matching the number of underscores with the length of the title.
+     *
+     * e.g.     Header 1    Header Two
+     *          ========    ----------
+     *
+     * Returns atx headers instead if $this->options['header_style'] is "atx"
+     *
+     * e.g.    # Header 1   ## Header Two
+     *
+     * @param string $level The header level, including the "h". e.g. h1
+     * @param string $node The node to convert.
+     * @return string The Markdown version of the header.
+     */
+    private function convert_header($level, $node)
+    {
+        $content = $node->nodeValue;
+
+        if (!$this->is_child_of('blockquote', $node) && $this->options['header_style'] == "setext") {
+            $length = (function_exists('mb_strlen')) ? mb_strlen($content, 'utf-8') : strlen($content);
+            $underline = ($level == "h1") ? "=" : "-";
+            $markdown = $content . PHP_EOL . str_repeat($underline, $length) . PHP_EOL . PHP_EOL; // setext style
+        } else {
+            $prefix = ($level == "h1") ? "# " : "## ";
+            $markdown = $prefix . $content . PHP_EOL . PHP_EOL; // atx style
+        }
+
+        return $markdown;
+    }
+
+
+    /**
+     * Converts inline styles
+     * This function is used to render strong and em tags
+     * 
+     * eg <strong>bold text</strong> becomes **bold text** or __bold text__
+     * 
+     * @param string $tag
+     * @param string $value
+     * @return string
+     */
+     private function convert_emphasis($tag, $value)
+     {
+        if ($tag == 'i' || $tag == 'em') {
+            $markdown = $this->options['italic_style'] . $value . $this->options['italic_style'];
+        } else {
+            $markdown = $this->options['bold_style'] . $value . $this->options['bold_style'];
+        }
+        
+        return $markdown;
+     }
+
+
+    /**
+     * Convert Image
+     *
+     * Converts <img /> tags to Markdown.
+     *
+     * e.g.     <img src="/path/img.jpg" alt="alt text" title="Title" />
+     * becomes  ![alt text](/path/img.jpg "Title")
+     *
+     * @param $node
+     * @return string
+     */
+    private function convert_image($node)
+    {
+        $src = $node->getAttribute('src');
+        $alt = $node->getAttribute('alt');
+        $title = $node->getAttribute('title');
+
+        if ($title != "") {
+            $markdown = '![' . $alt . '](' . $src . ' "' . $title . '")'; // No newlines added. <img> should be in a block-level element.
+        } else {
+            $markdown = '![' . $alt . '](' . $src . ')';
+        }
+
+        return $markdown;
+    }
+
+
+    /**
+     * Convert Anchor
+     *
+     * Converts <a> tags to Markdown.
+     *
+     * e.g.     <a href="http://modernnerd.net" title="Title">Modern Nerd</a>
+     * becomes  [Modern Nerd](http://modernnerd.net "Title")
+     *
+     * @param $node
+     * @return string
+     */
+    private function convert_anchor($node)
+    {
+        $href = $node->getAttribute('href');
+        $title = $node->getAttribute('title');
+        $text = $node->nodeValue;
+
+        if ($title != "") {
+            $markdown = '[' . $text . '](' . $href . ' "' . $title . '")';
+        } else {
+            $markdown = '[' . $text . '](' . $href . ')';
+        }
+
+        if (! $href)
+            $markdown = html_entity_decode($node->C14N());
+
+        // Append a space if the node after this one is also an anchor
+        $next_node_name = $this->get_next_node_name($node);
+
+        if ($next_node_name == 'a')
+            $markdown = $markdown . ' ';
+
+        return $markdown;
+    }
+
+
+    /**
+     * Convert List
+     *
+     * Converts <ul> and <ol> lists to Markdown.
+     *
+     * @param $node
+     * @return string
+     */
+    private function convert_list($node)
+    {
+        // If parent is an ol, use numbers, otherwise, use dashes
+        $list_type = $node->parentNode->nodeName;
+        $value = $node->nodeValue;
+
+        if ($list_type == "ul") {
+            $markdown = "- " . trim($value) . PHP_EOL;
+        } else {
+            $number = $this->get_position($node);
+            $markdown = $number . ". " . trim($value) . PHP_EOL;
+        }
+
+        return $markdown;
+    }
+
+
+    /**
+     * Convert Code
+     *
+     * Convert code tags by indenting blocks of code and wrapping single lines in backticks.
+     *
+     * @param DOMNode $node
+     * @return string
+     */
+    private function convert_code($node)
+    {
+        // Store the content of the code block in an array, one entry for each line
+
+        $markdown = '';
+
+        $code_content = html_entity_decode($node->C14N());
+        $code_content = str_replace(array("<code>", "</code>"), "", $code_content);
+        $code_content = str_replace(array("<pre>", "</pre>"), "", $code_content);
+
+        $lines = preg_split('/\r\n|\r|\n/', $code_content);
+        $total = count($lines);
+
+        // If there's more than one line of code, prepend each line with four spaces and no backticks.
+        if ($total > 1 || $node->nodeName === 'pre') {
+
+            // Remove the first and last line if they're empty
+            $first_line = trim($lines[0]);
+            $last_line = trim($lines[$total - 1]);
+            $first_line = trim($first_line, "&#xD;"); //trim XML style carriage returns too
+            $last_line = trim($last_line, "&#xD;");
+
+            if (empty($first_line))
+                array_shift($lines);
+
+            if (empty($last_line))
+                array_pop($lines);
+
+            $count = 1;
+            foreach ($lines as $line) {
+                $line = str_replace('&#xD;', '', $line);
+                $markdown .= "    " . $line;
+                // Add newlines, except final line of the code
+                if ($count != $total)
+                    $markdown .= PHP_EOL;
+                $count++;
+            }
+            $markdown .= PHP_EOL;
+
+        } else { // There's only one line of code. It's a code span, not a block. Just wrap it with backticks.
+
+            $markdown .= "`" . $lines[0] . "`";
+
+        }
+        
+        return $markdown;
+    }
+
+
+    /**
+     * Convert blockquote
+     *
+     * Prepend blockquotes with > chars.
+     *
+     * @param $node
+     * @return string
+     */
+    private function convert_blockquote($node)
+    {
+        // Contents should have already been converted to Markdown by this point,
+        // so we just need to add ">" symbols to each line.
+
+        $markdown = '';
+
+        $quote_content = trim($node->nodeValue);
+
+        $lines = preg_split('/\r\n|\r|\n/', $quote_content);
+
+        $total_lines = count($lines);
+
+        foreach ($lines as $i => $line) {
+            $markdown .= "> " . $line . PHP_EOL;
+            if ($i + 1 == $total_lines)
+                $markdown .= PHP_EOL;
+        }
+
+        return $markdown;
+    }
+
+
+    /**
+     * Get Position
+     *
+     * Returns the numbered position of a node inside its parent
+     *
+     * @param $node
+     * @return int The numbered position of the node, starting at 1.
+     */
+    private function get_position($node)
+    {
+        // Get all of the nodes inside the parent
+        $list_nodes = $node->parentNode->childNodes;
+        $total_nodes = $list_nodes->length;
+
+        $position = 1;
+
+        // Loop through all nodes and find the given $node
+        for ($a = 0; $a < $total_nodes; $a++) {
+            $current_node = $list_nodes->item($a);
+
+            if ($current_node->isSameNode($node))
+                $position = $a + 1;
+        }
+
+        return $position;
+    }
+
+
+    /**
+     * Get Next Node Name
+     *
+     * Return the name of the node immediately after the passed one.
+     *
+     * @param $node
+     * @return string|null The node name (e.g. 'h1') or null.
+     */
+    private function get_next_node_name($node)
+    {
+        $next_node_name = null;
+
+        $current_position = $this->get_position($node);
+        $next_node = $node->parentNode->childNodes->item($current_position);
+
+        if ($next_node)
+            $next_node_name = $next_node->nodeName;
+
+        return $next_node_name;
+    }
+
+
+    /**
+     * To String
+     *
+     * Magic method to return Markdown output when HTML_To_Markdown instance is treated as a string.
+     *
+     * @return string
+     */
+    public function __toString()
+    {
+        return $this->output();
+    }
+
+
+    /**
+     * Output
+     *
+     * Getter for the converted Markdown contents stored in $this->output
+     *
+     * @return string
+     */
+    public function output()
+    {
+        if (!$this->output) {
+            return '';
+        } else {
+            return $this->output;
+        }
+    }
+}

library/html-to-markdown/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 Nick Cernis
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

library/html-to-markdown/README.md

@@ -0,0 +1,138 @@
+HTML To Markdown for PHP
+========================
+
+A helper class that converts HTML to [Markdown](http://daringfireball.net/projects/markdown/) for your sanity and convenience.
+
+[![Build Status](https://travis-ci.org/nickcernis/html-to-markdown.png?branch=master)](https://travis-ci.org/nickcernis/html-to-markdown)
+
+**Version**: 2.2.1  
+**Requires**: PHP 5.3+  
+**Author**: [@nickcernis](http://twitter.com/nickcernis)  
+**License**: [MIT](http://www.opensource.org/licenses/mit-license.php)  
+
+### Why convert HTML to Markdown?
+
+*"What alchemy is this?"* you mutter. *"I can see why you'd convert [Markdown to HTML](http://michelf.com/projects/php-markdown/),"* you continue, already labouring the question somewhat, *"but why go the other way?"*
+
+Typically you would convert HTML to Markdown if:
+
+1. You have an existing HTML document that needs to be edited by people with good taste.
+2. You want to store new content in HTML format but edit it as Markdown.
+3. You want to convert HTML email to plain text email. 
+4. You know a guy who's been converting HTML to Markdown for years, and now he can speak Elvish. You'd quite like to be able to speak Elvish.
+5. You just really like Markdown.
+
+### How to use it
+
+Either include HTML_To_Markdown.php directly:
+
+    require_once( dirname( __FILE__) . '/HTML_To_Markdown.php' );
+
+Or, require the library in your composer.json:
+
+    {
+        "require": {
+            "nickcernis/html-to-markdown": "dev-master"
+        }
+    }
+
+Then `composer install` and add `require 'vendor/autoload.php';` to the top of your script.
+
+Next, create a new HTML_To_Markdown instance, passing in your valid HTML code:
+
+    $html = "<h3>Quick, to the Batpoles!</h3>";
+    $markdown = new HTML_To_Markdown($html);
+
+The `$markdown` object now contains the Markdown version of your HTML. Use it like a string:
+
+    echo $markdown; // ==> ### Quick, to the Batpoles!
+
+Or access the Markdown output directly:
+
+    $string = $markdown->output();
+
+The included `demo` directory contains an HTML->Markdown conversion form to try out.
+
+### Conversion options
+
+By default, HTML To Markdown preserves HTML tags without Markdown equivalents, like `<span>` and `<div>`.
+
+To strip HTML tags that don't have a Markdown equivalent while preserving the content inside them, set `strip_tags` to true, like this:
+
+    $html = '<span>Turnips!</span>';
+    $markdown = new HTML_To_Markdown($html, array('strip_tags' => true)); // $markdown now contains "Turnips!"    
+
+Or more explicitly, like this:
+
+    $html = '<span>Turnips!</span>';
+    $markdown = new HTML_To_Markdown();
+    $markdown->set_option('strip_tags', true);
+    $markdown->convert($html); // $markdown now contains "Turnips!"
+
+Note that only the tags themselves are stripped, not the content they hold.
+
+To strip tags and their content, pass a space-separated list of tags in `remove_nodes`, like this:
+
+    $html = '<span>Turnips!</span><div>Monkeys!</div>';
+    $markdown = new HTML_To_Markdown($html, array('remove_nodes' => 'span div')); // $markdown now contains ""
+
+### Style options
+
+Bold and italic tags are converted using the asterisk syntax by default. Change this to the underlined syntax using the `bold_style` and `italic_style` options.
+
+    $html = '<em>Italic</em> and a <strong>bold</strong>';
+    $markdown = new HTML_To_Markdown();
+    $markdown->set_option('italic_style', '_');
+    $markdown->set_option('bold_style', '__');
+    $markdown->convert($html); // $markdown now contains "_Italic_ and a __bold__"
+
+### Limitations
+
+- Markdown Extra, MultiMarkdown and other variants aren't supported – just Markdown.
+
+### Known issues
+
+- Nested lists and lists containing multiple paragraphs aren't converted correctly.
+- Lists inside blockquotes aren't converted correctly.
+- Any reported [open issues here](https://github.com/nickcernis/html-to-markdown/issues?state=open).
+
+[Report your issue or request a feature here.](https://github.com/nickcernis/html2markdown/issues/new) Issues with patches or failing tests are especially welcome.
+
+### Style notes
+
+- Setext (underlined) headers are the default for H1 and H2. If you prefer the ATX style for H1 and H2 (# Header 1 and ## Header 2), set `header_style` to 'atx' in the options array when you instantiate the object:
+
+    `$markdown = new HTML_To_Markdown( $html, array('header_style'=>'atx') );`
+
+     Headers of H3 priority and lower always use atx style.
+
+- Links and images are referenced inline. Footnote references (where image src and anchor href attributes are listed in the footnotes) are not used. 
+- Blockquotes aren't line wrapped – it makes the converted Markdown easier to edit.
+
+### Dependencies
+
+HTML To Markdown requires PHP's [xml](http://www.php.net/manual/en/xml.installation.php), [lib-xml](http://www.php.net/manual/en/libxml.installation.php), and [dom](http://www.php.net/manual/en/dom.installation.php) extensions, all of which are enabled by default on most distributions.
+
+Errors such as "Fatal error: Class 'DOMDocument' not found" on distributions such as CentOS that disable PHP's xml extension can be resolved by installing php-xml.
+
+### Architecture notes
+
+HTML To Markdown is a single file that uses native DOM manipulation libraries (DOMDocument), not regex voodoo, to convert code.
+
+### Contributors
+
+Many thanks to all [contributors](https://github.com/nickcernis/html2markdown/graphs/contributors) so far. Further improvements and feature suggestions are very welcome.
+
+### How it works
+
+HTML To Markdown creates a DOMDocument from the supplied HTML, walks through the tree, and converts each node to a text node containing the equivalent markdown, starting from the most deeply nested node and working inwards towards the root node.
+
+### To-do
+
+- Support for nested lists and lists inside blockquotes.
+- Offer an option to preserve tags as HTML if they contain attributes that can't be represented with Markdown (e.g. `style`).
+
+### Trying to convert Markdown to HTML?
+
+Use [PHP Markdown](http://michelf.com/projects/php-markdown/) from Michel Fortin. No guarantees about the Elvish, though.
+

library/html-to-markdown/circle.yml

@@ -0,0 +1,4 @@
+test:
+  override:
+    - phpunit --no-configuration HTML_To_MarkdownTest ./tests/HTML_To_MarkdownTest.php
+

library/html-to-markdown/composer.json

@@ -0,0 +1,25 @@
+{
+    "name": "nickcernis/html-to-markdown",
+    "type": "library",
+    "description": "An HTML-to-markdown conversion helper for PHP",
+    "keywords": ["markdown", "html"],
+    "homepage": "https://github.com/nickcernis/html-to-markdown",
+    "license": "MIT",
+    "authors": [
+        {
+            "name": "Nick Cernis",
+            "email": "nick@cern.is",
+            "homepage": "http://modernnerd.net"
+        }
+    ],
+    "autoload": {
+        "classmap": [ "HTML_To_Markdown.php" ]
+    },
+    "require": {
+        "php": ">=5.3"
+    },
+    "require-dev": {
+        "php": ">=5.3.3",
+        "phpunit/phpunit": "4.*"
+    }
+}

library/php-markdown/License.md

@@ -0,0 +1,36 @@
+PHP Markdown Lib  
+Copyright (c) 2004-2014 Michel Fortin  
+<http://michelf.ca/>  
+All rights reserved.
+
+Based on Markdown  
+Copyright (c) 2003-2006 John Gruber   
+<http://daringfireball.net/>   
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+* Redistributions of source code must retain the above copyright notice,
+  this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer in the
+  documentation and/or other materials provided with the distribution.
+
+* Neither the name "Markdown" nor the names of its contributors may
+  be used to endorse or promote products derived from this software
+  without specific prior written permission.
+
+This software is provided by the copyright holders and contributors "as
+is" and any express or implied warranties, including, but not limited
+to, the implied warranties of merchantability and fitness for a
+particular purpose are disclaimed. In no event shall the copyright owner
+or contributors be liable for any direct, indirect, incidental, special,
+exemplary, or consequential damages (including, but not limited to,
+procurement of substitute goods or services; loss of use, data, or
+profits; or business interruption) however caused and on any theory of
+liability, whether in contract, strict liability, or tort (including
+negligence or otherwise) arising in any way out of the use of this
+software, even if advised of the possibility of such damage.

library/php-markdown/Michelf/Markdown.inc.php

@@ -0,0 +1,10 @@
+<?php
+
+# Use this file if you cannot use class autoloading. It will include all the
+# files needed for the Markdown parser.
+#
+# Take a look at the PSR-0-compatible class autoloading implementation
+# in the Readme.php file if you want a simple autoloader setup.
+
+require_once dirname(__FILE__) . '/MarkdownInterface.php';
+require_once dirname(__FILE__) . '/Markdown.php';

library/php-markdown/Michelf/MarkdownExtra.inc.php

@@ -0,0 +1,11 @@
+<?php
+
+# Use this file if you cannot use class autoloading. It will include all the
+# files needed for the MarkdownExtra parser.
+#
+# Take a look at the PSR-0-compatible class autoloading implementation
+# in the Readme.php file if you want a simple autoloader setup.
+
+require_once dirname(__FILE__) . '/MarkdownInterface.php';
+require_once dirname(__FILE__) . '/Markdown.php';
+require_once dirname(__FILE__) . '/MarkdownExtra.php';

library/php-markdown/Michelf/MarkdownExtra.php

@@ -0,0 +1,38 @@
+<?php
+#
+# Markdown Extra  -  A text-to-HTML conversion tool for web writers
+#
+# PHP Markdown Extra
+# Copyright (c) 2004-2014 Michel Fortin  
+# <http://michelf.com/projects/php-markdown/>
+#
+# Original Markdown
+# Copyright (c) 2004-2006 John Gruber  
+# <http://daringfireball.net/projects/markdown/>
+#
+namespace Michelf;
+
+
+# Just force Michelf/Markdown.php to load. This is needed to load
+# the temporary implementation class. See below for details.
+\Michelf\Markdown::MARKDOWNLIB_VERSION;
+
+#
+# Markdown Extra Parser Class
+#
+# Note: Currently the implementation resides in the temporary class
+# \Michelf\MarkdownExtra_TmpImpl (in the same file as \Michelf\Markdown).
+# This makes it easier to propagate the changes between the three different
+# packaging styles of PHP Markdown. Once this issue is resolved, the
+# _MarkdownExtra_TmpImpl will disappear and this one will contain the code.
+#
+
+class MarkdownExtra extends \Michelf\_MarkdownExtra_TmpImpl {
+
+	### Parser Implementation ###
+
+	# Temporarily, the implemenation is in the _MarkdownExtra_TmpImpl class.
+	# See note above.
+
+}
+

library/php-markdown/Michelf/MarkdownInterface.inc.php

@@ -0,0 +1,9 @@
+<?php
+
+# Use this file if you cannot use class autoloading. It will include all the
+# files needed for the MarkdownInterface interface.
+#
+# Take a look at the PSR-0-compatible class autoloading implementation
+# in the Readme.php file if you want a simple autoloader setup.
+
+require_once dirname(__FILE__) . '/MarkdownInterface.php';

library/php-markdown/Michelf/MarkdownInterface.php

@@ -0,0 +1,34 @@
+<?php
+#
+# Markdown  -  A text-to-HTML conversion tool for web writers
+#
+# PHP Markdown
+# Copyright (c) 2004-2014 Michel Fortin
+# <http://michelf.com/projects/php-markdown/>
+#
+# Original Markdown
+# Copyright (c) 2004-2006 John Gruber
+# <http://daringfireball.net/projects/markdown/>
+#
+namespace Michelf;
+
+
+#
+# Markdown Parser Interface
+#
+
+interface MarkdownInterface {
+
+  #
+  # Initialize the parser and return the result of its transform method.
+  # This will work fine for derived classes too.
+  #
+  public static function defaultTransform($text);
+
+  #
+  # Main function. Performs some preprocessing on the input text
+  # and pass it through the document gamut.
+  #
+  public function transform($text);
+
+}

library/php-markdown/Readme.md

@@ -0,0 +1,315 @@
+PHP Markdown
+============
+
+PHP Markdown Lib 1.4.1 - 4 May 2013
+
+by Michel Fortin  
+<http://michelf.ca/>
+
+based on Markdown by John Gruber  
+<http://daringfireball.net/>
+
+
+Introduction
+------------
+
+This is a library package that includes the PHP Markdown parser and its 
+sibling PHP Markdown Extra with additional features.
+
+Markdown is a text-to-HTML conversion tool for web writers. Markdown
+allows you to write using an easy-to-read, easy-to-write plain text
+format, then convert it to structurally valid XHTML (or HTML).
+
+"Markdown" is actually two things: a plain text markup syntax, and a 
+software tool, originally written in Perl, that converts the plain text 
+markup to HTML. PHP Markdown is a port to PHP of the original Markdown 
+program by John Gruber.
+
+*	[Full documentation of the Markdown syntax](<http://daringfireball.net/projects/markdown/>)
+	- Daring Fireball (John Gruber)
+*	[Markdown Extra syntax additions](<http://michelf.ca/projects/php-markdown/extra/>)
+	- Michel Fortin
+
+
+Requirement
+-----------
+
+This library package requires PHP 5.3 or later.
+
+Note: The older plugin/library hybrid package for PHP Markdown and
+PHP Markdown Extra is still maintained and will work with PHP 4.0.5 and later.
+
+Before PHP 5.3.7, pcre.backtrack_limit defaults to 100 000, which is too small
+in many situations. You might need to set it to higher values. Later PHP 
+releases defaults to 1 000 000, which is usually fine.
+
+
+Usage
+-----
+
+This library package is meant to be used with class autoloading. For autoloading 
+to work, your project needs have setup a PSR-0-compatible autoloader. See the 
+included Readme.php file for a minimal autoloader setup. (If you cannot use 
+autoloading, see below.)
+
+With class autoloading in place, putting the 'Michelf' folder in your 
+include path should be enough for this to work:
+
+	use \Michelf\Markdown;
+	$my_html = Markdown::defaultTransform($my_text);
+
+Markdown Extra syntax is also available the same way:
+
+	use \Michelf\MarkdownExtra;
+	$my_html = MarkdownExtra::defaultTransform($my_text);
+
+If you wish to use PHP Markdown with another text filter function 
+built to parse HTML, you should filter the text *after* the `transform`
+function call. This is an example with [PHP SmartyPants][psp]:
+
+	use \Michelf\Markdown, \Michelf\SmartyPants;
+	$my_html = Markdown::defaultTransform($my_text);
+	$my_html = SmartyPants::defaultTransform($my_html);
+
+All these examples are using the static `defaultTransform` static function 
+found inside the parser class. If you want to customize the parser 
+configuration, you can also instantiate it directly and change some 
+configuration variables:
+
+	use \Michelf\MarkdownExtra;
+	$parser = new MarkdownExtra;
+	$parser->fn_id_prefix = "post22-";
+	$my_html = $parser->transform($my_text);
+
+To learn more, see the full list of [configuration variables].
+
+ [configuration variables]: http://michelf.ca/projects/php-markdown/configuration/
+
+
+### Usage without an autoloader
+
+If you cannot use class autoloading, you can still use `include` or `require` 
+to access the parser. To load the `\Michelf\Markdown` parser, do it this way:
+
+	require_once 'Michelf/Markdown.inc.php';
+
+Or, if you need the `\Michelf\MarkdownExtra` parser:
+
+	require_once 'Michelf/MarkdownExtra.inc.php';
+
+While the plain `.php` files depend on autoloading to work correctly, using the
+`.inc.php` files instead will eagerly load the dependencies that would be 
+loaded on demand if you were using autoloading.
+
+
+Public API and Versioning Policy
+---------------------------------
+
+Version numbers are of the form *major*.*minor*.*patch*.
+
+The public API of PHP Markdown consist of the two parser classes `Markdown`
+and `MarkdownExtra`, their constructors, the `transform` and `defaultTransform`
+functions and their configuration variables. The public API is stable for
+a given major version number. It might get additions when the minor version
+number increments.
+
+**Protected members are not considered public API.** This is unconventional 
+and deserves an explanation. Incrementing the major version number every time 
+the underlying implementation of something changes is going to give
+nonessential version numbers for the vast majority of people who just use the
+parser.  Protected members are meant to create parser subclasses that behave in
+different ways. Very few people create parser subclasses. I don't want to 
+discourage it by making everything private, but at the same time I can't 
+guarantee any stable hook between versions if you use protected members.
+
+**Syntax changes** will increment the minor number for new features, and the 
+patch number for small corrections. A *new feature* is something that needs a 
+change in the syntax documentation. Note that since PHP Markdown Lib includes
+two parsers, a syntax change for either of them will increment the minor 
+number. Also note that there is nothing perfectly backward-compatible with the
+Markdown syntax: all inputs are always valid, so new features always replace
+something that was previously legal, although generally nonsensical to do.
+
+
+Bugs
+----
+
+To file bug reports please send email to:
+<michel.fortin@michelf.ca>
+
+Please include with your report: (1) the example input; (2) the output you
+expected; (3) the output PHP Markdown actually produced.
+
+If you have a problem where Markdown gives you an empty result, first check 
+that the backtrack limit is not too low by running `php --info | grep pcre`.
+See Installation and Requirement above for details.
+
+
+Development and Testing
+-----------------------
+
+Pull requests for fixing bugs are welcome. Proposed new features are
+going meticulously reviewed -- taking into account backward compatibility, 
+potential side effects, and future extensibility -- before deciding on
+acceptance or rejection.
+
+If you make a pull request that includes changes to the parser please add 
+tests for what is being changed to [MDTest][] and make a pull request there 
+too.
+
+ [MDTest]: https://github.com/michelf/mdtest/
+
+
+Donations
+---------
+
+If you wish to make a donation that will help me devote more time to 
+PHP Markdown, please visit [michelf.ca/donate] or send Bitcoin to
+[1HiuX34czvVPPdhXbUAsAu7pZcesniDCGH].
+
+ [michelf.ca/donate]: https://michelf.ca/donate/#!Thanks%20for%20PHP%20Markdown
+ [1HiuX34czvVPPdhXbUAsAu7pZcesniDCGH]: bitcoin:1HiuX34czvVPPdhXbUAsAu7pZcesniDCGH
+
+
+Version History
+---------------
+
+Unreleased
+
+*	Added the ability to insert custom HTML attributes everywhere an extra
+	attribute block is allowed (links, images, headers). Credits to
+	Peter Droogmans for providing the implementation.
+
+*	Added a `url_filter_func` configuration variable which takes a function
+	that can rewrite any link or image URL to something different.
+
+
+PHP Markdown Lib 1.4.1 (4 May 2014)
+
+*	The HTML block parser will now treat `<figure>` as a block-level element
+	(as it should) and no longer wrap it in `<p>` or parse it's content with 
+	the as Markdown syntax (although with Extra you can use `markdown="1"`
+	if you wish to use the Markdown syntax inside it).
+
+*	The content of `<style>` elements will now be left alone, its content
+	won't be interpreted as Markdown.
+
+*	Corrected an bug where some inline links with spaces in them would not
+	work even when surounded with angle brackets:
+	
+		[link](<s p a c e s>)
+
+*	Fixed an issue where email addresses with quotes in them would not always
+	have the quotes escaped in the link attribute, causing broken links (and
+	invalid HTML).
+
+*	Fixed the case were a link definition following a footnote definition would
+	be swallowed by the footnote unless it was separated by a blank line.
+
+
+PHP Markdown Lib 1.4.0 (29 Nov 2013)
+
+*	Added support for the `tel:` URL scheme in automatic links.
+
+		<tel:+1-111-111-1111>
+	
+	It gets converted to this (note the `tel:` prefix becomes invisible):
+	
+		<a href="tel:+1-111-111-1111">+1-111-111-1111</a>
+
+*	Added backtick fenced code blocks to MarkdownExtra, originally from
+	Github-Flavored Markdown.
+
+*	Added an interface called MarkdownInterface implemented by both
+	the Markdown and MarkdownExtra parsers. You can use the interface if
+	you want to create a mockup parser object for unit testing.
+
+*	For those of you who cannot use class autoloading, you can now
+	include `Michelf/Markdown.inc.php` or `Michelf/MarkdownExtra.inc.php` (note 
+	the 	`.inc.php` extension) to automatically include other files required
+	by the parser.
+
+
+PHP Markdown Lib 1.3 (11 Apr 2013)
+
+This is the first release of PHP Markdown Lib. This package requires PHP 
+version 5.3 or later and is designed to work with PSR-0 autoloading and, 
+optionally with Composer. Here is a list of the changes since 
+PHP Markdown Extra 1.2.6:
+
+*	Plugin interface for WordPress and other systems is no longer present in
+	the Lib package. The classic package is still available if you need it:
+	<http://michelf.ca/projects/php-markdown/classic/>
+
+*	Added `public` and `protected` protection attributes, plus a section about
+	what is "public API" and what isn't in the Readme file.
+
+*	Changed HTML output for footnotes: now instead of adding `rel` and `rev`
+	attributes, footnotes links have the class name `footnote-ref` and
+	backlinks `footnote-backref`.
+
+*	Fixed some regular expressions to make PCRE not shout warnings about POSIX
+	collation classes (dependent on your version of PCRE).
+
+*	Added optional class and id attributes to images and links using the same
+	syntax as for headers:
+
+		[link](url){#id .class}  
+		![img](url){#id .class}
+	
+	It work too for reference-style links and images. In this case you need
+	to put those attributes at the reference definition:
+
+		[link][linkref] or [linkref]  
+		![img][linkref]
+		
+		[linkref]: url "optional title" {#id .class}
+
+*	Fixed a PHP notice message triggered when some table column separator 
+	markers are missing on the separator line below column headers.
+
+*	Fixed a small mistake that could cause the parser to retain an invalid
+	state related to parsing links across multiple runs. This was never 
+	observed (that I know of), but it's still worth fixing.
+
+
+Copyright and License
+---------------------
+
+PHP Markdown Lib
+Copyright (c) 2004-2014 Michel Fortin
+<http://michelf.ca/>  
+All rights reserved.
+
+Based on Markdown  
+Copyright (c) 2003-2005 John Gruber   
+<http://daringfireball.net/>   
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+*   Redistributions of source code must retain the above copyright 
+    notice, this list of conditions and the following disclaimer.
+
+*   Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the 
+    distribution.
+
+*   Neither the name "Markdown" nor the names of its contributors may
+    be used to endorse or promote products derived from this software
+    without specific prior written permission.
+
+This software is provided by the copyright holders and contributors "as
+is" and any express or implied warranties, including, but not limited
+to, the implied warranties of merchantability and fitness for a
+particular purpose are disclaimed. In no event shall the copyright owner
+or contributors be liable for any direct, indirect, incidental, special,
+exemplary, or consequential damages (including, but not limited to,
+procurement of substitute goods or services; loss of use, data, or
+profits; or business interruption) however caused and on any theory of
+liability, whether in contract, strict liability, or tort (including
+negligence or otherwise) arising in any way out of the use of this
+software, even if advised of the possibility of such damage.

library/php-markdown/Readme.php

@@ -0,0 +1,31 @@
+<?php
+
+# This file passes the content of the Readme.md file in the same directory
+# through the Markdown filter. You can adapt this sample code in any way
+# you like.
+
+# Install PSR-0-compatible class autoloader
+spl_autoload_register(function($class){
+	require preg_replace('{\\\\|_(?!.*\\\\)}', DIRECTORY_SEPARATOR, ltrim($class, '\\')).'.php';
+});
+
+# Get Markdown class
+use \Michelf\Markdown;
+
+# Read file and pass content through the Markdown parser
+$text = file_get_contents('Readme.md');
+$html = Markdown::defaultTransform($text);
+
+?>
+<!DOCTYPE html>
+<html>
+    <head>
+        <title>PHP Markdown Lib - Readme</title>
+    </head>
+    <body>
+		<?php
+			# Put HTML content in the document
+			echo $html;
+		?>
+    </body>
+</html>

library/php-markdown/composer.json

@@ -0,0 +1,31 @@
+{
+	"name": "michelf/php-markdown",
+	"type": "library",
+	"description": "PHP Markdown",
+	"homepage": "http://michelf.ca/projects/php-markdown/",
+	"keywords": ["markdown"],
+	"license": "BSD-3-Clause",
+	"authors": [
+		{
+			"name": "Michel Fortin",
+			"email": "michel.fortin@michelf.ca",
+			"homepage": "http://michelf.ca/",
+			"role": "Developer"
+		},
+		{
+			"name": "John Gruber",
+			"homepage": "http://daringfireball.net/"
+		}
+	],
+	"require": {
+		"php": ">=5.3.0"
+	},
+	"autoload": {
+ 		"psr-0": { "Michelf": "" }
+	},
+	"extra": {
+		"branch-alias": {
+			"dev-lib": "1.4.x-dev"
+		}
+	}
+}

mod/admin.php

@@ -121,6 +121,8 @@ function admin_content(&$a) {
 	}
 
 	$aside['logs'] = Array($a->get_baseurl(true)."/admin/logs/", t("Logs"), "logs");
+	$aside['diagnostics_probe'] = Array($a->get_baseurl(true).'/probe/', t('probe address'), 'probe');
+	$aside['diagnostics_webfinger'] = Array($a->get_baseurl(true).'/webfinger/', t('check webfinger'), 'webfinger');
 
 	$t = get_markup_template("admin_aside.tpl");
 	$a->page['aside'] .= replace_macros( $t, array(
@@ -128,6 +130,7 @@ function admin_content(&$a) {
 			'$admtxt' => t('Admin'),
 			'$plugadmtxt' => t('Plugin Features'),
 			'$logtxt' => t('Logs'),
+			'$diagnosticstxt' => t('diagnostics'),
 			'$h_pending' => t('User registrations waiting for confirmation'),
 			'$admurl'=> $a->get_baseurl(true)."/admin/"
 	));
@@ -278,7 +281,7 @@ function admin_page_site_post(&$a){
 			$q = sprintf("UPDATE %s SET %s;", $table_name, $upds);
 			$r = q($q);
 			if (!$r) {
-				notice( "Falied updating '$table_name': " . $db->error );
+				notice( "Failed updating '$table_name': " . $db->error );
 				goaway($a->get_baseurl(true) . '/admin/site' );
 			}
 		}
@@ -309,7 +312,10 @@ function admin_page_site_post(&$a){
 
 	$sitename 		=	((x($_POST,'sitename'))			? notags(trim($_POST['sitename']))		: '');
 	$hostname 		=	((x($_POST,'hostname'))			? notags(trim($_POST['hostname']))		: '');
+	$sender_email		=	((x($_POST,'sender_email'))		? notags(trim($_POST['sender_email']))		: '');
 	$banner			=	((x($_POST,'banner'))      		? trim($_POST['banner'])			: false);
+	$shortcut_icon 		=	((x($_POST,'shortcut_icon'))		? notags(trim($_POST['shortcut_icon']))		: '');
+	$touch_icon 		=	((x($_POST,'touch_icon'))		? notags(trim($_POST['touch_icon']))		: '');
 	$info			=	((x($_POST,'info'))      		? trim($_POST['info'])			: false);
 	$language		=	((x($_POST,'language'))			? notags(trim($_POST['language']))		: '');
 	$theme			=	((x($_POST,'theme'))			? notags(trim($_POST['theme']))			: '');
@@ -341,7 +347,8 @@ function admin_page_site_post(&$a){
 	$no_openid		=	!((x($_POST,'no_openid'))		? True						: False);
 	$no_regfullname		=	!((x($_POST,'no_regfullname'))		? True						: False);
 	$no_utf			=	!((x($_POST,'no_utf'))			? True						: False);
-	$no_community_page	=	!((x($_POST,'no_community_page'))	? True						: False);
+	$community_page_style	=	((x($_POST,'community_page_style'))	? intval(trim($_POST['community_page_style']))	: 0);
+	$max_author_posts_community_page	=	((x($_POST,'max_author_posts_community_page'))	? intval(trim($_POST['max_author_posts_community_page']))	: 0);
 
 	$verifyssl		=	((x($_POST,'verifyssl'))		? True						: False);
 	$proxyuser		=	((x($_POST,'proxyuser'))		? notags(trim($_POST['proxyuser']))		: '');
@@ -352,13 +359,14 @@ function admin_page_site_post(&$a){
 	$maxloadavg		=	((x($_POST,'maxloadavg'))		? intval(trim($_POST['maxloadavg']))		: 50);
 	$dfrn_only		=	((x($_POST,'dfrn_only'))		? True						: False);
 	$ostatus_disabled	=	!((x($_POST,'ostatus_disabled'))	? True  					: False);
-	$ostatus_poll_interval	=	((x($_POST,'ostatus_poll_interval'))	? intval(trim($_POST['ostatus_poll_interval']))		:  0);
+	$ostatus_poll_interval	=	((x($_POST,'ostatus_poll_interval'))	? intval(trim($_POST['ostatus_poll_interval']))	:  0);
 	$diaspora_enabled	=	((x($_POST,'diaspora_enabled'))		? True   					: False);
 	$ssl_policy		=	((x($_POST,'ssl_policy'))		? intval($_POST['ssl_policy']) 			: 0);
 	$force_ssl		=	((x($_POST,'force_ssl'))		? True   					: False);
 	$old_share		=	((x($_POST,'old_share'))		? True   					: False);
 	$hide_help		=	((x($_POST,'hide_help'))		? True   					: False);
 	$suppress_language	=	((x($_POST,'suppress_language'))	? True   					: False);
+	$suppress_tags		=	((x($_POST,'suppress_tags'))		? True   					: False);
 	$use_fulltext_engine	=	((x($_POST,'use_fulltext_engine'))	? True   					: False);
 	$itemcache		=	((x($_POST,'itemcache'))		? notags(trim($_POST['itemcache']))		: '');
 	$itemcache_duration	=	((x($_POST,'itemcache_duration'))	? intval($_POST['itemcache_duration'])		: 0);
@@ -368,7 +376,9 @@ function admin_page_site_post(&$a){
 	$basepath		=	((x($_POST,'basepath'))			? notags(trim($_POST['basepath']))		: '');
 	$singleuser		=	((x($_POST,'singleuser'))		? notags(trim($_POST['singleuser']))		: '');
 	$proxy_disabled		=	((x($_POST,'proxy_disabled'))		? True						: False);
-	$disable_noscrape = ((x($_POST,'disable_noscrape')) ? true : false);
+	$old_pager		=	((x($_POST,'old_pager'))		? True						: False);
+	$only_tag_search	=	((x($_POST,'only_tag_search'))		? True						: False);
+
 	if($ssl_policy != intval(get_config('system','ssl_policy'))) {
 		if($ssl_policy == SSL_POLICY_FULL) {
 			q("update `contact` set
@@ -415,7 +425,11 @@ function admin_page_site_post(&$a){
 	set_config('system','maxloadavg',$maxloadavg);
 	set_config('config','sitename',$sitename);
 	set_config('config','hostname',$hostname);
+	set_config('config','sender_email', $sender_email);
 	set_config('system','suppress_language',$suppress_language);
+	set_config('system','suppress_tags',$suppress_tags);
+	set_config('system','shortcut_icon',$shortcut_icon);
+	set_config('system','touch_icon',$touch_icon);
 	if ($banner==""){
 		// don't know why, but del_config doesn't work...
 		q("DELETE FROM `config` WHERE `cat` = '%s' AND `k` = '%s' LIMIT 1",
@@ -472,7 +486,8 @@ function admin_page_site_post(&$a){
 	set_config('system','block_extended_register', $no_multi_reg);
 	set_config('system','no_openid', $no_openid);
 	set_config('system','no_regfullname', $no_regfullname);
-	set_config('system','no_community_page', $no_community_page);
+	set_config('system','community_page_style', $community_page_style);
+	set_config('system','max_author_posts_community_page', $max_author_posts_community_page);
 	set_config('system','no_utf', $no_utf);
 	set_config('system','verifyssl', $verifyssl);
 	set_config('system','proxyuser', $proxyuser);
@@ -480,7 +495,7 @@ function admin_page_site_post(&$a){
 	set_config('system','curl_timeout', $timeout);
 	set_config('system','dfrn_only', $dfrn_only);
 	set_config('system','ostatus_disabled', $ostatus_disabled);
-		set_config('system','ostatus_poll_interval', $ostatus_poll_interval);
+	set_config('system','ostatus_poll_interval', $ostatus_poll_interval);
 	set_config('system','diaspora_enabled', $diaspora_enabled);
 	set_config('config','private_addons', $private_addons);
 
@@ -495,7 +510,8 @@ function admin_page_site_post(&$a){
 	set_config('system','temppath', $temppath);
 	set_config('system','basepath', $basepath);
 	set_config('system','proxy_disabled', $proxy_disabled);
-	set_config('system','disable_noscrape', $disable_noscrape);
+	set_config('system','old_pager', $old_pager);
+	set_config('system','only_tag_search', $only_tag_search);
 
 	info( t('Site settings updated.') . EOL);
 	goaway($a->get_baseurl(true) . '/admin/site' );
@@ -541,14 +557,21 @@ function admin_page_site(&$a) {
 		}
 		}
 
+		/* Community page style */
+		$community_page_style_choices = array(
+			CP_NO_COMMUNITY_PAGE => t("No community page"),
+			CP_USERS_ON_SERVER => t("Public postings from users of this site"),
+			CP_GLOBAL_COMMUNITY => t("Global community page")
+			);
+
 		/* OStatus conversation poll choices */
 		$ostatus_poll_choices = array(
-		"-2" => t("Never"),
+			"-2" => t("Never"),
-		"-1" => t("At post arrival"),
+			"-1" => t("At post arrival"),
-		"0" => t("Frequently"),
+			"0" => t("Frequently"),
-		"60" => t("Hourly"),
+			"60" => t("Hourly"),
-		"720" => t("Twice daily"),
+			"720" => t("Twice daily"),
-		"1440" => t("Daily")
+			"1440" => t("Daily")
 			);
 
 		/* get user names to make the install a personal install of X */
@@ -605,7 +628,10 @@ function admin_page_site(&$a) {
 		// name, label, value, help string, extra data...
 		'$sitename' 		=> array('sitename', t("Site name"), htmlentities($a->config['sitename'], ENT_QUOTES), 'UTF-8'),
 		'$hostname' 		=> array('hostname', t("Host name"), $a->config['hostname'], ""),
+		'$sender_email'		=> array('sender_email', t("Sender Email"), $a->config['sender_email'], "The email address your server shall use to send notification emails from.", "", "", "email"),
 		'$banner'		=> array('banner', t("Banner/Logo"), $banner, ""),
+		'$shortcut_icon'	=> array('shortcut_icon', t("Shortcut icon"), get_config('system','shortcut_icon'),  "Link to an icon that will be used for browsers."),
+		'$touch_icon'		=> array('touch_icon', t("Touch icon"), get_config('system','touch_icon'),  "Link to an icon that will be used for tablets and mobiles."),
 		'$info'	=> array('info',t('Additional Info'), $info, t('For public servers: you can add additional information here that will be listed at dir.friendica.com/siteinfo.')),
 		'$language' 		=> array('language', t("System language"), get_config('system','language'), "", $lang_choices),
 		'$theme' 		=> array('theme', t("System theme"), get_config('system','theme'), t("Default system theme - may be over-ridden by user profiles - <a href='#' id='cnftheme'>change theme settings</a>"), $theme_choices),
@@ -638,7 +664,8 @@ function admin_page_site(&$a) {
 		'$no_openid'		=> array('no_openid', t("OpenID support"), !get_config('system','no_openid'), t("OpenID support for registration and logins.")),
 		'$no_regfullname'	=> array('no_regfullname', t("Fullname check"), !get_config('system','no_regfullname'), t("Force users to register with a space between firstname and lastname in Full name, as an antispam measure")),
 		'$no_utf'		=> array('no_utf', t("UTF-8 Regular expressions"), !get_config('system','no_utf'), t("Use PHP UTF8 regular expressions")),
-		'$no_community_page' 	=> array('no_community_page', t("Show Community Page"), !get_config('system','no_community_page'), t("Display a Community page showing all recent public postings on this site.")),
+		'$community_page_style' => array('community_page_style', t("Community Page Style"), get_config('system','community_page_style'), t("Type of community page to show. 'Global community' shows every public posting from an open distributed network that arrived on this server."), $community_page_style_choices),
+		'$max_author_posts_community_page' => array('max_author_posts_community_page', t("Posts per user on community page"), get_config('system','max_author_posts_community_page'), t("The maximum number of posts per user on the community page. (Not valid for 'Global Community')")),
 		'$ostatus_disabled' 	=> array('ostatus_disabled', t("Enable OStatus support"), !get_config('system','ostatus_disabled'), t("Provide built-in OStatus \x28StatusNet, GNU Social etc.\x29 compatibility. All communications in OStatus are public, so privacy warnings will be occasionally displayed.")),
 		'$ostatus_poll_interval'	=> array('ostatus_poll_interval', t("OStatus conversation completion interval"), (string) intval(get_config('system','ostatus_poll_interval')), t("How often shall the poller check for new entries in OStatus conversations? This can be a very ressource task."), $ostatus_poll_choices),
 		'$diaspora_enabled' 	=> array('diaspora_enabled', t("Enable Diaspora support"), get_config('system','diaspora_enabled'), t("Provide built-in Diaspora network compatibility.")),
@@ -653,6 +680,7 @@ function admin_page_site(&$a) {
 
 		'$use_fulltext_engine'	=> array('use_fulltext_engine', t("Use MySQL full text engine"), get_config('system','use_fulltext_engine'), t("Activates the full text engine. Speeds up search - but can only search for four and more characters.")),
 		'$suppress_language'	=> array('suppress_language', t("Suppress Language"), get_config('system','suppress_language'), t("Suppress language information in meta information about a posting.")),
+		'$suppress_tags'	=> array('suppress_tags', t("Suppress Tags"), get_config('system','suppress_tags'), t("Suppress showing a list of hashtags at the end of the posting.")),
 		'$itemcache'		=> array('itemcache', t("Path to item cache"), get_config('system','itemcache'), "The item caches buffers generated bbcode and external images."),
 		'$itemcache_duration' 	=> array('itemcache_duration', t("Cache duration in seconds"), get_config('system','itemcache_duration'), t("How long should the cache files be hold? Default value is 86400 seconds (One day). To disable the item cache, set the value to -1.")),
 		'$max_comments' 	=> array('max_comments', t("Maximum numbers of comments per post"), get_config('system','max_comments'), t("How much comments should be shown for each post? Default value is 100.")),
@@ -660,10 +688,10 @@ function admin_page_site(&$a) {
 		'$temppath'		=> array('temppath', t("Temp path"), get_config('system','temppath'), "If you have a restricted system where the webserver can't access the system temp path, enter another path here."),
 		'$basepath'		=> array('basepath', t("Base path to installation"), get_config('system','basepath'), "If the system cannot detect the correct path to your installation, enter the correct path here. This setting should only be set if you are using a restricted system and symbolic links to your webroot."),
 		'$proxy_disabled'	=> array('proxy_disabled', t("Disable picture proxy"), get_config('system','proxy_disabled'), t("The picture proxy increases performance and privacy. It shouldn't be used on systems with very low bandwith.")),
+		'$old_pager'		=> array('old_pager', t("Enable old style pager"), get_config('system','old_pager'), t("The old style pager has page numbers but slows down massively the page speed.")),
+		'$only_tag_search'	=> array('only_tag_search', t("Only search in tags"), get_config('system','only_tag_search'), t("On large systems the text search can slow down the system extremely.")),
 
 		'$relocate_url'     => array('relocate_url', t("New base url"), $a->get_baseurl(), "Change base url for this server. Sends relocate message to all DFRN contacts of all users."),
-
-		'$disable_noscrape'=> array('disable_noscrape', t("Disable noscrape"), get_config('system','disable_noscrape'), t("The noscrape feature speeds up directory submissions by using JSON data instead of HTML scraping. Disabling it will cause higher load on your server and the directory server.")),
 	'$form_security_token' => get_form_security_token("admin_site")
 
 	));
@@ -1000,7 +1028,7 @@ function admin_page_users(&$a){
 		'$users' => $users,
 		'$newusername'  => array('new_user_name', t("Name"), '', t("Name of the new user.")),
 		'$newusernickname'  => array('new_user_nickname', t("Nickname"), '', t("Nickname of the new user.")),
-		'$newuseremail'  => array('new_user_email', t("Email"), '', t("Email address of the new user.")),
+		'$newuseremail'  => array('new_user_email', t("Email"), '', t("Email address of the new user."), '', '', 'email'),
 	));
 	$o .= paginate($a);
 	return $o;

mod/bookmarklet.php

@@ -0,0 +1,46 @@
+<?php
+
+require_once('include/conversation.php');
+require_once('include/items.php');
+
+function bookmarklet_init(&$a) {
+	$_GET["mode"] = "minimal";
+}
+
+function bookmarklet_content(&$a) {
+	if(!local_user()) {
+		$o = '<h2>'.t('Login').'</h2>';
+		$o .= login(($a->config['register_policy'] == REGISTER_CLOSED) ? false : true);
+		return $o;
+	}
+
+	$referer = normalise_link($_SERVER["HTTP_REFERER"]);
+	$page = normalise_link($a->get_baseurl()."/bookmarklet");
+
+	if (!strstr($referer, $page)) {
+		$content = add_page_info($_REQUEST["url"]);
+
+		$x = array(
+			'is_owner' => true,
+			'allow_location' => $a->user['allow_location'],
+			'default_location' => $a->user['default-location'],
+			'nickname' => $a->user['nickname'],
+			'lockstate' => ((is_array($a->user) && ((strlen($a->user['allow_cid'])) || (strlen($a->user['allow_gid'])) || (strlen($a->user['deny_cid'])) || (strlen($a->user['deny_gid'])))) ? 'lock' : 'unlock'),
+			'default_perms' => get_acl_permissions($a->user),
+			'acl' => populate_acl($a->user, $celeb),
+			'bang' => '',
+			'visitor' => 'block',
+			'profile_uid' => local_user(),
+			'acl_data' => construct_acl_data($a, $a->user), // For non-Javascript ACL selector
+			'title' => trim($_REQUEST["title"], "*"),
+			'content' => $content
+		);
+		$o = status_editor($a,$x, 0, false);
+		$o .= "<script>window.resizeTo(800,550);</script>";
+	} else {
+		$o = '<h2>'.t('The post was created').'</h2>';
+		$o .= "<script>window.close()</script>";
+	}
+
+	return $o;
+}

mod/community.php

@@ -19,7 +19,7 @@ function community_content(&$a, $update = 0) {
 		return;
 	}
 
-	if(get_config('system','no_community_page')) {
+	if(get_config('system','community_page_style') == CP_NO_COMMUNITY_PAGE) {
 		notice( t('Not available.') . EOL);
 		return;
 	}
@@ -44,7 +44,7 @@ function community_content(&$a, $update = 0) {
 	// Only public posts can be shown
 	// OR your own posts if you are a logged in member
 
-	if( (! get_config('alt_pager', 'global')) && (! get_pconfig(local_user(),'system','alt_pager')) ) {
+	if(get_config('system', 'old_pager')) {
 		$r = q("SELECT COUNT(distinct(`item`.`uri`)) AS `total`
 			FROM `item` INNER JOIN `contact` ON `contact`.`id` = `item`.`contact-id`
 			AND `contact`.`blocked` = 0 AND `contact`.`pending` = 0
@@ -103,10 +103,9 @@ function community_content(&$a, $update = 0) {
 
 	$o .= conversation($a,$s,'community',$update);
 
-	if(get_config('alt_pager', 'global') || get_pconfig(local_user(),'system','alt_pager') ) {
+	if(!get_config('system', 'old_pager')) {
 	        $o .= alt_pager($a,count($r));
-	}
+	} else {
-	else {
 	        $o .= paginate($a);
 	}
 
@@ -114,6 +113,9 @@ function community_content(&$a, $update = 0) {
 }
 
 function community_getitems($start, $itemspage) {
+	if (get_config('system','community_page_style') == CP_GLOBAL_COMMUNITY)
+		return(community_getpublicitems($start, $itemspage));
+
 	$r = q("SELECT `item`.`uri`, `item`.*, `item`.`id` AS `item_id`,
 		`contact`.`name`, `contact`.`photo`, `contact`.`url`, `contact`.`alias`, `contact`.`rel`,
 		`contact`.`network`, `contact`.`thumb`, `contact`.`self`, `contact`.`writable`,
@@ -136,3 +138,19 @@ function community_getitems($start, $itemspage) {
 	return($r);
 
 }
+
+function community_getpublicitems($start, $itemspage) {
+	$r = q("SELECT `item`.`uri`, `item`.*, `item`.`id` AS `item_id`,
+			`author-name` AS `name`, `owner-avatar` AS `photo`,
+			`owner-link` AS `url`, `owner-avatar` AS `thumb`
+		FROM `item` WHERE `item`.`uid` = 0 AND `item`.`id` = `item`.`parent`
+		AND `item`.`allow_cid` = '' AND `item`.`allow_gid` = ''
+		AND `item`.`deny_cid` = '' AND `item`.`deny_gid` = ''
+		ORDER BY `item`.`received` DESC LIMIT %d, %d",
+		intval($start),
+		intval($itemspage)
+	);
+
+	return($r);
+
+}

mod/contacts.php

@@ -3,6 +3,7 @@
 require_once('include/Contact.php');
 require_once('include/socgraph.php');
 require_once('include/contact_selectors.php');
+require_once('mod/proxy.php');
 
 function contacts_init(&$a) {
 	if(! local_user())
@@ -31,7 +32,8 @@ function contacts_init(&$a) {
 			$a->data['contact'] = $r[0];
 			$vcard_widget = replace_macros(get_markup_template("vcard-widget.tpl"),array(
 				'$name' => $a->data['contact']['name'],
-				'$photo' => $a->data['contact']['photo']
+				'$photo' => $a->data['contact']['photo'],
+			        '$url' => ($a->data['contact']['network'] == 'dfrn') ? $a->get_baseurl()."/redir/".$a->data['contact']['id'] : $a->data['contact']['url']
 			));
 			$follow_widget = '';
 	}
@@ -43,6 +45,9 @@ function contacts_init(&$a) {
 			$follow_widget = follow_widget();
 	}
 
+	if ($_GET['nets'] == "all")
+		$_GET['nets'] = "";
+
 	$groups_widget .= group_side('contacts','group',false,0,$contact_id);
 	$findpeople_widget .= findpeople_widget();
 	$networks_widget .= networks_widget('contacts',$_GET['nets']);
@@ -73,12 +78,12 @@ function contacts_init(&$a) {
 function contacts_batch_actions(&$a){
 	$contacts_id = $_POST['contact_batch'];
 	if (!is_array($contacts_id)) return;
-	
+
 	$orig_records = q("SELECT * FROM `contact` WHERE `id` IN (%s) AND `uid` = %d AND `self` = 0",
 		implode(",", $contacts_id),
 		intval(local_user())
 	);
-	
+
 	$count_actions=0;
 	foreach($orig_records as $orig_record) {
 		$contact_id = $orig_record['id'];
@@ -106,7 +111,7 @@ function contacts_batch_actions(&$a){
 	if ($count_actions>0) {
 		info ( sprintf( tt("%d contact edited.", "%d contacts edited", $count_actions), $count_actions) );
 	}
-	
+
 	if(x($_SESSION,'return_url'))
 		goaway($a->get_baseurl(true) . '/' . $_SESSION['return_url']);
 	else
@@ -401,36 +406,39 @@ function contacts_content(&$a) {
 				break;
 		}
 
+		if(!in_array($contact['network'], array(NETWORK_DFRN, NETWORK_OSTATUS, NETWORK_DIASPORA)))
+				$relation_text = "";
+
 		$relation_text = sprintf($relation_text,$contact['name']);
 
 		if(($contact['network'] === NETWORK_DFRN) && ($contact['rel'])) {
 			$url = "redir/{$contact['id']}";
 			$sparkle = ' class="sparkle" ';
 		}
-		else { 
+		else {
 			$url = $contact['url'];
 			$sparkle = '';
 		}
 
 		$insecure = t('Private communications are not available for this contact.');
 
-		$last_update = (($contact['last-update'] == '0000-00-00 00:00:00') 
+		$last_update = (($contact['last-update'] == '0000-00-00 00:00:00')
-				? t('Never') 
+				? t('Never')
 				: datetime_convert('UTC',date_default_timezone_get(),$contact['last-update'],'D, j M Y, g:i A'));
 
 		if($contact['last-update'] !== '0000-00-00 00:00:00')
-			$last_update .= ' ' . (($contact['last-update'] == $contact['success_update']) ? t("\x28Update was successful\x29") : t("\x28Update was not successful\x29"));
+			$last_update .= ' ' . (($contact['last-update'] <= $contact['success_update']) ? t("\x28Update was successful\x29") : t("\x28Update was not successful\x29"));
 
 		$lblsuggest = (($contact['network'] === NETWORK_DFRN) ? t('Suggest friends') : '');
 
-		$poll_enabled = (($contact['network'] !== NETWORK_DIASPORA) ? true : false);
+		$poll_enabled = in_array($contact['network'], array(NETWORK_DFRN, NETWORK_OSTATUS, NETWORK_FEED, NETWORK_MAIL, NETWORK_MAIL2));
 
 		$nettype = sprintf( t('Network type: %s'),network_to_name($contact['network']));
 
 		$common = count_common_friends(local_user(),$contact['id']);
 		$common_text = (($common) ? sprintf( tt('%d contact in common','%d contacts in common', $common),$common) : '');
 
-		$polling = (($contact['network'] === NETWORK_MAIL | $contact['network'] === NETWORK_FEED) ? 'polling' : ''); 
+		$polling = (($contact['network'] === NETWORK_MAIL | $contact['network'] === NETWORK_FEED) ? 'polling' : '');
 
 		$x = count_all_friends(local_user(), $contact['id']);
 		$all_friends = (($x) ? t('View all contacts') : '');
@@ -468,6 +476,16 @@ function contacts_content(&$a) {
 
 		$lost_contact = (($contact['archive'] && $contact['term-date'] != '0000-00-00 00:00:00' && $contact['term-date'] < datetime_convert('','','now')) ? t('Communications lost with this contact!') : '');
 
+		if ($contact['network'] == NETWORK_FEED)
+			$fetch_further_information = array('fetch_further_information', t('Fetch further information for feeds'), $contact['fetch_further_information'], t('Fetch further information for feeds'),
+									array('0'=>t('Disabled'), '1'=>t('Fetch information'), '2'=>t('Fetch information and keywords')));
+
+		if (in_array($contact['network'], array(NETWORK_FEED, NETWORK_MAIL, NETWORK_MAIL2)))
+			$poll_interval = contact_poll_interval($contact['priority'],(! $poll_enabled));
+
+		if ($contact['network'] == NETWORK_DFRN)
+			$profile_select = contact_profile_assign($contact['profile-id'],(($contact['network'] !== NETWORK_DFRN) ? true : false));
+
 		$o .= replace_macros($tpl, array(
 			'$header' => t('Contact Editor'),
 			'$tab_str' => $tab_str,
@@ -488,14 +506,14 @@ function contacts_content(&$a) {
 			'$lblsuggest' => $lblsuggest,
 			'$delete' => t('Delete contact'),
 			'$nettype' => $nettype,
-			'$poll_interval' => contact_poll_interval($contact['priority'],(! $poll_enabled)),
+			'$poll_interval' => $poll_interval,
 			'$poll_enabled' => $poll_enabled,
 			'$lastupdtext' => t('Last update:'),
 			'$lost_contact' => $lost_contact,
 			'$updpub' => t('Update public posts'),
 			'$last_update' => $last_update,
 			'$udnow' => t('Update now'),
-			'$profile_select' => contact_profile_assign($contact['profile-id'],(($contact['network'] !== NETWORK_DFRN) ? true : false)),
+			'$profile_select' => $profile_select,
 			'$contact_id' => $contact['id'],
 			'$block_text' => (($contact['blocked']) ? t('Unblock') : t('Block') ),
 			'$ignore_text' => (($contact['readonly']) ? t('Unignore') : t('Ignore') ),
@@ -506,8 +524,7 @@ function contacts_content(&$a) {
 			'$archived' => (($contact['archive']) ? t('Currently archived') : ''),
 			'$hidden' => array('hidden', t('Hide this contact from others'), ($contact['hidden'] == 1), t('Replies/likes to your public posts <strong>may</strong> still be visible')),
 			'$notify' => array('notify', t('Notification for new posts'), ($contact['notify_new_posts'] == 1), t('Send a notification of every new post of this contact')),
-			'$fetch_further_information' => array('fetch_further_information', t('Fetch further information for feeds'), $contact['fetch_further_information'], t('Fetch further information for feeds'),
+			'$fetch_further_information' => $fetch_further_information,
-								array('0'=>t('Disabled'), '1'=>t('Fetch information'), '2'=>t('Fetch information and keywords'))),
 			'$ffi_keyword_blacklist' => $contact['ffi_keyword_blacklist'],
 			'$ffi_keyword_blacklist' => array('ffi_keyword_blacklist', t('Blacklisted keywords'), $contact['ffi_keyword_blacklist'], t('Comma separated list of keywords that should not be converted to hashtags, when "Fetch information and keywords" is selected')),
 			'$photo' => $contact['photo'],
@@ -623,11 +640,11 @@ function contacts_content(&$a) {
 
 	if($nets)
 		$sql_extra .= sprintf(" AND network = '%s' ", dbesc($nets));
- 
-	$sql_extra2 = ((($sort_type > 0) && ($sort_type <= CONTACT_IS_FRIEND)) ? sprintf(" AND `rel` = %d ",intval($sort_type)) : ''); 
 
-	
+	$sql_extra2 = ((($sort_type > 0) && ($sort_type <= CONTACT_IS_FRIEND)) ? sprintf(" AND `rel` = %d ",intval($sort_type)) : '');
-	$r = q("SELECT COUNT(*) AS `total` FROM `contact` 
+
+
+	$r = q("SELECT COUNT(*) AS `total` FROM `contact`
 		WHERE `uid` = %d AND `self` = 0 AND `pending` = 0 $sql_extra $sql_extra2 ",
 		intval($_SESSION['uid']));
 	if(count($r)) {
@@ -668,7 +685,7 @@ function contacts_content(&$a) {
 				$url = "redir/{$rr['id']}";
 				$sparkle = ' class="sparkle" ';
 			}
-			else { 
+			else {
 				$url = $rr['url'];
 				$sparkle = '';
 			}
@@ -681,7 +698,7 @@ function contacts_content(&$a) {
 				'id' => $rr['id'],
 				'alt_text' => $alt_text,
 				'dir_icon' => $dir_icon,
-				'thumb' => $rr['thumb'], 
+				'thumb' => proxy_url($rr['thumb']),
 				'name' => $rr['name'],
 				'username' => $rr['name'],
 				'sparkle' => $sparkle,

mod/display.php

@@ -16,7 +16,7 @@ function display_init(&$a) {
 
 		// Does the local user have this item?
 		if (local_user()) {
-			$r = q("SELECT `id`, `parent`, `author-name`, `author-link`, `author-avatar`, `network`, `body` FROM `item`
+			$r = q("SELECT `id`, `parent`, `author-name`, `author-link`, `author-avatar`, `network`, `body`, `uid` FROM `item`
 				WHERE `item`.`visible` = 1 AND `item`.`deleted` = 0 and `item`.`moderated` = 0
 					AND `guid` = '%s' AND `uid` = %d", $a->argv[1], local_user());
 			if (count($r)) {
@@ -41,9 +41,21 @@ function display_init(&$a) {
 				$itemuid = $r[0]["uid"];
 			}
 		}
+
+		// Is it an item with uid=0?
+		if ($nick == "") {
+			$r = q("SELECT `item`.`id`, `item`.`parent`, `item`.`author-name`,
+				`item`.`author-link`, `item`.`author-avatar`, `item`.`network`, `item`.`uid`, `item`.`body`
+				FROM `item` WHERE `item`.`visible` = 1 AND `item`.`deleted` = 0 and `item`.`moderated` = 0
+					AND `item`.`allow_cid` = ''  AND `item`.`allow_gid` = ''
+					AND `item`.`deny_cid`  = '' AND `item`.`deny_gid`  = ''
+					AND `item`.`private` = 0 AND `item`.`uid` = 0
+					AND `item`.`guid` = '%s'", $a->argv[1]);
+				//	AND `item`.`private` = 0 AND `item`.`wall` = 1
+		}
 		if (count($r)) {
 			if ($r[0]["id"] != $r[0]["parent"])
-				$r = q("SELECT `id`, `author-name`, `author-link`, `author-avatar`, `network`, `body` FROM `item`
+				$r = q("SELECT `id`, `author-name`, `author-link`, `author-avatar`, `network`, `body`, `uid` FROM `item`
 					WHERE `item`.`visible` = 1 AND `item`.`deleted` = 0 and `item`.`moderated` = 0
 						AND `id` = %d", $r[0]["parent"]);
 
@@ -89,23 +101,28 @@ function display_fetchauthor($a, $item) {
 	$profiledata["url"] = $item["author-link"];
 	$profiledata["network"] = $item["network"];
 
-	// Fetching profile data from unique contacts
+	// Fetching further contact data from the contact table
-	// To-do: Extend "unique contacts" table for further contact data like location, ...
+	$r = q("SELECT `photo`, `nick`, `location`, `about` FROM `contact` WHERE `nurl` = '%s' AND `uid` = %d",
-	$r = q("SELECT `avatar`, `nick` FROM `unique_contacts` WHERE `url` = '%s'", normalise_link($profiledata["url"]));
+		normalise_link($profiledata["url"]), $item["uid"]);
 	if (count($r)) {
-		$profiledata["photo"] = proxy_url($r[0]["avatar"]);
+		$profiledata["photo"] = proxy_url($r[0]["photo"]);
+		$profiledata["address"] = proxy_parse_html(bbcode($r[0]["location"]));
+		$profiledata["about"] = proxy_parse_html(bbcode($r[0]["about"]));
 		if ($r[0]["nick"] != "")
 			$profiledata["nickname"] = $r[0]["nick"];
-	} else {
+	}
-		// Is this case possible?
+
-		// Fetching further contact data from the contact table, when it isn't available in the "unique contacts"
+	// Fetching profile data from unique contacts
-		$r = q("SELECT `photo`, `nick` FROM `contact` WHERE `nurl` = '%s' AND `uid` = %d",
+	$r = q("SELECT `avatar`, `nick`, `location`, `about` FROM `unique_contacts` WHERE `url` = '%s'", normalise_link($profiledata["url"]));
-			normalise_link($profiledata["url"]), $itemuid);
+	if (count($r)) {
-		if (count($r)) {
+		if ($profiledata["photo"] == "")
-			$profiledata["photo"] = proxy_url($r[0]["photo"]);
+			$profiledata["photo"] = proxy_url($r[0]["avatar"]);
-			if ($r[0]["nick"] != "")
+		if ($profiledata["address"] == "")
-				$profiledata["nickname"] = $r[0]["nick"];
+			$profiledata["address"] = proxy_parse_html(bbcode($r[0]["location"]));
-		}
+		if ($profiledata["about"] == "")
+			$profiledata["about"] = proxy_parse_html(bbcode($r[0]["about"]));
+		if (($profiledata["nickname"] == "") AND ($r[0]["nick"] != ""))
+			$profiledata["nickname"] = $r[0]["nick"];
 	}
 
 	// Check for a repeated message
@@ -158,6 +175,21 @@ function display_fetchauthor($a, $item) {
 
 		$profiledata["nickname"] = $profiledata["name"];
 		$profiledata["network"] = GetProfileUsername($profiledata["url"], "", false, true);
+
+		$profiledata["address"] = "";
+		$profiledata["about"] = "";
+
+		// Fetching profile data from unique contacts
+		if ($profiledata["url"] != "") {
+			$r = q("SELECT `avatar`, `nick`, `location`, `about` FROM `unique_contacts` WHERE `url` = '%s'", normalise_link($profiledata["url"]));
+			if (count($r)) {
+				$profiledata["photo"] = proxy_url($r[0]["avatar"]);
+				$profiledata["address"] = proxy_parse_html(bbcode($r[0]["location"]));
+				$profiledata["about"] = proxy_parse_html(bbcode($r[0]["about"]));
+				if ($r[0]["nick"] != "")
+					$profiledata["nickname"] = $r[0]["nick"];
+			}
+		}
 	}
 
 	if (local_user()) {
@@ -232,6 +264,18 @@ function display_content(&$a, $update = 0) {
 					$nick = $r[0]["nickname"];
 				}
 			}
+			if ($nick == "") {
+				$r = q("SELECT `item`.`id` FROM `item`
+					WHERE `item`.`visible` = 1 AND `item`.`deleted` = 0 and `item`.`moderated` = 0
+						AND `item`.`allow_cid` = ''  AND `item`.`allow_gid` = ''
+						AND `item`.`deny_cid`  = '' AND `item`.`deny_gid`  = ''
+						AND `item`.`private` = 0  AND `item`.`uid` = 0
+						AND `item`.`guid` = '%s'", $a->argv[1]);
+					//	AND `item`.`private` = 0 AND `item`.`wall` = 1
+				if (count($r)) {
+					$item_id = $r[0]["id"];
+				}
+			}
 		}
 	}
 

mod/friendica.php

@@ -38,11 +38,8 @@ function friendica_init(&$a) {
 			'site_name' => $a->config['sitename'],
 			'platform' => FRIENDICA_PLATFORM,
 			'info' => ((x($a->config,'info')) ? $a->config['info'] : ''),
+			'no_scrape_url' => $a->get_baseurl().'/noscrape'
 		);
-		
-		//Enable noscrape?
-		if(!get_config('system','disable_noscrape'))
-			$data['no_scrape_url'] = $a->get_baseurl().'/noscrape';
 
 		echo json_encode($data);
 		killme();

mod/help.php

@@ -1,53 +1,6 @@
 <?php
-define( 'MARKDOWN_PARSER_CLASS',  'ExtendedMarkdown' );
 require_once('library/markdown.php');
 
-class ExtendedMarkdown extends MarkdownExtra_Parser {
-
-	function ExtendedMarkdown() {
-		$this->block_gamut += array(
-			"doBlockWarning" => 45,
-		);
-		parent::MarkdownExtra_Parser();
-	}
-
-	function doBlockWarning($text) {
-		$text = preg_replace_callback('/
-			  (								# Wrap whole match in $1
-				(?>
-				  ^[ ]*![ ]?				# "!" at the start of a line
-					.+\n					# rest of the first line
-				  (.+\n)*					# subsequent consecutive lines
-				  \n*						# blanks
-				)+
-			  )
-			/xm', array(&$this, '_doBlockWarning_callback'), $text);
-
-		return $text;
-	}
-
-	function _doBlockWarning_callback($matches) {
-		$bq = $matches[1];
-		# trim one level of quoting - trim whitespace-only lines
-		$bq = preg_replace('/^[ ]*![ ]?|^[ ]+$/m', '', $bq);
-		$bq = $this->runBlockGamut($bq);  # recurse
-
-		$bq = preg_replace('/^/m', "  ", $bq);
-		# These leading spaces cause problem with <pre> content, 
-		# so we need to fix that:
-//		$bq = preg_replace_callback('{(\s*<pre>.+?</pre>)}sx', array(&$this, '__doBlockWarning_callback2'), $bq);
-
-		return "\n" . $this->hashBlock("<div class='md_warning'>\n$bq\n</div>") . "\n\n";
-	}
-
-	function _doBlockWarning_callback2($matches) {
-		$pre = $matches[1];
-		$pre = preg_replace('/^  /m', '', $pre);
-		return $pre;
-	}
-
-}
-
 if (!function_exists('load_doc_file')) {
 
 	function load_doc_file($s) {
@@ -66,8 +19,7 @@ if (!function_exists('load_doc_file')) {
 }
 
 function help_content(&$a) {
-	
+
-	
 	nav_set_selected('help');
 
 	global $lang;
@@ -93,9 +45,9 @@ function help_content(&$a) {
 					'$message' => t('Page not found.')
 				));
 	}
-	
+
 	$html = Markdown($text);
 	$html = "<style>.md_warning { padding: 1em; border: #ff0000 solid 2px; background-color: #f9a3a3; color: #ffffff;</style>".$html;
 	return $html;
-		
+
 }