From 344210bd6d58a8fa1e01c16b9c0366e44e1a6146 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 25 Jan 2015 00:19:59 +0100 Subject: [PATCH 1/7] Global contacts are now added with reshares as well. Better support for "aboutme" and "location" in poco. --- include/items.php | 19 ++++++++++++++----- include/socgraph.php | 41 ++++++++++++++++++++++++++++++++++++++++- mod/poco.php | 11 +++++++++-- 3 files changed, 63 insertions(+), 8 deletions(-) diff --git a/include/items.php b/include/items.php index 77337fe0d..45f762fa5 100644 --- a/include/items.php +++ b/include/items.php @@ -1350,6 +1350,9 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa return 0; } + // Store the unescaped version + $unescaped = $arr; + dbesc_array($arr); logger('item_store: ' . print_r($arr,true), LOGGER_DATA); @@ -1360,10 +1363,12 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa . implode("', '", array_values($arr)) . "')" ); - // find the item we just created + // And restore it + $arr = $unescaped; + // find the item we just created $r = q("SELECT `id` FROM `item` WHERE `uri` = '%s' AND `uid` = %d ORDER BY `id` ASC ", - $arr['uri'], // already dbesc'd + dbesc($arr['uri']), intval($arr['uid']) ); @@ -1374,8 +1379,12 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa // Add every contact to the global contact table // Contacts from the statusnet connector are also added since you could add them in OStatus as well. if (!$arr['private'] AND in_array($arr["network"], - array(NETWORK_DFRN, NETWORK_DIASPORA, NETWORK_OSTATUS, NETWORK_STATUSNET, ""))) - poco_check($arr["author-link"], $arr["author-name"], $arr["network"], $arr["author-avatar"], "", $arr["received"], $arr['contact-id'], $arr['uid']); + array(NETWORK_DFRN, NETWORK_DIASPORA, NETWORK_OSTATUS, NETWORK_STATUSNET, ""))) { + poco_check($arr["author-link"], $arr["author-name"], $arr["network"], $arr["author-avatar"], "", $arr["received"], $arr["contact-id"], $arr["uid"]); + + // Maybe its a body with a shared item? Then extract a global contact from it. + poco_contact_from_body($arr["body"], $arr["received"], $arr["contact-id"], $arr["uid"]); + } // Set "success_update" to the date of the last time we heard from this contact // This can be used to filter for inactive contacts and poco. @@ -1437,7 +1446,7 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa if(count($r) > 1) { logger('item_store: duplicated post occurred. Removing duplicates.'); q("DELETE FROM `item` WHERE `uri` = '%s' AND `uid` = %d AND `id` != %d ", - $arr['uri'], + dbesc($arr['uri']), intval($arr['uid']), intval($current_post) ); diff --git a/include/socgraph.php b/include/socgraph.php index 0a8503790..b6bfa6831 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -115,7 +115,22 @@ function poco_load($cid,$uid = 0,$zcid = 0,$url = null) { function poco_check($profile_url, $name, $network, $profile_photo, $connect_url, $updated, $cid = 0, $uid = 0, $zcid = 0) { $gcid = ""; - if (($profile_url == "") OR ($name == "") OR ($profile_photo == "")) + if ($profile_url == "") + return $gcid; + + if (($network == "") OR ($name == "") OR ($profile_photo == "")) { + require_once("include/Scrape.php"); + + $data = probe_url($profile_url, PROBE_DIASPORA); + $network = $data["network"]; + $name = $data["name"]; + $profile_photo = $data["photo"]; + } + + if (!in_array($network, array(NETWORK_DFRN, NETWORK_OSTATUS, NETWORK_DIASPORA, NETWORK_STATUSNET))) + return $gcid; + + if (($name == "") OR ($profile_photo == "")) return $gcid; logger("profile-check URL: ".$profile_url." name: ".$name." avatar: ".$profile_photo, LOGGER_DEBUG); @@ -194,6 +209,30 @@ function poco_check($profile_url, $name, $network, $profile_photo, $connect_url, return $gcid; } +function poco_contact_from_body($body, $created, $cid, $uid) { + preg_replace_callback("/\[share(.*?)\].*?\[\/share\]/ism", + function ($match) use ($created, $cid, $uid){ + return(sub_poco_from_share($match, $created, $cid, $uid)); + }, $body); +} + +function sub_poco_from_share($share, $created, $cid, $uid) { + $profile = ""; + preg_match("/profile='(.*?)'/ism", $share[1], $matches); + if ($matches[1] != "") + $profile = $matches[1]; + + preg_match('/profile="(.*?)"/ism', $share[1], $matches); + if ($matches[1] != "") + $profile = $matches[1]; + + if ($profile == "") + return; + + logger("prepare poco_check for profile ".$profile, LOGGER_DEBUG); + poco_check($profile, "", "", "", "", $created, $cid, $uid); +} + function count_common_friends($uid,$cid) { $r = q("SELECT count(*) as `total` diff --git a/mod/poco.php b/mod/poco.php index 154f13c7e..e7302a962 100644 --- a/mod/poco.php +++ b/mod/poco.php @@ -83,8 +83,9 @@ function poco_init(&$a) { if($system_mode) { - $r = q("SELECT * FROM `contact` WHERE `self` = 1 AND `network` IN ('%s', '%s', '%s', '%s', '') - AND `uid` IN (SELECT `uid` FROM `pconfig` WHERE `cat` = 'system' AND `k` = 'suggestme' AND `v` = 1) LIMIT %d, %d", + $r = q("SELECT `contact`.*, `profile`.`about` AS `pabout`, `profile`.`locality` AS `plocation` FROM `contact` INNER JOIN `profile` ON `profile`.`uid` = `contact`.`uid` + WHERE `self` = 1 AND `network` IN ('%s', '%s', '%s', '%s', '') + AND `contact`.`uid` IN (SELECT `uid` FROM `pconfig` WHERE `cat` = 'system' AND `k` = 'suggestme' AND `v` = 1) LIMIT %d, %d", dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_OSTATUS), @@ -143,6 +144,12 @@ function poco_init(&$a) { if(is_array($r)) { if(count($r)) { foreach($r as $rr) { + if (($rr['about'] == "") AND isset($rr['pabout'])) + $rr['about'] = $rr['pabout']; + + if (($rr['location'] == "") AND isset($rr['plocation'])) + $rr['location'] = $rr['plocation']; + $entry = array(); if($fields_ret['id']) $entry['id'] = $rr['id']; From 597ef20b71ff7950262d54e0374399fae1400d61 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 25 Jan 2015 02:29:46 +0100 Subject: [PATCH 2/7] "about" and "location" is now included in global contacts as well. --- boot.php | 2 +- include/dbstructure.php | 2 ++ include/items.php | 2 +- include/socgraph.php | 38 +++++++++++++++++++++++++++++++------- update.php | 2 +- 5 files changed, 36 insertions(+), 10 deletions(-) diff --git a/boot.php b/boot.php index 72cb8a341..b09da9c17 100644 --- a/boot.php +++ b/boot.php @@ -18,7 +18,7 @@ define ( 'FRIENDICA_PLATFORM', 'Friendica'); define ( 'FRIENDICA_CODENAME', 'Ginger'); define ( 'FRIENDICA_VERSION', '3.3.2' ); define ( 'DFRN_PROTOCOL_VERSION', '2.23' ); -define ( 'DB_UPDATE_VERSION', 1177 ); +define ( 'DB_UPDATE_VERSION', 1178 ); define ( 'EOL', "
\r\n" ); define ( 'ATOM_TIME', 'Y-m-d\TH:i:s\Z' ); diff --git a/include/dbstructure.php b/include/dbstructure.php index d0a7324e2..bd4ecb789 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -619,6 +619,8 @@ function db_definition() { "photo" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "connect" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "updated" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"), + "location" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), + "about" => array("type" => "text", "not null" => "1"), "network" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), ), "indexes" => array( diff --git a/include/items.php b/include/items.php index 45f762fa5..992722351 100644 --- a/include/items.php +++ b/include/items.php @@ -1380,7 +1380,7 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa // Contacts from the statusnet connector are also added since you could add them in OStatus as well. if (!$arr['private'] AND in_array($arr["network"], array(NETWORK_DFRN, NETWORK_DIASPORA, NETWORK_OSTATUS, NETWORK_STATUSNET, ""))) { - poco_check($arr["author-link"], $arr["author-name"], $arr["network"], $arr["author-avatar"], "", $arr["received"], $arr["contact-id"], $arr["uid"]); + poco_check($arr["author-link"], $arr["author-name"], $arr["network"], $arr["author-avatar"], "", "", "", $arr["received"], $arr["contact-id"], $arr["uid"]); // Maybe its a body with a shared item? Then extract a global contact from it. poco_contact_from_body($arr["body"], $arr["received"], $arr["contact-id"], $arr["uid"]); diff --git a/include/socgraph.php b/include/socgraph.php index b6bfa6831..e50474fd3 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -39,7 +39,7 @@ function poco_load($cid,$uid = 0,$zcid = 0,$url = null) { if(! $url) return; - $url = $url . (($uid) ? '/@me/@all?fields=displayName,urls,photos,updated,network' : '?fields=displayName,urls,photos,updated,network') ; + $url = $url . (($uid) ? '/@me/@all?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation' : '?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation') ; logger('poco_load: ' . $url, LOGGER_DEBUG); @@ -69,6 +69,8 @@ function poco_load($cid,$uid = 0,$zcid = 0,$url = null) { $name = ''; $network = ''; $updated = '0000-00-00 00:00:00'; + $location = ''; + $about = ''; $name = $entry->displayName; @@ -99,7 +101,19 @@ function poco_load($cid,$uid = 0,$zcid = 0,$url = null) { if(isset($entry->network)) $network = $entry->network; - poco_check($profile_url, $name, $network, $profile_photo, $connect_url, $updated, $cid, $uid, $zcid); + if(isset($entry->currentLocation)) + $location = $entry->currentLocation; + + if(isset($entry->aboutMe)) + $about = $entry->aboutMe; + + poco_check($profile_url, $name, $network, $profile_photo, $about, $location, $connect_url, $updated, $cid, $uid, $zcid); + + if (($location != "") OR ($about != "")) + q("UPDATE `contact` SET `location` = '%s', `about` = '%s' WHERE `nurl` = '%s' AND NOT `self`", + dbesc($location), + dbesc($about), + dbesc(normalise_link($profile_url))); } logger("poco_load: loaded $total entries",LOGGER_DEBUG); @@ -112,7 +126,7 @@ function poco_load($cid,$uid = 0,$zcid = 0,$url = null) { } -function poco_check($profile_url, $name, $network, $profile_photo, $connect_url, $updated, $cid = 0, $uid = 0, $zcid = 0) { +function poco_check($profile_url, $name, $network, $profile_photo, $about, $location, $connect_url, $updated, $cid = 0, $uid = 0, $zcid = 0) { $gcid = ""; if ($profile_url == "") @@ -142,8 +156,14 @@ function poco_check($profile_url, $name, $network, $profile_photo, $connect_url, if(count($x)) { $gcid = $x[0]['id']; + if (($location == "") AND ($x[0]['location'] != "")) + $location = $x[0]['location']; + + if (($about == "") AND ($x[0]['about'] != "")) + $about = $x[0]['about']; + if($x[0]['name'] != $name || $x[0]['photo'] != $profile_photo || $x[0]['updated'] < $updated) { - q("update gcontact set `name` = '%s', `network` = '%s', `photo` = '%s', `connect` = '%s', `url` = '%s', `updated` = '%s' + q("update gcontact set `name` = '%s', `network` = '%s', `photo` = '%s', `connect` = '%s', `url` = '%s', `updated` = '%s', `location` = '%s', `about` = '%s' where `nurl` = '%s'", dbesc($name), dbesc($network), @@ -151,11 +171,13 @@ function poco_check($profile_url, $name, $network, $profile_photo, $connect_url, dbesc($connect_url), dbesc($profile_url), dbesc($updated), + dbesc($location), + dbesc($about), dbesc(normalise_link($profile_url)) ); } } else { - q("insert into `gcontact` (`name`,`network`, `url`,`nurl`,`photo`,`connect`, `updated`) + q("insert into `gcontact` (`name`,`network`, `url`,`nurl`,`photo`,`connect`, `updated`, `location`, `about`) values ('%s', '%s', '%s', '%s', '%s','%s', '%s')", dbesc($name), dbesc($network), @@ -163,7 +185,9 @@ function poco_check($profile_url, $name, $network, $profile_photo, $connect_url, dbesc(normalise_link($profile_url)), dbesc($profile_photo), dbesc($connect_url), - dbesc($updated) + dbesc($updated), + dbesc($location), + dbesc($about) ); $x = q("SELECT * FROM `gcontact` WHERE `nurl` = '%s' LIMIT 1", dbesc(normalise_link($profile_url)) @@ -230,7 +254,7 @@ function sub_poco_from_share($share, $created, $cid, $uid) { return; logger("prepare poco_check for profile ".$profile, LOGGER_DEBUG); - poco_check($profile, "", "", "", "", $created, $cid, $uid); + poco_check($profile, "", "", "", "", "", "", $created, $cid, $uid); } function count_common_friends($uid,$cid) { diff --git a/update.php b/update.php index b4fb64048..019e4cad3 100644 --- a/update.php +++ b/update.php @@ -1,6 +1,6 @@ Date: Sun, 25 Jan 2015 13:19:37 +0100 Subject: [PATCH 3/7] Support for keywords and gender in (global) contacts. --- include/dbstructure.php | 4 ++++ include/diaspora.php | 22 +++++++++++++++++++- include/items.php | 2 +- include/socgraph.php | 46 +++++++++++++++++++++++++++++++---------- mod/poco.php | 31 ++++++++++++++++++++++++--- mod/profiles.php | 23 ++++++++++++++++++--- 6 files changed, 109 insertions(+), 19 deletions(-) diff --git a/include/dbstructure.php b/include/dbstructure.php index bd4ecb789..adb826c8b 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -415,6 +415,8 @@ function db_definition() { "nick" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "location" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "about" => array("type" => "text", "not null" => "1"), + "keywords" => array("type" => "text", "not null" => "1"), + "gender" => array("type" => "varchar(32)", "not null" => "1", "default" => ""), "attag" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "photo" => array("type" => "text", "not null" => "1"), "thumb" => array("type" => "text", "not null" => "1"), @@ -621,6 +623,8 @@ function db_definition() { "updated" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"), "location" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "about" => array("type" => "text", "not null" => "1"), + "keywords" => array("type" => "text", "not null" => "1"), + "gender" => array("type" => "varchar(32)", "not null" => "1", "default" => ""), "network" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), ), "indexes" => array( diff --git a/include/diaspora.php b/include/diaspora.php index 0cc97713e..affd59d48 100755 --- a/include/diaspora.php +++ b/include/diaspora.php @@ -2252,7 +2252,19 @@ function diaspora_profile($importer,$xml,$msg) { $birthday = unxmlify($xml->birthday); $location = diaspora2bb(unxmlify($xml->location)); $about = diaspora2bb(unxmlify($xml->bio)); + $gender = unxmlify($xml->gender); + $tags = unxmlify($xml->tag_string); + $tags = explode("#", $tags); + + $keywords = array(); + foreach ($tags as $tag) { + $tag = trim(strtolower($tag)); + if ($tag != "") + $keywords[] = $tag; + } + + $keywords = implode(", ", $keywords); $handle_parts = explode("@", $diaspora_handle); if($name === '') { @@ -2288,7 +2300,7 @@ function diaspora_profile($importer,$xml,$msg) { // TODO: update name on item['author-name'] if the name changed. See consume_feed() // Not doing this currently because D* protocol is scheduled for revision soon. - $r = q("UPDATE `contact` SET `name` = '%s', `name-date` = '%s', `photo` = '%s', `thumb` = '%s', `micro` = '%s', `avatar-date` = '%s' , `bd` = '%s', `location` = '%s', `about` = '%s' WHERE `id` = %d AND `uid` = %d", + $r = q("UPDATE `contact` SET `name` = '%s', `name-date` = '%s', `photo` = '%s', `thumb` = '%s', `micro` = '%s', `avatar-date` = '%s' , `bd` = '%s', `location` = '%s', `about` = '%s', `keywords` = '%s', `gender` = '%s' WHERE `id` = %d AND `uid` = %d", dbesc($name), dbesc(datetime_convert()), dbesc($images[0]), @@ -2298,10 +2310,18 @@ function diaspora_profile($importer,$xml,$msg) { dbesc($birthday), dbesc($location), dbesc($about), + dbesc($keywords), + dbesc($gender), intval($contact['id']), intval($importer['uid']) ); + if (unxmlify($xml->searchable) == "true") { + require_once('include/socgraph.php'); + poco_check($contact['url'], $name, NETWORK_DIASPORA, $images[0], $about, $location, $gender, $keywords, "", + datetime_convert(), $contact['id'], $importer['uid']); + } + $profileurl = ""; $author = q("SELECT * FROM `unique_contacts` WHERE `url`='%s' LIMIT 1", dbesc(normalise_link($contact['url']))); diff --git a/include/items.php b/include/items.php index 992722351..344b06ec8 100644 --- a/include/items.php +++ b/include/items.php @@ -1380,7 +1380,7 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa // Contacts from the statusnet connector are also added since you could add them in OStatus as well. if (!$arr['private'] AND in_array($arr["network"], array(NETWORK_DFRN, NETWORK_DIASPORA, NETWORK_OSTATUS, NETWORK_STATUSNET, ""))) { - poco_check($arr["author-link"], $arr["author-name"], $arr["network"], $arr["author-avatar"], "", "", "", $arr["received"], $arr["contact-id"], $arr["uid"]); + poco_check($arr["author-link"], $arr["author-name"], $arr["network"], $arr["author-avatar"], "", "", "", "", "", $arr["received"], $arr["contact-id"], $arr["uid"]); // Maybe its a body with a shared item? Then extract a global contact from it. poco_contact_from_body($arr["body"], $arr["received"], $arr["contact-id"], $arr["uid"]); diff --git a/include/socgraph.php b/include/socgraph.php index e50474fd3..f37fdca98 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -39,7 +39,7 @@ function poco_load($cid,$uid = 0,$zcid = 0,$url = null) { if(! $url) return; - $url = $url . (($uid) ? '/@me/@all?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation' : '?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation') ; + $url = $url . (($uid) ? '/@me/@all?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender' : '?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender') ; logger('poco_load: ' . $url, LOGGER_DEBUG); @@ -71,6 +71,8 @@ function poco_load($cid,$uid = 0,$zcid = 0,$url = null) { $updated = '0000-00-00 00:00:00'; $location = ''; $about = ''; + $keywords = ''; + $gender = ''; $name = $entry->displayName; @@ -107,13 +109,24 @@ function poco_load($cid,$uid = 0,$zcid = 0,$url = null) { if(isset($entry->aboutMe)) $about = $entry->aboutMe; - poco_check($profile_url, $name, $network, $profile_photo, $about, $location, $connect_url, $updated, $cid, $uid, $zcid); + if(isset($entry->gender)) + $gender = $entry->gender; - if (($location != "") OR ($about != "")) - q("UPDATE `contact` SET `location` = '%s', `about` = '%s' WHERE `nurl` = '%s' AND NOT `self`", + if(isset($entry->tags)) + $keywords = implode(", ", $entry->tags); + + poco_check($profile_url, $name, $network, $profile_photo, $about, $location, $gender, $keywords, $connect_url, $updated, $cid, $uid, $zcid); + + // Update the Friendica contacts. Diaspora is doing it via a message. (See include/diaspora.php) + if (($location != "") OR ($about != "") OR ($keywords != "") OR ($gender != "")) + q("UPDATE `contact` SET `location` = '%s', `about` = '%s', `keywords` = '%s', `gender` = '%s' + WHERE `nurl` = '%s' AND NOT `self` AND `network` = '%s'", dbesc($location), dbesc($about), - dbesc(normalise_link($profile_url))); + dbesc($keywords), + dbesc($gender), + dbesc(normalise_link($profile_url)), + dbesc(NETWORK_DFRN)); } logger("poco_load: loaded $total entries",LOGGER_DEBUG); @@ -126,7 +139,7 @@ function poco_load($cid,$uid = 0,$zcid = 0,$url = null) { } -function poco_check($profile_url, $name, $network, $profile_photo, $about, $location, $connect_url, $updated, $cid = 0, $uid = 0, $zcid = 0) { +function poco_check($profile_url, $name, $network, $profile_photo, $about, $location, $gender, $keywords, $connect_url, $updated, $cid = 0, $uid = 0, $zcid = 0) { $gcid = ""; if ($profile_url == "") @@ -162,8 +175,15 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca if (($about == "") AND ($x[0]['about'] != "")) $about = $x[0]['about']; + if (($gender == "") AND ($x[0]['gender'] != "")) + $gender = $x[0]['gender']; + + if (($keywords == "") AND ($x[0]['keywords'] != "")) + $keywords = $x[0]['keywords']; + if($x[0]['name'] != $name || $x[0]['photo'] != $profile_photo || $x[0]['updated'] < $updated) { - q("update gcontact set `name` = '%s', `network` = '%s', `photo` = '%s', `connect` = '%s', `url` = '%s', `updated` = '%s', `location` = '%s', `about` = '%s' + q("update gcontact set `name` = '%s', `network` = '%s', `photo` = '%s', `connect` = '%s', `url` = '%s', + `updated` = '%s', `location` = '%s', `about` = '%s', `keywords` = '%s', `gender` = '%s' where `nurl` = '%s'", dbesc($name), dbesc($network), @@ -173,12 +193,14 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca dbesc($updated), dbesc($location), dbesc($about), + dbesc($keywords), + dbesc($gender), dbesc(normalise_link($profile_url)) ); } } else { - q("insert into `gcontact` (`name`,`network`, `url`,`nurl`,`photo`,`connect`, `updated`, `location`, `about`) - values ('%s', '%s', '%s', '%s', '%s','%s', '%s')", + q("insert into `gcontact` (`name`,`network`, `url`,`nurl`,`photo`,`connect`, `updated`, `location`, `about`, `keywords`, `gender`) + values ('%s', '%s', '%s', '%s', '%s','%s', '%s', '%s', '%s', '%s', '%s')", dbesc($name), dbesc($network), dbesc($profile_url), @@ -187,7 +209,9 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca dbesc($connect_url), dbesc($updated), dbesc($location), - dbesc($about) + dbesc($about), + dbesc($keywords), + dbesc($gender) ); $x = q("SELECT * FROM `gcontact` WHERE `nurl` = '%s' LIMIT 1", dbesc(normalise_link($profile_url)) @@ -254,7 +278,7 @@ function sub_poco_from_share($share, $created, $cid, $uid) { return; logger("prepare poco_check for profile ".$profile, LOGGER_DEBUG); - poco_check($profile, "", "", "", "", "", "", $created, $cid, $uid); + poco_check($profile, "", "", "", "", "", "", "", "", $created, $cid, $uid); } function count_common_friends($uid,$cid) { diff --git a/mod/poco.php b/mod/poco.php index e7302a962..aa570ec34 100644 --- a/mod/poco.php +++ b/mod/poco.php @@ -83,8 +83,9 @@ function poco_init(&$a) { if($system_mode) { - $r = q("SELECT `contact`.*, `profile`.`about` AS `pabout`, `profile`.`locality` AS `plocation` FROM `contact` INNER JOIN `profile` ON `profile`.`uid` = `contact`.`uid` - WHERE `self` = 1 AND `network` IN ('%s', '%s', '%s', '%s', '') + $r = q("SELECT `contact`.*, `profile`.`about` AS `pabout`, `profile`.`locality` AS `plocation`, `profile`.`pub_keywords`, `profile`.`gender` AS `pgender` + FROM `contact` INNER JOIN `profile` ON `profile`.`uid` = `contact`.`uid` + WHERE `self` = 1 AND `network` IN ('%s', '%s', '%s', '%s', '') AND `profile`.`is-default` AND `contact`.`uid` IN (SELECT `uid` FROM `pconfig` WHERE `cat` = 'system' AND `k` = 'suggestme' AND `v` = 1) LIMIT %d, %d", dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), @@ -129,7 +130,9 @@ function poco_init(&$a) { 'photos' => false, 'aboutMe' => false, 'currentLocation' => false, - 'network' => false + 'network' => false, + 'gender' => false, + 'tags' => false ); if((! x($_GET,'fields')) || ($_GET['fields'] === '@all')) @@ -150,6 +153,12 @@ function poco_init(&$a) { if (($rr['location'] == "") AND isset($rr['plocation'])) $rr['location'] = $rr['plocation']; + if (($rr['gender'] == "") AND isset($rr['pgender'])) + $rr['gender'] = $rr['pgender']; + + if (($rr['keywords'] == "") AND isset($rr['pub_keywords'])) + $rr['keywords'] = $rr['pub_keywords']; + $entry = array(); if($fields_ret['id']) $entry['id'] = $rr['id']; @@ -159,6 +168,8 @@ function poco_init(&$a) { $entry['aboutMe'] = bbcode($rr['about'], false, false); if($fields_ret['currentLocation']) $entry['currentLocation'] = $rr['location']; + if($fields_ret['gender']) + $entry['gender'] = $rr['gender']; if($fields_ret['urls']) { $entry['urls'] = array(array('value' => $rr['url'], 'type' => 'profile')); if($rr['addr'] && ($rr['network'] !== NETWORK_MAIL)) @@ -189,6 +200,20 @@ function poco_init(&$a) { if (($entry['network'] == "") AND ($rr['self'])) $entry['network'] = NETWORK_DFRN; } + if($fields_ret['tags']) { + $tags = str_replace(","," ",$rr['keywords']); + $tags = explode(" ", $tags); + + $cleaned = array(); + foreach ($tags as $tag) { + $tag = trim(strtolower($tag)); + if ($tag != "") + $cleaned[] = $tag; + } + + $entry['tags'] = array($cleaned); + } + $ret['entry'][] = $entry; } } diff --git a/mod/profiles.php b/mod/profiles.php index e08b621c0..a0f10fb95 100644 --- a/mod/profiles.php +++ b/mod/profiles.php @@ -142,6 +142,22 @@ function profiles_init(&$a) { } +function profile_clean_keywords($keywords) { + $keywords = str_replace(","," ",$keywords); + $keywords = explode(" ", $keywords); + + $cleaned = array(); + foreach ($keywords as $keyword) { + $keyword = trim(strtolower($keyword)); + if ($keyword != "") + $cleaned[] = $keyword; + } + + $keywords = implode(", ", $cleaned); + + return $keywords; +} + function profiles_post(&$a) { if(! local_user()) { @@ -212,8 +228,8 @@ function profiles_post(&$a) { $region = notags(trim($_POST['region'])); $postal_code = notags(trim($_POST['postal_code'])); $country_name = notags(trim($_POST['country_name'])); - $pub_keywords = notags(trim($_POST['pub_keywords'])); - $prv_keywords = notags(trim($_POST['prv_keywords'])); + $pub_keywords = profile_clean_keywords(notags(trim($_POST['pub_keywords']))); + $prv_keywords = profile_clean_keywords(notags(trim($_POST['prv_keywords']))); $marital = notags(trim($_POST['marital'])); $howlong = notags(trim($_POST['howlong'])); @@ -469,9 +485,10 @@ function profiles_post(&$a) { if($is_default) { - $r = q("UPDATE `contact` SET `about` = '%s', `location` = '%s' WHERE `self` = 1 AND `uid` = %d", + $r = q("UPDATE `contact` SET `about` = '%s', `location` = '%s', `keywords` = '%s' WHERE `self` = 1 AND `uid` = %d", dbesc($about), dbesc($locality), + dbesc($pub_keywords), intval(local_user()) ); From 105d6a22a4905afc60bed8a826842f7e0ce3ec44 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 25 Jan 2015 13:22:25 +0100 Subject: [PATCH 4/7] gender will be updated in the profiles as well. --- mod/profiles.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mod/profiles.php b/mod/profiles.php index a0f10fb95..e71a4c603 100644 --- a/mod/profiles.php +++ b/mod/profiles.php @@ -485,10 +485,11 @@ function profiles_post(&$a) { if($is_default) { - $r = q("UPDATE `contact` SET `about` = '%s', `location` = '%s', `keywords` = '%s' WHERE `self` = 1 AND `uid` = %d", + $r = q("UPDATE `contact` SET `about` = '%s', `location` = '%s', `keywords` = '%s', `gender` = '%s' WHERE `self` = 1 AND `uid` = %d", dbesc($about), dbesc($locality), dbesc($pub_keywords), + dbesc($gender), intval(local_user()) ); From bc2c046446baab7cfe143700e9ddab714f7488f1 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 25 Jan 2015 16:40:20 +0100 Subject: [PATCH 5/7] poco: Only update the global contacts when there is a network value/Export your own contact as well --- include/Scrape.php | 2 +- include/socgraph.php | 28 +++++++++++++++------------- mod/poco.php | 4 ++-- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/include/Scrape.php b/include/Scrape.php index fa2d7944a..31e2be263 100644 --- a/include/Scrape.php +++ b/include/Scrape.php @@ -345,7 +345,7 @@ function probe_url($url, $mode = PROBE_NORMAL) { $result = Cache::get("probe_url:".$mode.":".$url); if (!is_null($result)) { - $result = unserialize($result); + $result = unserialize($result); return $result; } diff --git a/include/socgraph.php b/include/socgraph.php index f37fdca98..f2ccd946a 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -113,21 +113,23 @@ function poco_load($cid,$uid = 0,$zcid = 0,$url = null) { $gender = $entry->gender; if(isset($entry->tags)) - $keywords = implode(", ", $entry->tags); + foreach($entry->tags as $tag) + $keywords = implode(", ", $tag); - poco_check($profile_url, $name, $network, $profile_photo, $about, $location, $gender, $keywords, $connect_url, $updated, $cid, $uid, $zcid); - - // Update the Friendica contacts. Diaspora is doing it via a message. (See include/diaspora.php) - if (($location != "") OR ($about != "") OR ($keywords != "") OR ($gender != "")) - q("UPDATE `contact` SET `location` = '%s', `about` = '%s', `keywords` = '%s', `gender` = '%s' - WHERE `nurl` = '%s' AND NOT `self` AND `network` = '%s'", - dbesc($location), - dbesc($about), - dbesc($keywords), - dbesc($gender), - dbesc(normalise_link($profile_url)), - dbesc(NETWORK_DFRN)); + if ($network != "") { + poco_check($profile_url, $name, $network, $profile_photo, $about, $location, $gender, $keywords, $connect_url, $updated, $cid, $uid, $zcid); + // Update the Friendica contacts. Diaspora is doing it via a message. (See include/diaspora.php) + if (($location != "") OR ($about != "") OR ($keywords != "") OR ($gender != "")) + q("UPDATE `contact` SET `location` = '%s', `about` = '%s', `keywords` = '%s', `gender` = '%s' + WHERE `nurl` = '%s' AND NOT `self` AND `network` = '%s'", + dbesc($location), + dbesc($about), + dbesc($keywords), + dbesc($gender), + dbesc(normalise_link($profile_url)), + dbesc(NETWORK_DFRN)); + } } logger("poco_load: loaded $total entries",LOGGER_DEBUG); diff --git a/mod/poco.php b/mod/poco.php index aa570ec34..70514606d 100644 --- a/mod/poco.php +++ b/mod/poco.php @@ -46,8 +46,8 @@ function poco_init(&$a) { if($justme) $sql_extra = " AND `contact`.`self` = 1 "; - else - $sql_extra = " AND `contact`.`self` = 0 "; +// else +// $sql_extra = " AND `contact`.`self` = 0 "; if($cid) $sql_extra = sprintf(" AND `contact`.`id` = %d ",intval($cid)); From e8bfa32b37665a6c88e3ae4b43880a7f84be084f Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Mon, 26 Jan 2015 01:07:15 +0100 Subject: [PATCH 6/7] Scrape now doesn't fetch page data when its twitter and Co./Update script that updates the contacts from the profile --- include/Scrape.php | 24 +++++++++++++++++++++-- include/socgraph.php | 45 +++++++++++++++++++++++++------------------- mod/profiles.php | 1 + update.php | 21 +++++++++++++++++++++ 4 files changed, 70 insertions(+), 21 deletions(-) diff --git a/include/Scrape.php b/include/Scrape.php index 31e2be263..7df86d9f2 100644 --- a/include/Scrape.php +++ b/include/Scrape.php @@ -356,6 +356,23 @@ function probe_url($url, $mode = PROBE_NORMAL) { $diaspora_key = ''; $has_lrdd = false; $email_conversant = false; + $connectornetworks = false; + $appnet = false; + + if (strpos($url,'twitter.com')) { + $connectornetworks = true; + $network = NETWORK_TWITTER; + } + + if (strpos($url,'www.facebook.com')) { + $connectornetworks = true; + $network = NETWORK_FACEBOOK; + } + + if (strpos($url,'alpha.app.net')) { + $appnet = true; + $network = NETWORK_APPNET; + } // Twitter is deactivated since twitter closed its old API //$twitter = ((strpos($url,'twitter.com') !== false) ? true : false); @@ -363,7 +380,7 @@ function probe_url($url, $mode = PROBE_NORMAL) { $at_addr = ((strpos($url,'@') !== false) ? true : false); - if((! $twitter) && (! $lastfm)) { + if((!$appnet) && (!$lastfm) && !$connectornetworks) { if(strpos($url,'mailto:') !== false && $at_addr) { $url = str_replace('mailto:','',$url); @@ -606,13 +623,16 @@ function probe_url($url, $mode = PROBE_NORMAL) { // Will leave it to others to figure out how to grab the avatar, which is on the $url page in the open graph meta links } - if($twitter || ! $poll) + if($appnet || ! $poll) $check_feed = true; if((! isset($vcard)) || (! x($vcard,'fn')) || (! $profile)) $check_feed = true; if(($at_addr) && (! count($links))) $check_feed = false; + if ($connectornetworks) + $check_feed = false; + if($check_feed) { $feedret = scrape_feed(($poll) ? $poll : $url); diff --git a/include/socgraph.php b/include/socgraph.php index f2ccd946a..07bbee32a 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -116,20 +116,18 @@ function poco_load($cid,$uid = 0,$zcid = 0,$url = null) { foreach($entry->tags as $tag) $keywords = implode(", ", $tag); - if ($network != "") { - poco_check($profile_url, $name, $network, $profile_photo, $about, $location, $gender, $keywords, $connect_url, $updated, $cid, $uid, $zcid); + poco_check($profile_url, $name, $network, $profile_photo, $about, $location, $gender, $keywords, $connect_url, $updated, $cid, $uid, $zcid); - // Update the Friendica contacts. Diaspora is doing it via a message. (See include/diaspora.php) - if (($location != "") OR ($about != "") OR ($keywords != "") OR ($gender != "")) - q("UPDATE `contact` SET `location` = '%s', `about` = '%s', `keywords` = '%s', `gender` = '%s' - WHERE `nurl` = '%s' AND NOT `self` AND `network` = '%s'", - dbesc($location), - dbesc($about), - dbesc($keywords), - dbesc($gender), - dbesc(normalise_link($profile_url)), - dbesc(NETWORK_DFRN)); - } + // Update the Friendica contacts. Diaspora is doing it via a message. (See include/diaspora.php) + if (($location != "") OR ($about != "") OR ($keywords != "") OR ($gender != "")) + q("UPDATE `contact` SET `location` = '%s', `about` = '%s', `keywords` = '%s', `gender` = '%s' + WHERE `nurl` = '%s' AND NOT `self` AND `network` = '%s'", + dbesc($location), + dbesc($about), + dbesc($keywords), + dbesc($gender), + dbesc(normalise_link($profile_url)), + dbesc(NETWORK_DFRN)); } logger("poco_load: loaded $total entries",LOGGER_DEBUG); @@ -147,6 +145,12 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca if ($profile_url == "") return $gcid; + $x = q("SELECT * FROM `gcontact` WHERE `nurl` = '%s' LIMIT 1", + dbesc(normalise_link($profile_url)) + ); + if(count($x)) + $network = $x[0]["network"]; + if (($network == "") OR ($name == "") OR ($profile_photo == "")) { require_once("include/Scrape.php"); @@ -156,17 +160,20 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca $profile_photo = $data["photo"]; } - if (!in_array($network, array(NETWORK_DFRN, NETWORK_OSTATUS, NETWORK_DIASPORA, NETWORK_STATUSNET))) - return $gcid; + if (count($x) AND ($x[0]["network"] == "") AND ($network != "")) { + q("UPDATE `gcontact` SET `network` = '%s' WHERE `nurl` = '%s'", + dbesc($network), + dbesc(normalise_link($profile_url)) + ); + } if (($name == "") OR ($profile_photo == "")) return $gcid; - logger("profile-check URL: ".$profile_url." name: ".$name." avatar: ".$profile_photo, LOGGER_DEBUG); + if (!in_array($network, array(NETWORK_DFRN, NETWORK_OSTATUS, NETWORK_DIASPORA, NETWORK_STATUSNET))) + return $gcid; - $x = q("SELECT * FROM `gcontact` WHERE `nurl` = '%s' LIMIT 1", - dbesc(normalise_link($profile_url)) - ); + logger("profile-check URL: ".$profile_url." name: ".$name." avatar: ".$profile_photo, LOGGER_DEBUG); if(count($x)) { $gcid = $x[0]['id']; diff --git a/mod/profiles.php b/mod/profiles.php index e71a4c603..befeedb2e 100644 --- a/mod/profiles.php +++ b/mod/profiles.php @@ -149,6 +149,7 @@ function profile_clean_keywords($keywords) { $cleaned = array(); foreach ($keywords as $keyword) { $keyword = trim(strtolower($keyword)); + $keyword = trim($keyword, "#"); if ($keyword != "") $cleaned[] = $keyword; } diff --git a/update.php b/update.php index 019e4cad3..e80de3526 100644 --- a/update.php +++ b/update.php @@ -1609,3 +1609,24 @@ All following update functions are ONLY for jobs that need to run AFTER the data Database changes are ONLY applied in the file include/dbstructure.php. */ + +function update_1177() { + require_once("mod/profiles.php"); + + $profiles = q("SELECT `uid`, `about`, `locality`, `pub_keywords`, `gender` FROM `profile` WHERE `is-default`"); + + foreach ($profiles AS $profile) { + if ($profile["about"].$profile["locality"].$profile["pub_keywords"].$profile["gender"] == "") + continue; + + $profile["pub_keywords"] = profile_clean_keywords($profile["pub_keywords"]); + + $r = q("UPDATE `contact` SET `about` = '%s', `location` = '%s', `keywords` = '%s', `gender` = '%s' WHERE `self` AND `uid` = %d", + dbesc($profile["about"]), + dbesc($profile["locality"]), + dbesc($profile["pub_keywords"]), + dbesc($profile["gender"]), + intval($profile["uid"]) + ); + } +} From 583c2ce13e710433d30490bae6b37e536c8e8596 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Tue, 27 Jan 2015 08:01:37 +0100 Subject: [PATCH 7/7] poco can now be used to query the global contacts. --- mod/poco.php | 58 ++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 18 deletions(-) diff --git a/mod/poco.php b/mod/poco.php index 70514606d..c8d8f72e8 100644 --- a/mod/poco.php +++ b/mod/poco.php @@ -22,7 +22,12 @@ function poco_init(&$a) { $format = (($_GET['format']) ? $_GET['format'] : 'json'); $justme = false; + $global = false; + if($a->argc > 1 && $a->argv[1] === '@global') { + $global = true; + $update_limit = date("Y-m-d H:i:s", time() - 30 * 86400); + } if($a->argc > 2 && $a->argv[2] === '@me') $justme = true; if($a->argc > 3 && $a->argv[3] === '@all') @@ -33,7 +38,7 @@ function poco_init(&$a) { $cid = intval($a->argv[4]); - if(! $system_mode) { + if(!$system_mode AND !$global) { $r = q("SELECT `user`.*,`profile`.`hide-friends` from user left join profile on `user`.`uid` = `profile`.`uid` where `user`.`nickname` = '%s' and `profile`.`is-default` = 1 limit 1", dbesc($user) @@ -52,7 +57,15 @@ function poco_init(&$a) { if($cid) $sql_extra = sprintf(" AND `contact`.`id` = %d ",intval($cid)); - if($system_mode) { + if(x($_GET,'updatedSince')) + $update_limit = date("Y-m-d H:i:s",strtotime($_GET['updatedSince'])); + + if ($global) { + $r = q("SELECT count(*) AS `total` FROM `gcontact` WHERE `updated` >= '%s' AND `network` IN ('%s')", + dbesc($update_limit), + dbesc(NETWORK_DFRN) + ); + } elseif($system_mode) { $r = q("SELECT count(*) AS `total` FROM `contact` WHERE `self` = 1 AND `network` IN ('%s', '%s', '%s', '%s', '') AND `uid` IN (SELECT `uid` FROM `pconfig` WHERE `cat` = 'system' AND `k` = 'suggestme' AND `v` = 1) ", dbesc(NETWORK_DFRN), @@ -60,8 +73,7 @@ function poco_init(&$a) { dbesc(NETWORK_OSTATUS), dbesc(NETWORK_STATUSNET) ); - } - else { + } else { $r = q("SELECT count(*) AS `total` FROM `contact` WHERE `uid` = %d AND `blocked` = 0 AND `pending` = 0 AND `hidden` = 0 AND `archive` = 0 AND `network` IN ('%s', '%s', '%s', '%s', '') $sql_extra", intval($user['uid']), @@ -82,7 +94,14 @@ function poco_init(&$a) { $itemsPerPage = ((x($_GET,'count') && intval($_GET['count'])) ? intval($_GET['count']) : $totalResults); - if($system_mode) { + if ($global) { + $r = q("SELECT * FROM `gcontact` WHERE `updated` > '%s' AND `network` IN ('%s') LIMIT %d, %d", + dbesc($update_limit), + dbesc(NETWORK_DFRN), + intval($startIndex), + intval($itemsPerPage) + ); + } elseif($system_mode) { $r = q("SELECT `contact`.*, `profile`.`about` AS `pabout`, `profile`.`locality` AS `plocation`, `profile`.`pub_keywords`, `profile`.`gender` AS `pgender` FROM `contact` INNER JOIN `profile` ON `profile`.`uid` = `contact`.`uid` WHERE `self` = 1 AND `network` IN ('%s', '%s', '%s', '%s', '') AND `profile`.`is-default` @@ -94,8 +113,7 @@ function poco_init(&$a) { intval($startIndex), intval($itemsPerPage) ); - } - else { + } else { $r = q("SELECT * FROM `contact` WHERE `uid` = %d AND `blocked` = 0 AND `pending` = 0 AND `hidden` = 0 AND `archive` = 0 AND `network` IN ('%s', '%s', '%s', '%s', '') $sql_extra LIMIT %d, %d", intval($user['uid']), @@ -107,13 +125,14 @@ function poco_init(&$a) { intval($itemsPerPage) ); } + $ret = array(); if(x($_GET,'sorted')) - $ret['sorted'] = 'false'; + $ret['sorted'] = false; if(x($_GET,'filtered')) - $ret['filtered'] = 'false'; - if(x($_GET,'updatedSince')) - $ret['updateSince'] = 'false'; + $ret['filtered'] = false; + if(x($_GET,'updatedSince') AND !$global) + $ret['updatedSince'] = false; $ret['startIndex'] = (string) $startIndex; $ret['itemsPerPage'] = (string) $itemsPerPage; @@ -178,16 +197,19 @@ function poco_init(&$a) { if($fields_ret['preferredUsername']) $entry['preferredUsername'] = $rr['nick']; if($fields_ret['updated']) { - $entry['updated'] = $rr['success_update']; + if (!$global) { + $entry['updated'] = $rr['success_update']; - if ($rr['name-date'] > $entry['updated']) - $entry['updated'] = $rr['name-date']; + if ($rr['name-date'] > $entry['updated']) + $entry['updated'] = $rr['name-date']; - if ($rr['uri-date'] > $entry['updated']) - $entry['updated'] = $rr['uri-date']; + if ($rr['uri-date'] > $entry['updated']) + $entry['updated'] = $rr['uri-date']; - if ($rr['avatar-date'] > $entry['updated']) - $entry['updated'] = $rr['avatar-date']; + if ($rr['avatar-date'] > $entry['updated']) + $entry['updated'] = $rr['avatar-date']; + } else + $entry['updated'] = $rr['updated']; $entry['updated'] = date("c", strtotime($entry['updated'])); }