keith/friendica
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge remote-tracking branch 'upstream/2022.09-rc' into duplicates

Browse source
This commit is contained in:
Michael 2022-09-21 03:46:25 +00:00
commit 36668dfdb1
15 changed files with 276 additions and 580 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

673
composer.lock generated
View file

File diff suppressed because it is too large Load diff

1
database.sql
View file

@ -639,6 +639,7 @@ CREATE TABLE IF NOT EXISTS `fcontact` (
`network` char(4) NOT NULL DEFAULT '' COMMENT '',
`alias` varbinary(383) NOT NULL DEFAULT '' COMMENT '',
`pubkey` text COMMENT '',
`created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
`updated` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
`interacting_count` int unsigned DEFAULT 0 COMMENT 'Number of contacts this contact interactes with',
`interacted_count` int unsigned DEFAULT 0 COMMENT 'Number of contacts that interacted with this contact',

1
doc/database/db_fcontact.md
View file

@ -25,6 +25,7 @@ Fields
| network | | char(4) | NO | | | |
| alias | | varbinary(383) | NO | | | |
| pubkey | | text | YES | | NULL | |
| created | | datetime | NO | | 0001-01-01 00:00:00 | |
| updated | | datetime | NO | | 0001-01-01 00:00:00 | |
| interacting_count | Number of contacts this contact interactes with | int unsigned | YES | | 0 | |
| interacted_count | Number of contacts that interacted with this contact | int unsigned | YES | | 0 | |

2
src/App.php
View file

@ -586,7 +586,7 @@ class App
$this->profiler->set(microtime(true), 'classinit');
$moduleName = $this->args->getModuleName();
$page->setLogging($this->args->getCommand(), $this->args->getMethod());
$page->setLogging($this->args->getMethod(), $this->args->getModuleName(), $this->args->getCommand());
try {
// Missing DB connection: ERROR

17
src/App/Page.php
View file

@ -80,8 +80,9 @@ class Page implements ArrayAccess
private $basePath;
private $timestamp = 0;
private $command = '';
private $method = '';
private $module = '';
private $command = '';
/**
* @param string $basepath The Page basepath
@ -92,21 +93,25 @@ class Page implements ArrayAccess
$this->basePath = $basepath;
}
public function setLogging(string $command, string $method)
public function setLogging(string $method, string $module, string $command)
{
$this->command = $command;
$this->method = $method;
$this->module = $module;
$this->command = $command;
}
public function logRuntime(IManageConfigValues $config, string $origin = '')
{
if (in_array($this->command, $config->get('system', 'runtime_ignore'))) {
$ignore = $config->get('system', 'runtime_ignore');
if (in_array($this->module, $ignore) || in_array($this->command, $ignore)) {
return;
}
$runtime = number_format(microtime(true) - $this->timestamp, 3);
$signature = !empty($_SERVER['HTTP_SIGNATURE']);
$load = number_format(System::currentLoad(), 2);
$runtime = number_format(microtime(true) - $this->timestamp, 3);
if ($runtime > $config->get('system', 'runtime_loglimit')) {
Logger::debug('Runtime', ['method' => $this->method, 'command' => $this->command, 'runtime' => $runtime, 'origin' => $origin]);
Logger::debug('Runtime', ['method' => $this->method, 'module' => $this->module, 'runtime' => $runtime, 'load' => $load, 'origin' => $origin, 'signature' => $signature, 'request' => $_SERVER['REQUEST_URI'] ?? '']);
}
}

22
src/Model/Contact.php
View file

@ -277,7 +277,7 @@ class Contact
// Add internal fields
$removal = [];
if (!empty($fields)) {
foreach (['id', 'avatar', 'created', 'updated', 'last-update', 'success_update', 'failure_update', 'network'] as $internal) {
foreach (['id', 'next-update', 'network'] as $internal) {
if (!in_array($internal, $fields)) {
$fields[] = $internal;
$removal[] = $internal;
@ -307,9 +307,8 @@ class Contact
}
// Update the contact in the background if needed
$updated = max($contact['success_update'], $contact['created'], $contact['updated'], $contact['last-update'], $contact['failure_update']);
if (($updated < DateTimeFormat::utc('now -7 days')) && in_array($contact['network'], Protocol::FEDERATED) && !self::isLocalById($contact['id'])) {
Worker::add(PRIORITY_LOW, "UpdateContact", $contact['id']);
if (Probe::isProbable($contact['network']) && ($contact['next-update'] < DateTimeFormat::utcNow())) {
Worker::add(['priority' => PRIORITY_LOW, 'dont_fork' => true], 'UpdateContact', $contact['id']);
}
// Remove the internal fields
@ -962,7 +961,6 @@ class Contact
DI::cache()->delete(ActivityPub\Transmitter::CACHEKEY_CONTACTS . 'followers:' . $uid);
DI::cache()->delete(ActivityPub\Transmitter::CACHEKEY_CONTACTS . 'following:' . $uid);
DI::cache()->delete(NoScrape::CACHEKEY . $uid);
}
/**
@ -1221,11 +1219,15 @@ class Contact
return 0;
}
$contact = self::getByURL($url, false, ['id', 'network', 'uri-id'], $uid);
$contact = self::getByURL($url, false, ['id', 'network', 'uri-id', 'next-update'], $uid);
if (!empty($contact)) {
$contact_id = $contact['id'];
if (Probe::isProbable($contact['network']) && ($contact['next-update'] < DateTimeFormat::utcNow())) {
Worker::add(['priority' => PRIORITY_LOW, 'dont_fork' => true], 'UpdateContact', $contact['id']);
}
if (empty($update) && (!empty($contact['uri-id']) || is_bool($update))) {
Logger::debug('Contact found', ['url' => $url, 'uid' => $uid, 'update' => $update, 'cid' => $contact_id]);
return $contact_id;
@ -2524,7 +2526,7 @@ class Contact
$has_local_data = self::hasLocalData($id, $contact);
if (!in_array($ret['network'], array_merge(Protocol::FEDERATED, [Protocol::ZOT, Protocol::PHANTOM]))) {
if (!Probe::isProbable($ret['network'])) {
// Periodical checks are only done on federated contacts
$failed_next_update = null;
$success_next_update = null;
@ -3382,12 +3384,12 @@ class Contact
if (empty($url) || !is_string($url)) {
continue;
}
$contact = self::getByURL($url, false, ['id', 'updated']);
$contact = self::getByURL($url, false, ['id', 'network', 'next-update']);
if (empty($contact['id']) && Network::isValidHttpUrl($url)) {
Worker::add(PRIORITY_LOW, 'AddContact', 0, $url);
++$added;
} elseif ($contact['updated'] < DateTimeFormat::utc('now -7 days')) {
Worker::add(PRIORITY_LOW, 'UpdateContact', $contact['id']);
} elseif (Probe::isProbable($contact['network']) && ($contact['next-update'] < DateTimeFormat::utcNow())) {
Worker::add(['priority' => PRIORITY_LOW, 'dont_fork' => true], 'UpdateContact', $contact['id']);
++$updated;
} else {
++$unchanged;

29
src/Model/FContact.php
View file

@ -23,6 +23,7 @@ namespace Friendica\Model;
use Friendica\Core\Logger;
use Friendica\Core\Protocol;
use Friendica\Core\Worker;
use Friendica\Database\DBA;
use Friendica\DI;
use Friendica\Network\Probe;
@ -43,6 +44,7 @@ class FContact
*/
public static function getByURL(string $handle, $update = null): array
{
Logger::debug('Fetch fcontact', ['handle' => $handle, 'update' => $update]);
$person = DBA::selectFirst('fcontact', [], ['network' => Protocol::DIASPORA, 'addr' => $handle]);
if (!DBA::isResult($person)) {
$urls = [$handle, str_replace('http://', 'https://', $handle), Strings::normaliseLink($handle)];
@ -50,21 +52,17 @@ class FContact
}
if (DBA::isResult($person)) {
Logger::debug('In cache', ['person' => $person]);
Logger::debug('In cache', ['handle' => $handle]);
if (is_null($update)) {
// update record occasionally so it doesn't get stale
$d = strtotime($person['updated'] . ' +00:00');
if ($d < strtotime('now - 14 days')) {
$update = true;
}
if (empty($person['guid']) || empty($person['uri-id'])) {
$update = true;
$update = empty($person['guid']) || empty($person['uri-id']) || ($person['created'] <= DBA::NULL_DATETIME);
if (GServer::getNextUpdateDate(true, $person['created'], $person['updated'], false) < DateTimeFormat::utcNow()) {
Logger::debug('Start background update', ['handle' => $handle]);
Worker::add(['priority' => PRIORITY_LOW, 'dont_fork' => true], 'UpdateFContact', $handle);
}
}
} elseif (is_null($update)) {
$update = !DBA::isResult($person);
$update = true;
} else {
$person = [];
}
@ -95,7 +93,8 @@ class FContact
{
$uriid = ItemURI::insert(['uri' => $arr['url'], 'guid' => $arr['guid']]);
$contact = Contact::getByUriId($uriid, ['id']);
$fcontact = DBA::selectFirst('fcontact', ['created'], ['url' => $arr['url'], 'network' => $arr['network']]);
$contact = Contact::getByUriId($uriid, ['id', 'created']);
$apcontact = APContact::getByURL($arr['url'], false);
if (!empty($apcontact)) {
$interacted = $apcontact['following_count'];
@ -129,10 +128,14 @@ class FContact
'updated' => DateTimeFormat::utcNow(),
];
$condition = ['url' => $arr['url'], 'network' => $arr['network']];
if (empty($fcontact['created'])) {
$fields['created'] = $fields['updated'];
} elseif (!empty($contact['created']) && ($fcontact['created'] <= DBA::NULL_DATETIME)) {
$fields['created'] = $contact['created'];
}
$fields = DI::dbaDefinition()->truncateFieldsForTable('fcontact', $fields);
DBA::update('fcontact', $fields, $condition, true);
DBA::update('fcontact', $fields, ['url' => $arr['url'], 'network' => $arr['network']], true);
}
/**

34
src/Model/Item.php
View file

@ -1870,23 +1870,40 @@ class Item
return '';
}
// Convert attachments to links
$naked_body = BBCode::removeAttachment($item['body']);
if (empty($naked_body)) {
$languages = self::getLanguageArray(trim($item['title'] . "\n" . $item['body']), 3);
if (empty($languages)) {
return '';
}
return json_encode($languages);
}
/**
* Get a language array from a given text
*
* @param string $body
* @param integer $count
* @return array
*/
public static function getLanguageArray(string $body, int $count): array
{
// Convert attachments to links
$naked_body = BBCode::removeAttachment($body);
if (empty($naked_body)) {
return [];
}
// Remove links and pictures
$naked_body = BBCode::removeLinks($naked_body);
// Convert the title and the body to plain text
$naked_body = trim($item['title'] . "\n" . BBCode::toPlaintext($naked_body));
$naked_body = BBCode::toPlaintext($naked_body);
// Remove possibly remaining links
$naked_body = preg_replace(Strings::autoLinkRegEx(), '', $naked_body);
if (empty($naked_body)) {
return '';
return [];
}
$naked_body = self::getDominantLanguage($naked_body);
@ -1898,12 +1915,7 @@ class Item
$availableLanguages['fa'] = 'fa';
$ld = new Language(array_keys($availableLanguages));
$languages = $ld->detect($naked_body)->limit(0, 3)->close();
if (is_array($languages)) {
return json_encode($languages);
}
return '';
return $ld->detect($naked_body)->limit(0, $count)->close() ?: [];
}
/**

25
src/Module/NoScrape.php
View file

@ -22,11 +22,10 @@
namespace Friendica\Module;
use Friendica\BaseModule;
use Friendica\Core\Cache\Enum\Duration;
use Friendica\Core\Protocol;
use Friendica\Core\System;
use Friendica\Database\DBA;
use Friendica\DI;
use Friendica\Model\APContact;
use Friendica\Model\User;
/**
@ -36,8 +35,6 @@ use Friendica\Model\User;
*/
class NoScrape extends BaseModule
{
const CACHEKEY = 'noscrape:';
protected function rawContent(array $request = [])
{
$a = DI::app();
@ -58,12 +55,6 @@ class NoScrape extends BaseModule
System::jsonError(404, 'Profile not found');
}
$cachekey = self::CACHEKEY . $owner['uid'];
$result = DI::cache()->get($cachekey);
if (!is_null($result)) {
System::jsonExit($result);
}
$json_info = [
'addr' => $owner['addr'],
'nick' => $which,
@ -98,16 +89,8 @@ class NoScrape extends BaseModule
}
if (!($owner['hide-friends'] ?? false)) {
$json_info['contacts'] = DBA::count('contact',
[
'uid' => $owner['uid'],
'self' => 0,
'blocked' => 0,
'pending' => 0,
'hidden' => 0,
'archive' => 0,
'network' => [Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS]
]);
$apcontact = APContact::getByURL($owner['url']);
$json_info['contacts'] = max($apcontact['following_count'], $apcontact['followers_count']);
}
// We display the last activity (post or login), reduced to year and week number
@ -135,8 +118,6 @@ class NoScrape extends BaseModule
}
}
DI::cache()->set($cachekey, $json_info, Duration::DAY);
System::jsonExit($json_info);
}
}

11
src/Network/Probe.php
View file

@ -57,6 +57,17 @@ class Probe
private static $baseurl;
private static $istimeout;
/**
* Checks if the provided network can be probed
*
* @param string $network
* @return boolean
*/
public static function isProbable(string $network): bool
{
return (in_array($network, array_merge(Protocol::FEDERATED, [Protocol::ZOT, Protocol::PHANTOM])));
}
/**
* Remove stuff from an URI that doesn't belong there
*

2
src/Protocol/ActivityPub/Processor.php
View file

@ -91,7 +91,7 @@ class Processor
* @param string $body
* @return string
*/
protected static function normalizeMentionLinks(string $body): string
public static function normalizeMentionLinks(string $body): string
{
return preg_replace('%\[url=([^\[\]]*)]([#@!])(.*?)\[/url]%ism', '$2[url=$1]$3[/url]', $body);
}

22
src/Protocol/Relay.php
View file

@ -29,6 +29,7 @@ use Friendica\DI;
use Friendica\Model\APContact;
use Friendica\Model\Contact;
use Friendica\Model\GServer;
use Friendica\Model\Item;
use Friendica\Model\Post;
use Friendica\Model\Search;
use Friendica\Model\Tag;
@ -76,6 +77,8 @@ class Relay
return false;
}
$body = ActivityPub\Processor::normalizeMentionLinks($body);
$systemTags = [];
$userTags = [];
$denyTags = [];
@ -125,6 +128,25 @@ class Relay
}
}
$languages = [];
foreach (Item::getLanguageArray($body, 10) as $language => $reliability) {
if ($reliability > 0) {
$languages[] = $language;
}
}
Logger::debug('Got languages', ['languages' => $languages, 'body' => $body]);
if (!empty($languages)) {
if (in_array($languages[0], $config->get('system', 'relay_deny_languages'))) {
Logger::info('Unwanted language found - rejected', ['language' => $languages[0], 'network' => $network, 'url' => $url]);
return false;
}
} elseif ($config->get('system', 'relay_deny_undetected_language')) {
Logger::info('Undetected language found - rejected', ['body' => $body, 'network' => $network, 'url' => $url]);
return false;
}
if ($scope == self::SCOPE_ALL) {
Logger::info('Server accept all posts - accepted', ['network' => $network, 'url' => $url]);
return true;

8
src/Util/Profiler.php
View file

@ -152,13 +152,13 @@ class Profiler implements ContainerInterface
* Saves a timestamp for a value - f.e. a call
* Necessary for profiling Friendica
*
* @param int $timestamp the Timestamp
* @param float $timestamp the Timestamp
* @param string $value A value to profile
* @param string $callstack A callstack string, generated if absent
*
* @return void
*/
public function saveTimestamp(int $timestamp, string $value, string $callstack = '')
public function saveTimestamp(float $timestamp, string $value, string $callstack = '')
{
if (!$this->enabled) {
return;
@ -358,9 +358,9 @@ class Profiler implements ContainerInterface
* @throws NotFoundExceptionInterface No entry was found for **this** identifier.
* @throws ContainerExceptionInterface Error while retrieving the entry.
*
* @return int Entry.
* @return float Entry.
*/
public function get(string $id): int
public function get(string $id): float
{
if (!$this->has($id)) {
return 0;

1
static/dbstructure.config.php
View file

@ -697,6 +697,7 @@ return [
"network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => ""],
"alias" => ["type" => "varbinary(383)", "not null" => "1", "default" => "", "comment" => ""],
"pubkey" => ["type" => "text", "comment" => ""],
"created" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => ""],
"updated" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => ""],
"interacting_count" => ["type" => "int unsigned", "default" => 0, "comment" => "Number of contacts this contact interactes with"],
"interacted_count" => ["type" => "int unsigned", "default" => 0, "comment" => "Number of contacts that interacted with this contact"],

8
static/defaults.config.php
View file

@ -525,6 +525,14 @@ return [
// The authentication password for the redis database
'redis_password' => null,
// relay_deny_languages (Array)
// Array of languages (two digit format) that are rejected.
'relay_deny_languages' => [],
// relay_deny_undetected_language (Boolean)
// Deny undetected languages
'relay_deny_undetected_language' => false,
// session_handler (database|cache|native)
// Whether to use Cache to store session data or to use PHP native session storage.
'session_handler' => 'database',