diff --git a/mod/ostatus_subscribe.php b/mod/ostatus_subscribe.php index f4d5077c0..c1a9df7a5 100644 --- a/mod/ostatus_subscribe.php +++ b/mod/ostatus_subscribe.php @@ -24,6 +24,8 @@ use Friendica\Core\Protocol; use Friendica\DI; use Friendica\Model\APContact; use Friendica\Model\Contact; +use Friendica\Network\HTTPClient\Client\HttpClient; +use Friendica\Network\HTTPClient\Client\HttpClientOptions; use Friendica\Protocol\ActivityPub; function ostatus_subscribe_content(App $a) @@ -57,7 +59,7 @@ function ostatus_subscribe_content(App $a) $api = $contact['baseurl'] . '/api/'; // Fetching friends - $curlResult = DI::httpClient()->get($api . 'statuses/friends.json?screen_name=' . $contact['nick']); + $curlResult = DI::httpClient()->get($api . 'statuses/friends.json?screen_name=' . $contact['nick'], [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_JSON]); if (!$curlResult->isSuccess()) { DI::pConfig()->delete($uid, 'ostatus', 'legacy_contact'); diff --git a/mod/redir.php b/mod/redir.php index fe0da9e66..5a707d229 100644 --- a/mod/redir.php +++ b/mod/redir.php @@ -27,6 +27,8 @@ use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\Contact; use Friendica\Model\Profile; +use Friendica\Network\HTTPClient\Client\HttpClient; +use Friendica\Network\HTTPClient\Client\HttpClientOptions; use Friendica\Util\Strings; function redir_init(App $a) { @@ -142,7 +144,7 @@ function redir_magic($a, $cid, $url) } // Test for magic auth on the target system - $serverret = DI::httpClient()->get($basepath . '/magic'); + $serverret = DI::httpClient()->get($basepath . '/magic', [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_DEFAULT]); if ($serverret->isSuccess()) { $separator = strpos($target_url, '?') ? '&' : '?'; $target_url .= $separator . 'zrl=' . urlencode($visitor) . '&addr=' . urlencode($contact_url); diff --git a/src/Content/OEmbed.php b/src/Content/OEmbed.php index 4e1ae2946..0be220c51 100644 --- a/src/Content/OEmbed.php +++ b/src/Content/OEmbed.php @@ -32,6 +32,7 @@ use Friendica\Core\Renderer; use Friendica\Database\Database; use Friendica\Database\DBA; use Friendica\DI; +use Friendica\Network\HTTPClient\Client\HttpClient; use Friendica\Util\DateTimeFormat; use Friendica\Util\Network; use Friendica\Util\ParseUrl; @@ -97,7 +98,7 @@ class OEmbed if (!in_array($ext, $noexts)) { // try oembed autodiscovery - $html_text = DI::httpClient()->fetch($embedurl, 15, 'text/*'); + $html_text = DI::httpClient()->fetch($embedurl, 15, HttpClient::ACCEPT_HTML); if (!empty($html_text)) { $dom = new DOMDocument(); if (@$dom->loadHTML($html_text)) { diff --git a/src/Content/Text/BBCode.php b/src/Content/Text/BBCode.php index 2c17663c1..f30b21351 100644 --- a/src/Content/Text/BBCode.php +++ b/src/Content/Text/BBCode.php @@ -39,6 +39,7 @@ use Friendica\Model\Event; use Friendica\Model\Photo; use Friendica\Model\Post; use Friendica\Model\Tag; +use Friendica\Network\HTTPClient\Client\HttpClient; use Friendica\Network\HTTPClient\Client\HttpClientOptions; use Friendica\Object\Image; use Friendica\Protocol\Activity; @@ -501,11 +502,13 @@ class BBCode continue; } - $curlResult = DI::httpClient()->get($mtch[1]); + $curlResult = DI::httpClient()->get($mtch[1], [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_IMAGE]); if (!$curlResult->isSuccess()) { continue; } + Logger::debug('Got picture', ['Content-Type' => $curlResult->getHeader('Content-Type'), 'url' => $mtch[1]]); + $i = $curlResult->getBody(); $type = $curlResult->getContentType(); $type = Images::getMimeTypeByData($i, $mtch[1], $type); @@ -1096,7 +1099,7 @@ class BBCode $attributes[$field] = html_entity_decode($matches[2] ?? '', ENT_QUOTES, 'UTF-8'); } - $img_str = ' $value) { if (!empty($value)) { @@ -1214,7 +1217,7 @@ class BBCode $text = "[url=" . $match[2] . ']' . $match[2] . "[/url]"; // if its not a picture then look if its a page that contains a picture link - $body = DI::httpClient()->fetch($match[1]); + $body = DI::httpClient()->fetch($match[1], 0, HttpClient::ACCEPT_HTML); if (empty($body)) { DI::cache()->set($cache_key, $text); return $text; @@ -1290,7 +1293,7 @@ class BBCode } // if its not a picture then look if its a page that contains a picture link - $body = DI::httpClient()->fetch($match[1]); + $body = DI::httpClient()->fetch($match[1], 0, HttpClient::ACCEPT_HTML); if (empty($body)) { DI::cache()->set($cache_key, $text); return $text; @@ -1780,7 +1783,7 @@ class BBCode $text = preg_replace("/\[img\](.*?)\[\/img\]/ism", '' . DI::l10n()->t('Image/photo') . '', $text); $text = preg_replace("/\[zmg\](.*?)\[\/zmg\]/ism", '' . DI::l10n()->t('Image/photo') . '', $text); - + $text = self::convertImages($text, $simple_html, $uriid); $text = preg_replace("/\[crypt\](.*?)\[\/crypt\]/ism", '
' . DI::l10n()->t('Encrypted content') . '
', $text); @@ -1894,7 +1897,7 @@ class BBCode } else { $text = preg_replace("/([#@!])\[url\=(.*?)\](.*?)\[\/url\]/ism", '$1$3', $text); } - + if (!$for_plaintext) { if (in_array($simple_html, [self::OSTATUS, self::API, self::ACTIVITYPUB])) { $text = preg_replace_callback("/\[url\](.*?)\[\/url\]/ism", 'self::convertUrlForActivityPubCallback', $text); diff --git a/src/Core/Search.php b/src/Core/Search.php index 6c741821e..3e768b6c7 100644 --- a/src/Core/Search.php +++ b/src/Core/Search.php @@ -23,6 +23,7 @@ namespace Friendica\Core; use Friendica\DI; use Friendica\Model\Contact; +use Friendica\Network\HTTPClient\Client\HttpClient; use Friendica\Network\HTTPException; use Friendica\Network\HTTPClient\Client\HttpClientOptions; use Friendica\Object\Search\ContactResult; @@ -122,7 +123,7 @@ class Search $searchUrl .= '&page=' . $page; } - $resultJson = DI::httpClient()->fetch($searchUrl, 0, 'application/json'); + $resultJson = DI::httpClient()->fetch($searchUrl, 0, HttpClient::ACCEPT_JSON); $results = json_decode($resultJson, true); @@ -228,7 +229,7 @@ class Search $return = Contact::searchByName($search, $mode); } else { $p = $page > 1 ? 'p=' . $page : ''; - $curlResult = DI::httpClient()->get(self::getGlobalDirectory() . '/search/people?' . $p . '&q=' . urlencode($search), [HttpClientOptions::ACCEPT_CONTENT => ['application/json']]); + $curlResult = DI::httpClient()->get(self::getGlobalDirectory() . '/search/people?' . $p . '&q=' . urlencode($search), [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_JSON]); if ($curlResult->isSuccess()) { $searchResult = json_decode($curlResult->getBody(), true); if (!empty($searchResult['profiles'])) { diff --git a/src/Core/Storage/Type/ExternalResource.php b/src/Core/Storage/Type/ExternalResource.php index 767ec7957..23239965d 100644 --- a/src/Core/Storage/Type/ExternalResource.php +++ b/src/Core/Storage/Type/ExternalResource.php @@ -22,8 +22,10 @@ namespace Friendica\Core\Storage\Type; use Exception; +use Friendica\Core\Logger; use Friendica\Core\Storage\Exception\ReferenceStorageException; use Friendica\Core\Storage\Capability\ICanReadFromStorage; +use Friendica\Network\HTTPClient\Client\HttpClient; use Friendica\Util\HTTPSignature; /** @@ -52,11 +54,12 @@ class ExternalResource implements ICanReadFromStorage } try { - $fetchResult = HTTPSignature::fetchRaw($data->url, $data->uid, ['accept_content' => []]); + $fetchResult = HTTPSignature::fetchRaw($data->url, $data->uid, ['accept_content' => [HttpClient::ACCEPT_IMAGE]]); } catch (Exception $exception) { throw new ReferenceStorageException(sprintf('External resource failed to get %s', $reference), $exception->getCode(), $exception); } if ($fetchResult->isSuccess()) { + Logger::debug('Got picture', ['Content-Type' => $fetchResult->getHeader('Content-Type'), 'uid' => $data->uid, 'url' => $data->url]); return $fetchResult->getBody(); } else { throw new ReferenceStorageException(sprintf('External resource failed to get %s', $reference), $fetchResult->getReturnCode(), new Exception($fetchResult->getBody())); diff --git a/src/Model/GServer.php b/src/Model/GServer.php index 3cb4dc5f1..731175ea5 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -1946,7 +1946,7 @@ class GServer $protocols = ['activitypub', 'diaspora', 'dfrn', 'ostatus']; foreach ($protocols as $protocol) { $query = '{nodes(protocol:"' . $protocol . '"){host}}'; - $curlResult = DI::httpClient()->fetch('https://the-federation.info/graphql?query=' . urlencode($query)); + $curlResult = DI::httpClient()->fetch('https://the-federation.info/graphql?query=' . urlencode($query), 0, HttpClient::ACCEPT_JSON); if (!empty($curlResult)) { $data = json_decode($curlResult, true); if (!empty($data['data']['nodes'])) { diff --git a/src/Model/Photo.php b/src/Model/Photo.php index 8ba01e336..74171bd7b 100644 --- a/src/Model/Photo.php +++ b/src/Model/Photo.php @@ -32,6 +32,8 @@ use Friendica\Core\Storage\Exception\InvalidClassStorageException; use Friendica\Core\Storage\Exception\ReferenceStorageException; use Friendica\Core\Storage\Exception\StorageException; use Friendica\Core\Storage\Type\SystemResource; +use Friendica\Network\HTTPClient\Client\HttpClient; +use Friendica\Network\HTTPClient\Client\HttpClientOptions; use Friendica\Object\Image; use Friendica\Util\DateTimeFormat; use Friendica\Util\Images; @@ -497,7 +499,8 @@ class Photo $filename = basename($image_url); if (!empty($image_url)) { - $ret = DI::httpClient()->get($image_url); + $ret = DI::httpClient()->get($image_url, [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_IMAGE]); + Logger::debug('Got picture', ['Content-Type' => $ret->getHeader('Content-Type'), 'url' => $image_url]); $img_str = $ret->getBody(); $type = $ret->getContentType(); } else { @@ -912,7 +915,8 @@ class Photo { $filename = basename($image_url); if (!empty($image_url)) { - $ret = DI::httpClient()->get($image_url); + $ret = DI::httpClient()->get($image_url, [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_IMAGE]); + Logger::debug('Got picture', ['Content-Type' => $ret->getHeader('Content-Type'), 'url' => $image_url]); $img_str = $ret->getBody(); $type = $ret->getContentType(); } else { diff --git a/src/Model/Post/Media.php b/src/Model/Post/Media.php index 254b88115..660d5f542 100644 --- a/src/Model/Post/Media.php +++ b/src/Model/Post/Media.php @@ -30,6 +30,7 @@ use Friendica\DI; use Friendica\Model\Item; use Friendica\Model\Photo; use Friendica\Model\Post; +use Friendica\Network\HTTPClient\Client\HttpClient; use Friendica\Network\HTTPClient\Client\HttpClientOptions; use Friendica\Util\Images; use Friendica\Util\Network; @@ -168,11 +169,11 @@ class Media // Fetch the mimetype or size if missing. if (empty($media['mimetype']) || empty($media['size'])) { $timeout = DI::config()->get('system', 'xrd_timeout'); - $curlResult = DI::httpClient()->head($media['url'], [HttpClientOptions::TIMEOUT => $timeout]); + $curlResult = DI::httpClient()->head($media['url'], [HttpClientOptions::TIMEOUT => $timeout, HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_DEFAULT]); // Workaround for systems that can't handle a HEAD request if (!$curlResult->isSuccess() && ($curlResult->getReturnCode() == 405)) { - $curlResult = DI::httpClient()->get($media['url'], [HttpClientOptions::TIMEOUT => $timeout]); + $curlResult = DI::httpClient()->get($media['url'], [HttpClientOptions::TIMEOUT => $timeout, HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_DEFAULT]); } if ($curlResult->isSuccess()) { diff --git a/src/Model/Profile.php b/src/Model/Profile.php index d77935520..3a453cc87 100644 --- a/src/Model/Profile.php +++ b/src/Model/Profile.php @@ -35,6 +35,8 @@ use Friendica\Core\System; use Friendica\Core\Worker; use Friendica\Database\DBA; use Friendica\DI; +use Friendica\Network\HTTPClient\Client\HttpClient; +use Friendica\Network\HTTPClient\Client\HttpClientOptions; use Friendica\Network\HTTPException; use Friendica\Protocol\Activity; use Friendica\Protocol\Diaspora; @@ -749,7 +751,7 @@ class Profile $magic_path = $basepath . '/magic' . '?owa=1&dest=' . $dest . '&' . $addr_request; // We have to check if the remote server does understand /magic without invoking something - $serverret = DI::httpClient()->get($basepath . '/magic'); + $serverret = DI::httpClient()->get($basepath . '/magic', [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_DEFAULT]); if ($serverret->isSuccess()) { Logger::info('Doing magic auth for visitor ' . $my_url . ' to ' . $magic_path); System::externalRedirect($magic_path); diff --git a/src/Model/User.php b/src/Model/User.php index 93538bce6..497787d1c 100644 --- a/src/Model/User.php +++ b/src/Model/User.php @@ -34,6 +34,8 @@ use Friendica\Core\System; use Friendica\Core\Worker; use Friendica\Database\DBA; use Friendica\DI; +use Friendica\Network\HTTPClient\Client\HttpClient; +use Friendica\Network\HTTPClient\Client\HttpClientOptions; use Friendica\Security\TwoFactor\Model\AppSpecificPassword; use Friendica\Network\HTTPException; use Friendica\Object\Image; @@ -1150,8 +1152,9 @@ class User $photo_failure = false; $filename = basename($photo); - $curlResult = DI::httpClient()->get($photo); + $curlResult = DI::httpClient()->get($photo, [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_IMAGE]); if ($curlResult->isSuccess()) { + Logger::debug('Got picture', ['Content-Type' => $curlResult->getHeader('Content-Type'), 'url' => $photo]); $img_str = $curlResult->getBody(); $type = $curlResult->getContentType(); } else { diff --git a/src/Module/Admin/Summary.php b/src/Module/Admin/Summary.php index 500732a9f..4c6eebf33 100644 --- a/src/Module/Admin/Summary.php +++ b/src/Module/Admin/Summary.php @@ -32,6 +32,8 @@ use Friendica\DI; use Friendica\Core\Config\Factory\Config; use Friendica\Model\Register; use Friendica\Module\BaseAdmin; +use Friendica\Network\HTTPClient\Client\HttpClient; +use Friendica\Network\HTTPClient\Client\HttpClientOptions; use Friendica\Network\HTTPException\ServiceUnavailableException; use Friendica\Util\DateTimeFormat; @@ -248,7 +250,7 @@ class Summary extends BaseAdmin private static function checkSelfHostMeta() { // Fetch the host-meta to check if this really is a vital server - return DI::httpClient()->get(DI::baseUrl()->get() . '/.well-known/host-meta')->isSuccess(); + return DI::httpClient()->get(DI::baseUrl()->get() . '/.well-known/host-meta', [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_XRD_XML])->isSuccess(); } } diff --git a/src/Module/Debug/Feed.php b/src/Module/Debug/Feed.php index 470b61df2..cd80453dd 100644 --- a/src/Module/Debug/Feed.php +++ b/src/Module/Debug/Feed.php @@ -28,6 +28,7 @@ use Friendica\Core\Renderer; use Friendica\Model; use Friendica\Module\Response; use Friendica\Network\HTTPClient\Capability\ICanSendHttpRequests; +use Friendica\Network\HTTPClient\Client\HttpClient; use Friendica\Protocol; use Friendica\Util\Profiler; use Psr\Log\LoggerInterface; @@ -60,7 +61,7 @@ class Feed extends BaseModule $contact = Model\Contact::getByURLForUser($url, local_user(), null); - $xml = $this->httpClient->fetch($contact['poll']); + $xml = $this->httpClient->fetch($contact['poll'], 0, HttpClient::ACCEPT_FEED_XML); $import_result = Protocol\Feed::import($xml); diff --git a/src/Module/Magic.php b/src/Module/Magic.php index f6f3bbc59..16b349237 100644 --- a/src/Module/Magic.php +++ b/src/Module/Magic.php @@ -29,6 +29,7 @@ use Friendica\Database\Database; use Friendica\Model\Contact; use Friendica\Model\User; use Friendica\Network\HTTPClient\Capability\ICanSendHttpRequests; +use Friendica\Network\HTTPClient\Client\HttpClient; use Friendica\Network\HTTPClient\Client\HttpClientOptions; use Friendica\Util\HTTPSignature; use Friendica\Util\Profiler; diff --git a/src/Module/Photo.php b/src/Module/Photo.php index 3d3110fd8..0af2d2eb0 100644 --- a/src/Module/Photo.php +++ b/src/Module/Photo.php @@ -32,6 +32,7 @@ use Friendica\Model\Profile; use Friendica\Core\Storage\Type\ExternalResource; use Friendica\Core\Storage\Type\SystemResource; use Friendica\Model\User; +use Friendica\Network\HTTPClient\Client\HttpClient; use Friendica\Network\HTTPException; use Friendica\Network\HTTPException\NotModifiedException; use Friendica\Object\Image; @@ -308,7 +309,7 @@ class Photo extends BaseModule } $mimetext = ''; if (!empty($url)) { - $mime = ParseUrl::getContentType($url); + $mime = ParseUrl::getContentType($url, HttpClient::ACCEPT_IMAGE); if (!empty($mime)) { $mimetext = $mime[0] . '/' . $mime[1]; } else { @@ -317,6 +318,8 @@ class Photo extends BaseModule if (!empty($mimetext) && ($mime[0] != 'image') && ($mimetext != 'application/octet-stream')) { Logger::info('Unexpected Content-Type', ['mime' => $mimetext, 'url' => $url]); $mimetext = ''; + } if (!empty($mimetext)) { + Logger::debug('Expected Content-Type', ['mime' => $mimetext, 'url' => $url]); } } if (empty($mimetext)) { diff --git a/src/Module/Proxy.php b/src/Module/Proxy.php index 419289c39..dfc8113a4 100644 --- a/src/Module/Proxy.php +++ b/src/Module/Proxy.php @@ -25,6 +25,7 @@ use Friendica\BaseModule; use Friendica\Core\Logger; use Friendica\Core\System; use Friendica\DI; +use Friendica\Network\HTTPClient\Client\HttpClient; use Friendica\Network\HTTPException\NotModifiedException; use Friendica\Object\Image; use Friendica\Util\HTTPSignature; @@ -81,7 +82,7 @@ class Proxy extends BaseModule $request['url'] = str_replace(' ', '+', $request['url']); // Fetch the content with the local user - $fetchResult = HTTPSignature::fetchRaw($request['url'], local_user(), ['accept_content' => [], 'timeout' => 10]); + $fetchResult = HTTPSignature::fetchRaw($request['url'], local_user(), ['accept_content' => [HttpClient::ACCEPT_IMAGE], 'timeout' => 10]); $img_str = $fetchResult->getBody(); if (!$fetchResult->isSuccess() || empty($img_str)) { @@ -90,6 +91,8 @@ class Proxy extends BaseModule // stop. } + Logger::debug('Got picture', ['Content-Type' => $fetchResult->getHeader('Content-Type'), 'uid' => local_user(), 'image' => $request['url']]); + $mime = Images::getMimeTypeByData($img_str); $image = new Image($img_str, $mime); diff --git a/src/Network/HTTPClient/Client/HttpClient.php b/src/Network/HTTPClient/Client/HttpClient.php index 1f6839762..792cc7965 100644 --- a/src/Network/HTTPClient/Client/HttpClient.php +++ b/src/Network/HTTPClient/Client/HttpClient.php @@ -52,7 +52,9 @@ class HttpClient implements ICanSendHttpRequests const ACCEPT_JRD_JSON = 'application/jrd+json,application/json;q=0.9'; const ACCEPT_JSON = 'application/json,*/*;q=0.9'; const ACCEPT_JSON_AS = 'application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams"'; + const ACCEPT_MAGIC = 'application/magic-envelope+xml'; const ACCEPT_RSS_XML = 'application/rss+xml,text/xml;q=0.9,*/*;q=0.8'; + const ACCEPT_TEXT = 'text/plain,text/*;q=0.9,*/*;q=0.8'; const ACCEPT_VIDEO = 'video/mp4,video/*;q=0.9,*/*;q=0.8'; const ACCEPT_XRD_XML = 'application/xrd+xml,text/xml;q=0.9,*/*;q=0.8'; diff --git a/src/Network/Probe.php b/src/Network/Probe.php index 810885323..ceb684b70 100644 --- a/src/Network/Probe.php +++ b/src/Network/Probe.php @@ -170,7 +170,7 @@ class Probe Logger::info('Probing', ['host' => $host, 'ssl_url' => $ssl_url, 'url' => $url, 'callstack' => System::callstack(20)]); $xrd = null; - $curlResult = DI::httpClient()->get($ssl_url, [HttpClientOptions::TIMEOUT => $xrd_timeout, HttpClientOptions::ACCEPT_CONTENT => ['application/xrd+xml']]); + $curlResult = DI::httpClient()->get($ssl_url, [HttpClientOptions::TIMEOUT => $xrd_timeout, HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_XRD_XML]); $ssl_connection_error = ($curlResult->getErrorNumber() == CURLE_COULDNT_CONNECT) || ($curlResult->getReturnCode() == 0); if ($curlResult->isSuccess()) { $xml = $curlResult->getBody(); @@ -187,7 +187,7 @@ class Probe } if (!is_object($xrd) && !empty($url)) { - $curlResult = DI::httpClient()->get($url, [HttpClientOptions::TIMEOUT => $xrd_timeout, HttpClientOptions::ACCEPT_CONTENT => ['application/xrd+xml']]); + $curlResult = DI::httpClient()->get($url, [HttpClientOptions::TIMEOUT => $xrd_timeout, HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_XRD_XML]); $connection_error = ($curlResult->getErrorNumber() == CURLE_COULDNT_CONNECT) || ($curlResult->getReturnCode() == 0); if ($curlResult->isTimeout()) { Logger::info('Probing timeout', ['url' => $url]); @@ -429,7 +429,7 @@ class Probe */ private static function getHideStatus($url) { - $curlResult = DI::httpClient()->get($url, [HttpClientOptions::CONTENT_LENGTH => 1000000]); + $curlResult = DI::httpClient()->get($url, [HttpClientOptions::CONTENT_LENGTH => 1000000, HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_HTML]); if (!$curlResult->isSuccess()) { return false; } @@ -834,7 +834,8 @@ class Probe public static function pollZot($url, $data) { - $curlResult = DI::httpClient()->get($url); + // @todo Check if this endpoint might be queried via a POST + $curlResult = DI::httpClient()->get($url, [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_DEFAULT]); if ($curlResult->isTimeout()) { return $data; } @@ -1000,7 +1001,7 @@ class Probe */ private static function pollNoscrape($noscrape_url, $data) { - $curlResult = DI::httpClient()->get($noscrape_url); + $curlResult = DI::httpClient()->get($noscrape_url, [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_JSON]); if ($curlResult->isTimeout()) { self::$istimeout = true; return $data; @@ -1266,7 +1267,7 @@ class Probe */ private static function pollHcard($hcard_url, $data, $dfrn = false) { - $curlResult = DI::httpClient()->get($hcard_url); + $curlResult = DI::httpClient()->get($hcard_url, [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_HTML]); if ($curlResult->isTimeout()) { self::$istimeout = true; return []; @@ -1527,7 +1528,7 @@ class Probe $pubkey = substr($pubkey, 5); } } elseif (Strings::normaliseLink($pubkey) == 'http://') { - $curlResult = DI::httpClient()->get($pubkey); + $curlResult = DI::httpClient()->get($pubkey, [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_DEFAULT]); if ($curlResult->isTimeout()) { self::$istimeout = true; return $short ? false : []; @@ -1561,7 +1562,7 @@ class Probe } // Fetch all additional data from the feed - $curlResult = DI::httpClient()->get($data["poll"]); + $curlResult = DI::httpClient()->get($data["poll"], [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_FEED_XML]); if ($curlResult->isTimeout()) { self::$istimeout = true; return []; @@ -1613,7 +1614,7 @@ class Probe */ private static function pumpioProfileData($profile_link) { - $curlResult = DI::httpClient()->get($profile_link); + $curlResult = DI::httpClient()->get($profile_link, [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_HTML]); if (!$curlResult->isSuccess() || empty($curlResult->getBody())) { return []; } @@ -1826,7 +1827,7 @@ class Probe */ private static function feed($url, $probe = true) { - $curlResult = DI::httpClient()->get($url); + $curlResult = DI::httpClient()->get($url, [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_FEED_XML]); if ($curlResult->isTimeout()) { self::$istimeout = true; return []; @@ -2048,7 +2049,7 @@ class Probe return ''; } - $curlResult = DI::httpClient()->get($gserver['noscrape'] . '/' . $data['nick']); + $curlResult = DI::httpClient()->get($gserver['noscrape'] . '/' . $data['nick'], [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_JSON]); if ($curlResult->isSuccess() && !empty($curlResult->getBody())) { $noscrape = json_decode($curlResult->getBody(), true); @@ -2123,7 +2124,7 @@ class Probe private static function updateFromFeed(array $data) { // Search for the newest entry in the feed - $curlResult = DI::httpClient()->get($data['poll']); + $curlResult = DI::httpClient()->get($data['poll'], [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_ATOM_XML]); if (!$curlResult->isSuccess() || !$curlResult->getBody()) { return ''; } diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php index 9f7781d13..154d65f6f 100644 --- a/src/Protocol/Diaspora.php +++ b/src/Protocol/Diaspora.php @@ -41,6 +41,7 @@ use Friendica\Model\Mail; use Friendica\Model\Post; use Friendica\Model\Tag; use Friendica\Model\User; +use Friendica\Network\HTTPClient\Client\HttpClient; use Friendica\Network\Probe; use Friendica\Util\Crypto; use Friendica\Util\DateTimeFormat; @@ -1059,7 +1060,7 @@ class Diaspora Logger::info("Fetch post from ".$source_url); - $envelope = DI::httpClient()->fetch($source_url); + $envelope = DI::httpClient()->fetch($source_url, 0, HttpClient::ACCEPT_MAGIC); if ($envelope) { Logger::info("Envelope was fetched."); $x = self::verifyMagicEnvelope($envelope); diff --git a/src/Protocol/OStatus.php b/src/Protocol/OStatus.php index a2c9a8447..076602f9a 100644 --- a/src/Protocol/OStatus.php +++ b/src/Protocol/OStatus.php @@ -38,6 +38,7 @@ use Friendica\Model\ItemURI; use Friendica\Model\Post; use Friendica\Model\Tag; use Friendica\Model\User; +use Friendica\Network\HTTPClient\Client\HttpClient; use Friendica\Network\HTTPClient\Client\HttpClientOptions; use Friendica\Network\Probe; use Friendica\Util\DateTimeFormat; @@ -731,7 +732,7 @@ class OStatus self::$conv_list[$conversation] = true; - $curlResult = DI::httpClient()->get($conversation, [HttpClientOptions::ACCEPT_CONTENT => ['application/atom+xml', 'text/html']]); + $curlResult = DI::httpClient()->get($conversation, [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_ATOM_XML]); if (!$curlResult->isSuccess() || empty($curlResult->getBody())) { return; @@ -761,7 +762,7 @@ class OStatus } } if ($file != '') { - $conversation_atom = DI::httpClient()->get($attribute['href']); + $conversation_atom = DI::httpClient()->get($attribute['href'], [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_ATOM_XML]); if ($conversation_atom->isSuccess()) { $xml = $conversation_atom->getBody(); @@ -875,7 +876,7 @@ class OStatus return; } - $curlResult = DI::httpClient()->get($self); + $curlResult = DI::httpClient()->get($self, [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_ATOM_XML]); if (!$curlResult->isSuccess()) { return; @@ -925,7 +926,7 @@ class OStatus } $stored = false; - $curlResult = DI::httpClient()->get($related, [HttpClientOptions::ACCEPT_CONTENT => ['application/atom+xml', 'text/html']]); + $curlResult = DI::httpClient()->get($related, [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_ATOM_XML]); if (!$curlResult->isSuccess() || empty($curlResult->getBody())) { return; @@ -957,7 +958,7 @@ class OStatus } } if ($atom_file != '') { - $curlResult = DI::httpClient()->get($atom_file); + $curlResult = DI::httpClient()->get($atom_file, [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_ATOM_XML]); if ($curlResult->isSuccess()) { Logger::info('Fetched XML for URI ' . $related_uri); @@ -969,7 +970,7 @@ class OStatus // Workaround for older GNU Social servers if (($xml == '') && strstr($related, '/notice/')) { - $curlResult = DI::httpClient()->get(str_replace('/notice/', '/api/statuses/show/', $related) . '.atom'); + $curlResult = DI::httpClient()->get(str_replace('/notice/', '/api/statuses/show/', $related) . '.atom', [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_ATOM_XML]); if ($curlResult->isSuccess()) { Logger::info('GNU Social workaround to fetch XML for URI ' . $related_uri); @@ -980,7 +981,7 @@ class OStatus // Even more worse workaround for GNU Social ;-) if ($xml == '') { $related_guess = self::convertHref($related_uri); - $curlResult = DI::httpClient()->get(str_replace('/notice/', '/api/statuses/show/', $related_guess) . '.atom'); + $curlResult = DI::httpClient()->get(str_replace('/notice/', '/api/statuses/show/', $related_guess) . '.atom', [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_ATOM_XML]); if ($curlResult->isSuccess()) { Logger::info('GNU Social workaround 2 to fetch XML for URI ' . $related_uri); diff --git a/src/Protocol/Salmon.php b/src/Protocol/Salmon.php index aecc35e9f..12d05f9b3 100644 --- a/src/Protocol/Salmon.php +++ b/src/Protocol/Salmon.php @@ -23,6 +23,7 @@ namespace Friendica\Protocol; use Friendica\Core\Logger; use Friendica\DI; +use Friendica\Network\HTTPClient\Client\HttpClient; use Friendica\Network\Probe; use Friendica\Util\Crypto; use Friendica\Util\Strings; @@ -72,7 +73,7 @@ class Salmon $ret[$x] = substr($ret[$x], 5); } } elseif (Strings::normaliseLink($ret[$x]) == 'http://') { - $ret[$x] = DI::httpClient()->fetch($ret[$x]); + $ret[$x] = DI::httpClient()->fetch($ret[$x], 0, HttpClient::ACCEPT_DEFAULT); } } } diff --git a/src/Security/ExAuth.php b/src/Security/ExAuth.php index a402218f5..f324b0a96 100644 --- a/src/Security/ExAuth.php +++ b/src/Security/ExAuth.php @@ -57,6 +57,8 @@ use Friendica\Core\PConfig\Capability\IManagePersonalConfigValues; use Friendica\Database\Database; use Friendica\DI; use Friendica\Model\User; +use Friendica\Network\HTTPClient\Client\HttpClient; +use Friendica\Network\HTTPClient\Client\HttpClientOptions; use Friendica\Network\HTTPException; use Friendica\Util\PidFile; @@ -239,7 +241,7 @@ class ExAuth $url = ($ssl ? 'https' : 'http') . '://' . $host . '/noscrape/' . $user; - $curlResult = DI::httpClient()->get($url); + $curlResult = DI::httpClient()->get($url, [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_JSON]); if (!$curlResult->isSuccess()) { return false; diff --git a/src/Util/Images.php b/src/Util/Images.php index d7a58b52a..f1e85a37d 100644 --- a/src/Util/Images.php +++ b/src/Util/Images.php @@ -24,6 +24,7 @@ namespace Friendica\Util; use Friendica\Core\Logger; use Friendica\DI; use Friendica\Model\Photo; +use Friendica\Network\HTTPClient\Client\HttpClient; /** * Image utilities @@ -219,7 +220,7 @@ class Images } if (empty($img_str)) { - $img_str = DI::httpClient()->fetch($url, 4); + $img_str = DI::httpClient()->fetch($url, 4, HttpClient::ACCEPT_IMAGE); } if (!$img_str) { diff --git a/src/Util/ParseUrl.php b/src/Util/ParseUrl.php index 24be55cab..06ef3bdd0 100644 --- a/src/Util/ParseUrl.php +++ b/src/Util/ParseUrl.php @@ -29,6 +29,7 @@ use Friendica\Core\Logger; use Friendica\Database\Database; use Friendica\Database\DBA; use Friendica\DI; +use Friendica\Network\HTTPClient\Client\HttpClient; use Friendica\Network\HTTPException; use Friendica\Network\HTTPClient\Client\HttpClientOptions; @@ -54,16 +55,17 @@ class ParseUrl /** * Fetch the content type of the given url - * @param string $url URL of the page + * @param string $url URL of the page + * @param string $accept content-type to accept * @return array content type */ - public static function getContentType(string $url) + public static function getContentType(string $url, string $accept = HttpClient::ACCEPT_DEFAULT) { - $curlResult = DI::httpClient()->head($url); + $curlResult = DI::httpClient()->head($url, [HttpClientOptions::ACCEPT_CONTENT => $accept]); // Workaround for systems that can't handle a HEAD request if (!$curlResult->isSuccess() && ($curlResult->getReturnCode() == 405)) { - $curlResult = DI::httpClient()->get($url, [HttpClientOptions::CONTENT_LENGTH => 1000000]); + $curlResult = DI::httpClient()->get($url, [HttpClientOptions::CONTENT_LENGTH => 1000000, HttpClientOptions::ACCEPT_CONTENT => $accept]); } if (!$curlResult->isSuccess()) { @@ -220,7 +222,7 @@ class ParseUrl return $siteinfo; } - $curlResult = DI::httpClient()->get($url, [HttpClientOptions::CONTENT_LENGTH => 1000000]); + $curlResult = DI::httpClient()->get($url, [HttpClientOptions::CONTENT_LENGTH => 1000000, HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_HTML]); if (!$curlResult->isSuccess() || empty($curlResult->getBody())) { Logger::info('Empty body or error when fetching', ['url' => $url, 'success' => $curlResult->isSuccess(), 'code' => $curlResult->getReturnCode()]); return $siteinfo; diff --git a/src/Worker/CheckVersion.php b/src/Worker/CheckVersion.php index 14ebd8738..d803d9e3a 100644 --- a/src/Worker/CheckVersion.php +++ b/src/Worker/CheckVersion.php @@ -24,6 +24,7 @@ namespace Friendica\Worker; use Friendica\Core\Logger; use Friendica\Database\DBA; use Friendica\DI; +use Friendica\Network\HTTPClient\Client\HttpClient; /** * Check the git repository VERSION file and save the version to the DB @@ -54,7 +55,7 @@ class CheckVersion Logger::info("Checking VERSION from: ".$checked_url); // fetch the VERSION file - $gitversion = DBA::escape(trim(DI::httpClient()->fetch($checked_url))); + $gitversion = DBA::escape(trim(DI::httpClient()->fetch($checked_url, 0, HttpClient::ACCEPT_TEXT))); Logger::notice("Upstream VERSION is: ".$gitversion); DI::config()->set('system', 'git_friendica_version', $gitversion); diff --git a/src/Worker/Directory.php b/src/Worker/Directory.php index d499f778c..db199a856 100644 --- a/src/Worker/Directory.php +++ b/src/Worker/Directory.php @@ -26,6 +26,7 @@ use Friendica\Core\Logger; use Friendica\Core\Worker; use Friendica\Database\DBA; use Friendica\DI; +use Friendica\Network\HTTPClient\Client\HttpClient; /** * Sends updated profile data to the directory @@ -53,7 +54,7 @@ class Directory Logger::info('Updating directory: ' . $arr['url']); if (strlen($arr['url'])) { - DI::httpClient()->fetch($dir . '?url=' . bin2hex($arr['url'])); + DI::httpClient()->fetch($dir . '?url=' . bin2hex($arr['url']), 0, HttpClient::ACCEPT_DEFAULT); } return; diff --git a/src/Worker/NodeInfo.php b/src/Worker/NodeInfo.php index 4f18c7060..f55c494eb 100644 --- a/src/Worker/NodeInfo.php +++ b/src/Worker/NodeInfo.php @@ -24,6 +24,7 @@ namespace Friendica\Worker; use Friendica\Core\Logger; use Friendica\DI; use Friendica\Model\Nodeinfo as ModelNodeInfo; +use Friendica\Network\HTTPClient\Client\HttpClient; class NodeInfo { @@ -34,7 +35,7 @@ class NodeInfo // Now trying to register $url = 'http://the-federation.info/register/' . DI::baseUrl()->getHostname(); Logger::debug('Check registering url', ['url' => $url]); - $ret = DI::httpClient()->fetch($url); + $ret = DI::httpClient()->fetch($url, 0, HttpClient::ACCEPT_HTML); Logger::debug('Check registering answer', ['answer' => $ret]); Logger::info('end'); } diff --git a/src/Worker/OnePoll.php b/src/Worker/OnePoll.php index 40295f0c9..839ab6ce1 100644 --- a/src/Worker/OnePoll.php +++ b/src/Worker/OnePoll.php @@ -155,7 +155,7 @@ class OnePoll } $cookiejar = tempnam(System::getTempPath(), 'cookiejar-onepoll-'); - $curlResult = DI::httpClient()->get($contact['poll'], [HttpClientOptions::COOKIEJAR => $cookiejar], [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_FEED_XML]); + $curlResult = DI::httpClient()->get($contact['poll'], [HttpClientOptions::COOKIEJAR => $cookiejar, HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_FEED_XML]); unlink($cookiejar); if ($curlResult->isTimeout()) { diff --git a/src/Worker/PullDirectory.php b/src/Worker/PullDirectory.php index 04aab5016..0ea03a17f 100644 --- a/src/Worker/PullDirectory.php +++ b/src/Worker/PullDirectory.php @@ -24,6 +24,7 @@ namespace Friendica\Worker; use Friendica\Core\Logger; use Friendica\DI; use Friendica\Model\Contact; +use Friendica\Network\HTTPClient\Client\HttpClient; class PullDirectory { @@ -47,7 +48,7 @@ class PullDirectory Logger::info('Synchronization started.', ['now' => $now, 'directory' => $directory]); - $result = DI::httpClient()->fetch($directory . '/sync/pull/since/' . $now); + $result = DI::httpClient()->fetch($directory . '/sync/pull/since/' . $now, 0, HttpClient::ACCEPT_JSON); if (empty($result)) { Logger::info('Directory server return empty result.', ['directory' => $directory]); return; diff --git a/src/Worker/SearchDirectory.php b/src/Worker/SearchDirectory.php index 9d6426856..2344bb1bf 100644 --- a/src/Worker/SearchDirectory.php +++ b/src/Worker/SearchDirectory.php @@ -26,6 +26,7 @@ use Friendica\Core\Logger; use Friendica\Core\Search; use Friendica\DI; use Friendica\Model\Contact; +use Friendica\Network\HTTPClient\Client\HttpClient; class SearchDirectory { @@ -46,7 +47,7 @@ class SearchDirectory } } - $x = DI::httpClient()->fetch(Search::getGlobalDirectory() . '/lsearch?p=1&n=500&search=' . urlencode($search)); + $x = DI::httpClient()->fetch(Search::getGlobalDirectory() . '/lsearch?p=1&n=500&search=' . urlencode($search), 0, HttpClient::ACCEPT_JSON); $j = json_decode($x); if (!empty($j->results)) { diff --git a/src/Worker/UpdateServerDirectory.php b/src/Worker/UpdateServerDirectory.php index 10cc66cc1..248ab1121 100644 --- a/src/Worker/UpdateServerDirectory.php +++ b/src/Worker/UpdateServerDirectory.php @@ -26,6 +26,7 @@ use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\Contact; use Friendica\Model\GServer; +use Friendica\Network\HTTPClient\Client\HttpClient; class UpdateServerDirectory { @@ -45,7 +46,7 @@ class UpdateServerDirectory private static function discoverPoCo(array $gserver) { - $result = DI::httpClient()->fetch($gserver['poco'] . '?fields=urls'); + $result = DI::httpClient()->fetch($gserver['poco'] . '?fields=urls', 0, HttpClient::ACCEPT_JSON); if (empty($result)) { Logger::info('Empty result', ['url' => $gserver['url']]); return; @@ -78,7 +79,7 @@ class UpdateServerDirectory private static function discoverMastodonDirectory(array $gserver) { - $result = DI::httpClient()->fetch($gserver['url'] . '/api/v1/directory?order=new&local=true&limit=200&offset=0'); + $result = DI::httpClient()->fetch($gserver['url'] . '/api/v1/directory?order=new&local=true&limit=200&offset=0', 0, HttpClient::ACCEPT_JSON); if (empty($result)) { Logger::info('Empty result', ['url' => $gserver['url']]); return; diff --git a/src/Worker/UpdateServerPeers.php b/src/Worker/UpdateServerPeers.php index c14dccc3a..d2a127b82 100644 --- a/src/Worker/UpdateServerPeers.php +++ b/src/Worker/UpdateServerPeers.php @@ -25,6 +25,8 @@ use Friendica\Core\Logger; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\GServer; +use Friendica\Network\HTTPClient\Client\HttpClient; +use Friendica\Network\HTTPClient\Client\HttpClientOptions; use Friendica\Util\Strings; class UpdateServerPeers @@ -35,7 +37,7 @@ class UpdateServerPeers */ public static function execute(string $url) { - $ret = DI::httpClient()->get($url . '/api/v1/instance/peers'); + $ret = DI::httpClient()->get($url . '/api/v1/instance/peers', [HttpClientOptions::ACCEPT_CONTENT => HttpClient::ACCEPT_JSON]); if (!$ret->isSuccess() || empty($ret->getBody())) { Logger::info('Server is not reachable or does not offer the "peers" endpoint', ['url' => $url]); return;