1 |
85ad3d82
|
Assos Assos
|
<?php
|
2 |
|
|
|
3 |
|
|
/**
|
4 |
|
|
* @file
|
5 |
|
|
* Download via HTTP.
|
6 |
|
|
*
|
7 |
|
|
* Support caching, HTTP Basic Authentication, detection of RSS/Atom feeds,
|
8 |
|
|
* redirects.
|
9 |
|
|
*/
|
10 |
|
|
|
11 |
|
|
/**
|
12 |
|
|
* PCRE for finding the link tags in html.
|
13 |
|
|
*/
|
14 |
|
|
define('HTTP_REQUEST_PCRE_LINK_TAG', '/<link((?:[\x09\x0A\x0B\x0C\x0D\x20]+[^\x09\x0A\x0B\x0C\x0D\x20\x2F\x3E][^\x09\x0A\x0B\x0C\x0D\x20\x2F\x3D\x3E]*(?:[\x09\x0A\x0B\x0C\x0D\x20]*=[\x09\x0A\x0B\x0C\x0D\x20]*(?:"(?:[^"]*)"|\'(?:[^\']*)\'|(?:[^\x09\x0A\x0B\x0C\x0D\x20\x22\x27\x3E][^\x09\x0A\x0B\x0C\x0D\x20\x3E]*)?))?)*)[\x09\x0A\x0B\x0C\x0D\x20]*(>(.*)<\/link>|(\/)?>)/si');
|
15 |
|
|
|
16 |
|
|
/**
|
17 |
|
|
* PCRE for matching all the attributes in a tag.
|
18 |
|
|
*/
|
19 |
|
|
define('HTTP_REQUEST_PCRE_TAG_ATTRIBUTES', '/[\x09\x0A\x0B\x0C\x0D\x20]+([^\x09\x0A\x0B\x0C\x0D\x20\x2F\x3E][^\x09\x0A\x0B\x0C\x0D\x20\x2F\x3D\x3E]*)(?:[\x09\x0A\x0B\x0C\x0D\x20]*=[\x09\x0A\x0B\x0C\x0D\x20]*(?:"([^"]*)"|\'([^\']*)\'|([^\x09\x0A\x0B\x0C\x0D\x20\x22\x27\x3E][^\x09\x0A\x0B\x0C\x0D\x20\x3E]*)?))?/');
|
20 |
|
|
|
21 |
|
|
/**
|
22 |
|
|
* For cUrl specific errors.
|
23 |
|
|
*/
|
24 |
|
|
class HRCurlException extends Exception {}
|
25 |
|
|
|
26 |
|
|
/**
|
27 |
|
|
* Discovers RSS or atom feeds at the given URL.
|
28 |
|
|
*
|
29 |
|
|
* If document in given URL is an HTML document, function attempts to discover
|
30 |
|
|
* RSS or Atom feeds.
|
31 |
|
|
*
|
32 |
|
|
* @param string $url
|
33 |
|
|
* The url of the feed to retrieve.
|
34 |
|
|
* @param array $settings
|
35 |
|
|
* An optional array of settings. Valid options are: accept_invalid_cert.
|
36 |
|
|
*
|
37 |
|
|
* @return bool|string
|
38 |
|
|
* The discovered feed, or FALSE if the URL is not reachable or there was an
|
39 |
|
|
* error.
|
40 |
|
|
*/
|
41 |
|
|
function http_request_get_common_syndication($url, $settings = NULL) {
|
42 |
|
|
|
43 |
|
|
$accept_invalid_cert = isset($settings['accept_invalid_cert']) ? $settings['accept_invalid_cert'] : FALSE;
|
44 |
|
|
$download = http_request_get($url, NULL, NULL, $accept_invalid_cert);
|
45 |
|
|
|
46 |
|
|
// Cannot get the feed, return.
|
47 |
|
|
// http_request_get() always returns 200 even if its 304.
|
48 |
|
|
if ($download->code != 200) {
|
49 |
|
|
return FALSE;
|
50 |
|
|
}
|
51 |
|
|
|
52 |
|
|
// Drop the data into a seperate variable so all manipulations of the html
|
53 |
|
|
// will not effect the actual object that exists in the static cache.
|
54 |
|
|
// @see http_request_get.
|
55 |
|
|
$downloaded_string = $download->data;
|
56 |
|
|
// If this happens to be a feed then just return the url.
|
57 |
2c8c2b87
|
Assos Assos
|
if (isset($download->headers['content-type']) && http_request_is_feed($download->headers['content-type'], $downloaded_string)) {
|
58 |
85ad3d82
|
Assos Assos
|
return $url;
|
59 |
|
|
}
|
60 |
|
|
|
61 |
|
|
$discovered_feeds = http_request_find_feeds($downloaded_string);
|
62 |
|
|
foreach ($discovered_feeds as $feed_url) {
|
63 |
|
|
$absolute = http_request_create_absolute_url($feed_url, $url);
|
64 |
|
|
if (!empty($absolute)) {
|
65 |
|
|
// @TODO: something more intelligent?
|
66 |
|
|
return $absolute;
|
67 |
|
|
}
|
68 |
|
|
}
|
69 |
|
|
}
|
70 |
|
|
|
71 |
|
|
/**
|
72 |
|
|
* Get the content from the given URL.
|
73 |
|
|
*
|
74 |
|
|
* @param string $url
|
75 |
|
|
* A valid URL (not only web URLs).
|
76 |
|
|
* @param string $username
|
77 |
|
|
* If the URL uses authentication, supply the username.
|
78 |
|
|
* @param string $password
|
79 |
|
|
* If the URL uses authentication, supply the password.
|
80 |
|
|
* @param bool $accept_invalid_cert
|
81 |
|
|
* Whether to accept invalid certificates.
|
82 |
|
|
* @param integer $timeout
|
83 |
|
|
* Timeout in seconds to wait for an HTTP get request to finish.
|
84 |
|
|
*
|
85 |
|
|
* @return stdClass
|
86 |
|
|
* An object that describes the data downloaded from $url.
|
87 |
|
|
*/
|
88 |
|
|
function http_request_get($url, $username = NULL, $password = NULL, $accept_invalid_cert = FALSE, $timeout = NULL) {
|
89 |
|
|
// Intra-pagedownload cache, avoid to download the same content twice within
|
90 |
|
|
// one page download (it's possible, compatible and parse calls).
|
91 |
|
|
static $download_cache = array();
|
92 |
|
|
if (isset($download_cache[$url])) {
|
93 |
|
|
return $download_cache[$url];
|
94 |
|
|
}
|
95 |
|
|
|
96 |
|
|
// Determine request timeout.
|
97 |
|
|
$request_timeout = !empty($timeout) ? $timeout : variable_get('http_request_timeout', 30);
|
98 |
|
|
|
99 |
|
|
if (!$username && valid_url($url, TRUE)) {
|
100 |
|
|
// Handle password protected feeds.
|
101 |
|
|
$url_parts = parse_url($url);
|
102 |
|
|
if (!empty($url_parts['user'])) {
|
103 |
|
|
$password = $url_parts['pass'];
|
104 |
|
|
$username = $url_parts['user'];
|
105 |
|
|
}
|
106 |
|
|
}
|
107 |
|
|
|
108 |
|
|
$curl = http_request_use_curl();
|
109 |
|
|
|
110 |
|
|
// Only download and parse data if really needs refresh.
|
111 |
|
|
// Based on "Last-Modified" and "If-Modified-Since".
|
112 |
|
|
$headers = array();
|
113 |
|
|
if ($cache = cache_get('feeds_http_download_' . md5($url))) {
|
114 |
|
|
$last_result = $cache->data;
|
115 |
|
|
$last_headers = array_change_key_case($last_result->headers);
|
116 |
|
|
|
117 |
|
|
if (!empty($last_headers['etag'])) {
|
118 |
|
|
if ($curl) {
|
119 |
|
|
$headers[] = 'If-None-Match: ' . $last_headers['etag'];
|
120 |
|
|
}
|
121 |
|
|
else {
|
122 |
|
|
$headers['If-None-Match'] = $last_headers['etag'];
|
123 |
|
|
}
|
124 |
|
|
}
|
125 |
|
|
if (!empty($last_headers['last-modified'])) {
|
126 |
|
|
if ($curl) {
|
127 |
|
|
$headers[] = 'If-Modified-Since: ' . $last_headers['last-modified'];
|
128 |
|
|
}
|
129 |
|
|
else {
|
130 |
|
|
$headers['If-Modified-Since'] = $last_headers['last-modified'];
|
131 |
|
|
}
|
132 |
|
|
}
|
133 |
|
|
if (!empty($username) && !$curl) {
|
134 |
|
|
$headers['Authorization'] = 'Basic ' . base64_encode("$username:$password");
|
135 |
|
|
}
|
136 |
|
|
}
|
137 |
|
|
|
138 |
|
|
// Support the 'feed' and 'webcal' schemes by converting them into 'http'.
|
139 |
|
|
$url = strtr($url, array('feed://' => 'http://', 'webcal://' => 'http://'));
|
140 |
|
|
|
141 |
|
|
if ($curl) {
|
142 |
|
|
$headers[] = 'User-Agent: Drupal (+http://drupal.org/)';
|
143 |
|
|
$result = new stdClass();
|
144 |
|
|
$result->headers = array();
|
145 |
|
|
|
146 |
|
|
// Parse the URL and make sure we can handle the schema.
|
147 |
|
|
// cURL can only support either http:// or https://.
|
148 |
|
|
// CURLOPT_PROTOCOLS is only supported with cURL 7.19.4
|
149 |
|
|
$uri = parse_url($url);
|
150 |
|
|
if (!isset($uri['scheme'])) {
|
151 |
|
|
$result->error = 'missing schema';
|
152 |
|
|
$result->code = -1002;
|
153 |
|
|
}
|
154 |
|
|
else {
|
155 |
|
|
switch ($uri['scheme']) {
|
156 |
|
|
case 'http':
|
157 |
|
|
case 'https':
|
158 |
|
|
// Valid scheme.
|
159 |
|
|
break;
|
160 |
|
|
|
161 |
|
|
default:
|
162 |
|
|
$result->error = 'invalid schema ' . $uri['scheme'];
|
163 |
|
|
$result->code = -1003;
|
164 |
|
|
break;
|
165 |
|
|
}
|
166 |
|
|
}
|
167 |
|
|
|
168 |
|
|
// If the scheme was valid, continue to request the feed using cURL.
|
169 |
|
|
if (empty($result->error)) {
|
170 |
|
|
$download = curl_init($url);
|
171 |
|
|
curl_setopt($download, CURLOPT_FOLLOWLOCATION, TRUE);
|
172 |
|
|
if (!empty($username)) {
|
173 |
|
|
curl_setopt($download, CURLOPT_USERPWD, "{$username}:{$password}");
|
174 |
|
|
curl_setopt($download, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
|
175 |
|
|
}
|
176 |
|
|
curl_setopt($download, CURLOPT_HTTPHEADER, $headers);
|
177 |
|
|
curl_setopt($download, CURLOPT_HEADER, TRUE);
|
178 |
|
|
curl_setopt($download, CURLOPT_RETURNTRANSFER, TRUE);
|
179 |
|
|
curl_setopt($download, CURLOPT_ENCODING, '');
|
180 |
|
|
curl_setopt($download, CURLOPT_TIMEOUT, $request_timeout);
|
181 |
|
|
if ($accept_invalid_cert) {
|
182 |
|
|
curl_setopt($download, CURLOPT_SSL_VERIFYPEER, 0);
|
183 |
|
|
}
|
184 |
|
|
$header = '';
|
185 |
|
|
$data = curl_exec($download);
|
186 |
|
|
if (curl_error($download)) {
|
187 |
|
|
throw new HRCurlException(
|
188 |
|
|
t('cURL error (@code) @error for @url', array(
|
189 |
|
|
'@code' => curl_errno($download),
|
190 |
|
|
'@error' => curl_error($download),
|
191 |
|
|
'@url' => $url,
|
192 |
|
|
)), curl_errno($download)
|
193 |
|
|
);
|
194 |
|
|
}
|
195 |
|
|
|
196 |
|
|
$header_size = curl_getinfo($download, CURLINFO_HEADER_SIZE);
|
197 |
|
|
$header = substr($data, 0, $header_size - 1);
|
198 |
|
|
$result->data = substr($data, $header_size);
|
199 |
|
|
$headers = preg_split("/(\r\n){2}/", $header);
|
200 |
|
|
$header_lines = preg_split("/\r\n|\n|\r/", end($headers));
|
201 |
|
|
// Skip HTTP response status.
|
202 |
|
|
array_shift($header_lines);
|
203 |
|
|
|
204 |
|
|
while ($line = trim(array_shift($header_lines))) {
|
205 |
|
|
list($header, $value) = explode(':', $line, 2);
|
206 |
|
|
// Normalize the headers.
|
207 |
|
|
$header = strtolower($header);
|
208 |
|
|
|
209 |
|
|
if (isset($result->headers[$header]) && $header == 'set-cookie') {
|
210 |
|
|
// RFC 2109: the Set-Cookie response header comprises the token Set-
|
211 |
|
|
// Cookie:, followed by a comma-separated list of one or more cookies.
|
212 |
|
|
$result->headers[$header] .= ',' . trim($value);
|
213 |
|
|
}
|
214 |
|
|
else {
|
215 |
|
|
$result->headers[$header] = trim($value);
|
216 |
|
|
}
|
217 |
|
|
}
|
218 |
|
|
$result->code = curl_getinfo($download, CURLINFO_HTTP_CODE);
|
219 |
|
|
|
220 |
|
|
curl_close($download);
|
221 |
|
|
}
|
222 |
|
|
}
|
223 |
|
|
else {
|
224 |
|
|
$result = drupal_http_request($url, array('headers' => $headers, 'timeout' => $request_timeout));
|
225 |
|
|
$result->headers = isset($result->headers) ? $result->headers : array();
|
226 |
|
|
}
|
227 |
|
|
|
228 |
|
|
$result->code = isset($result->code) ? $result->code : 200;
|
229 |
|
|
|
230 |
|
|
// In case of 304 Not Modified try to return cached data.
|
231 |
|
|
if ($result->code == 304) {
|
232 |
|
|
|
233 |
|
|
if (isset($last_result)) {
|
234 |
|
|
$last_result->from_cache = TRUE;
|
235 |
|
|
return $last_result;
|
236 |
|
|
}
|
237 |
|
|
else {
|
238 |
|
|
// It's a tragedy, this file must exist and contain good data.
|
239 |
|
|
// In this case, clear cache and repeat.
|
240 |
|
|
cache_clear_all('feeds_http_download_' . md5($url), 'cache');
|
241 |
|
|
return http_request_get($url, $username, $password, $accept_invalid_cert, $request_timeout);
|
242 |
|
|
}
|
243 |
|
|
}
|
244 |
|
|
|
245 |
|
|
// Set caches.
|
246 |
|
|
cache_set('feeds_http_download_' . md5($url), $result);
|
247 |
|
|
$download_cache[$url] = $result;
|
248 |
|
|
|
249 |
|
|
return $result;
|
250 |
|
|
}
|
251 |
|
|
|
252 |
|
|
/**
|
253 |
|
|
* Decides if it's possible to use cURL or not.
|
254 |
|
|
*
|
255 |
|
|
* @return bool
|
256 |
|
|
* TRUE if cURL is available, FALSE otherwise.
|
257 |
|
|
*/
|
258 |
|
|
function http_request_use_curl() {
|
259 |
|
|
// Allow site administrators to choose to not use cURL.
|
260 |
|
|
if (variable_get('feeds_never_use_curl', FALSE)) {
|
261 |
|
|
return FALSE;
|
262 |
|
|
}
|
263 |
|
|
|
264 |
|
|
// Check availability of cURL on the system.
|
265 |
|
|
$basedir = ini_get("open_basedir");
|
266 |
|
|
return function_exists('curl_init') && !ini_get('safe_mode') && empty($basedir);
|
267 |
|
|
}
|
268 |
|
|
|
269 |
|
|
/**
|
270 |
|
|
* Clear cache for a specific URL.
|
271 |
|
|
*/
|
272 |
|
|
function http_request_clear_cache($url) {
|
273 |
|
|
cache_clear_all('feeds_http_download_' . md5($url), 'cache');
|
274 |
|
|
}
|
275 |
|
|
|
276 |
|
|
/**
|
277 |
|
|
* Returns if the provided $content_type is a feed.
|
278 |
|
|
*
|
279 |
|
|
* @param string $content_type
|
280 |
|
|
* The Content-Type header.
|
281 |
|
|
*
|
282 |
|
|
* @param string $data
|
283 |
|
|
* The actual data from the http request.
|
284 |
|
|
*
|
285 |
|
|
* @return bool
|
286 |
|
|
* Returns TRUE if this is a parsable feed.
|
287 |
|
|
*/
|
288 |
|
|
function http_request_is_feed($content_type, $data) {
|
289 |
|
|
$pos = strpos($content_type, ';');
|
290 |
|
|
if ($pos !== FALSE) {
|
291 |
|
|
$content_type = substr($content_type, 0, $pos);
|
292 |
|
|
}
|
293 |
|
|
$content_type = strtolower($content_type);
|
294 |
|
|
if (strpos($content_type, 'xml') !== FALSE) {
|
295 |
|
|
return TRUE;
|
296 |
|
|
}
|
297 |
|
|
|
298 |
|
|
// @TODO: Sometimes the content-type can be text/html but still be a valid
|
299 |
|
|
// feed.
|
300 |
|
|
return FALSE;
|
301 |
|
|
}
|
302 |
|
|
|
303 |
|
|
/**
|
304 |
|
|
* Finds potential feed tags in the HTML document.
|
305 |
|
|
*
|
306 |
|
|
* @param string $html
|
307 |
|
|
* The html string to search.
|
308 |
|
|
*
|
309 |
|
|
* @return array
|
310 |
|
|
* An array of href to feeds.
|
311 |
|
|
*/
|
312 |
|
|
function http_request_find_feeds($html) {
|
313 |
|
|
$matches = array();
|
314 |
|
|
preg_match_all(HTTP_REQUEST_PCRE_LINK_TAG, $html, $matches);
|
315 |
|
|
$links = $matches[1];
|
316 |
|
|
$valid_links = array();
|
317 |
|
|
|
318 |
|
|
// Build up all the links information.
|
319 |
|
|
foreach ($links as $link_tag) {
|
320 |
|
|
$attributes = array();
|
321 |
|
|
$candidate = array();
|
322 |
|
|
|
323 |
|
|
preg_match_all(HTTP_REQUEST_PCRE_TAG_ATTRIBUTES, $link_tag, $attributes, PREG_SET_ORDER);
|
324 |
|
|
foreach ($attributes as $attribute) {
|
325 |
|
|
// Find the key value pairs, attribute[1] is key and attribute[2] is the
|
326 |
|
|
// value.
|
327 |
|
|
if (!empty($attribute[1]) && !empty($attribute[2])) {
|
328 |
|
|
$candidate[drupal_strtolower($attribute[1])] = drupal_strtolower(decode_entities($attribute[2]));
|
329 |
|
|
}
|
330 |
|
|
}
|
331 |
|
|
|
332 |
|
|
// Examine candidate to see if it s a feed.
|
333 |
|
|
// @TODO: could/should use http_request_is_feed ??
|
334 |
|
|
if (isset($candidate['rel']) && $candidate['rel'] == 'alternate') {
|
335 |
|
|
if (isset($candidate['href']) && isset($candidate['type']) && strpos($candidate['type'], 'xml') !== FALSE) {
|
336 |
|
|
// All tests pass, its a valid candidate.
|
337 |
|
|
$valid_links[] = $candidate['href'];
|
338 |
|
|
}
|
339 |
|
|
}
|
340 |
|
|
}
|
341 |
|
|
|
342 |
|
|
return $valid_links;
|
343 |
|
|
}
|
344 |
|
|
|
345 |
|
|
/**
|
346 |
|
|
* Create an absolute url.
|
347 |
|
|
*
|
348 |
|
|
* @param string $url
|
349 |
|
|
* The href to transform.
|
350 |
|
|
* @param string $base_url
|
351 |
|
|
* The url to be used as the base for a relative $url.
|
352 |
|
|
*
|
353 |
|
|
* @return string
|
354 |
|
|
* An absolute url
|
355 |
|
|
*/
|
356 |
|
|
function http_request_create_absolute_url($url, $base_url) {
|
357 |
|
|
$url = trim($url);
|
358 |
|
|
if (valid_url($url, TRUE)) {
|
359 |
|
|
// Valid absolute url already.
|
360 |
|
|
return $url;
|
361 |
|
|
}
|
362 |
|
|
|
363 |
|
|
// Turn relative url into absolute.
|
364 |
|
|
if (valid_url($url, FALSE)) {
|
365 |
|
|
// Produces variables $scheme, $host, $user, $pass, $path, $query and
|
366 |
|
|
// $fragment.
|
367 |
|
|
$parsed_url = parse_url($base_url);
|
368 |
|
|
|
369 |
|
|
$path = dirname($parsed_url['path']);
|
370 |
|
|
|
371 |
|
|
// Adding to the existing path.
|
372 |
|
|
if ($url{0} == '/') {
|
373 |
|
|
$cparts = array_filter(explode("/", $url));
|
374 |
|
|
}
|
375 |
|
|
else {
|
376 |
|
|
// Backtracking from the existing path.
|
377 |
|
|
$cparts = array_merge(array_filter(explode("/", $path)), array_filter(explode("/", $url)));
|
378 |
|
|
foreach ($cparts as $i => $part) {
|
379 |
|
|
if ($part == '.') {
|
380 |
|
|
$cparts[$i] = NULL;
|
381 |
|
|
}
|
382 |
|
|
if ($part == '..') {
|
383 |
|
|
$cparts[$i - 1] = NULL;
|
384 |
|
|
$cparts[$i] = NULL;
|
385 |
|
|
}
|
386 |
|
|
}
|
387 |
|
|
$cparts = array_filter($cparts);
|
388 |
|
|
}
|
389 |
|
|
$path = implode("/", $cparts);
|
390 |
|
|
|
391 |
|
|
// Build the prefix to the path.
|
392 |
|
|
$absolute_url = '';
|
393 |
|
|
if (isset($parsed_url['scheme'])) {
|
394 |
|
|
$absolute_url = $parsed_url['scheme'] . '://';
|
395 |
|
|
}
|
396 |
|
|
|
397 |
|
|
if (isset($parsed_url['user'])) {
|
398 |
|
|
$absolute_url .= $parsed_url['user'];
|
399 |
|
|
if (isset($pass)) {
|
400 |
|
|
$absolute_url .= ':' . $parsed_url['pass'];
|
401 |
|
|
}
|
402 |
|
|
$absolute_url .= '@';
|
403 |
|
|
}
|
404 |
|
|
if (isset($parsed_url['host'])) {
|
405 |
|
|
$absolute_url .= $parsed_url['host'] . '/';
|
406 |
|
|
}
|
407 |
|
|
|
408 |
|
|
$absolute_url .= $path;
|
409 |
|
|
|
410 |
|
|
if (valid_url($absolute_url, TRUE)) {
|
411 |
|
|
return $absolute_url;
|
412 |
|
|
}
|
413 |
|
|
}
|
414 |
|
|
return FALSE;
|
415 |
|
|
} |