Project

General

Profile

Paste
Download (12.8 KB) Statistics
| Branch: | Revision:

root / drupal7 / sites / all / modules / feeds / libraries / http_request.inc @ 2c8c2b87

1
<?php
2

    
3
/**
4
 * @file
5
 * Download via HTTP.
6
 *
7
 * Support caching, HTTP Basic Authentication, detection of RSS/Atom feeds,
8
 * redirects.
9
 */
10

    
11
/**
12
 * PCRE for finding the link tags in html.
13
 */
14
define('HTTP_REQUEST_PCRE_LINK_TAG', '/<link((?:[\x09\x0A\x0B\x0C\x0D\x20]+[^\x09\x0A\x0B\x0C\x0D\x20\x2F\x3E][^\x09\x0A\x0B\x0C\x0D\x20\x2F\x3D\x3E]*(?:[\x09\x0A\x0B\x0C\x0D\x20]*=[\x09\x0A\x0B\x0C\x0D\x20]*(?:"(?:[^"]*)"|\'(?:[^\']*)\'|(?:[^\x09\x0A\x0B\x0C\x0D\x20\x22\x27\x3E][^\x09\x0A\x0B\x0C\x0D\x20\x3E]*)?))?)*)[\x09\x0A\x0B\x0C\x0D\x20]*(>(.*)<\/link>|(\/)?>)/si');
15

    
16
/**
17
 * PCRE for matching all the attributes in a tag.
18
 */
19
define('HTTP_REQUEST_PCRE_TAG_ATTRIBUTES', '/[\x09\x0A\x0B\x0C\x0D\x20]+([^\x09\x0A\x0B\x0C\x0D\x20\x2F\x3E][^\x09\x0A\x0B\x0C\x0D\x20\x2F\x3D\x3E]*)(?:[\x09\x0A\x0B\x0C\x0D\x20]*=[\x09\x0A\x0B\x0C\x0D\x20]*(?:"([^"]*)"|\'([^\']*)\'|([^\x09\x0A\x0B\x0C\x0D\x20\x22\x27\x3E][^\x09\x0A\x0B\x0C\x0D\x20\x3E]*)?))?/');
20

    
21
/**
22
 * For cUrl specific errors.
23
 */
24
class HRCurlException extends Exception {}
25

    
26
/**
27
 * Discovers RSS or atom feeds at the given URL.
28
 *
29
 * If document in given URL is an HTML document, function attempts to discover
30
 * RSS or Atom feeds.
31
 *
32
 * @param string $url
33
 *   The url of the feed to retrieve.
34
 * @param array $settings
35
 *   An optional array of settings. Valid options are: accept_invalid_cert.
36
 *
37
 * @return bool|string
38
 *   The discovered feed, or FALSE if the URL is not reachable or there was an
39
 *   error.
40
 */
41
function http_request_get_common_syndication($url, $settings = NULL) {
42

    
43
  $accept_invalid_cert = isset($settings['accept_invalid_cert']) ? $settings['accept_invalid_cert'] : FALSE;
44
  $download = http_request_get($url, NULL, NULL, $accept_invalid_cert);
45

    
46
  // Cannot get the feed, return.
47
  // http_request_get() always returns 200 even if its 304.
48
  if ($download->code != 200) {
49
    return FALSE;
50
  }
51

    
52
  // Drop the data into a seperate variable so all manipulations of the html
53
  // will not effect the actual object that exists in the static cache.
54
  // @see http_request_get.
55
  $downloaded_string = $download->data;
56
  // If this happens to be a feed then just return the url.
57
  if (isset($download->headers['content-type']) && http_request_is_feed($download->headers['content-type'], $downloaded_string)) {
58
    return $url;
59
  }
60

    
61
  $discovered_feeds = http_request_find_feeds($downloaded_string);
62
  foreach ($discovered_feeds as $feed_url) {
63
    $absolute = http_request_create_absolute_url($feed_url, $url);
64
    if (!empty($absolute)) {
65
      // @TODO: something more intelligent?
66
      return $absolute;
67
    }
68
  }
69
}
70

    
71
/**
72
 * Get the content from the given URL.
73
 *
74
 * @param string $url
75
 *   A valid URL (not only web URLs).
76
 * @param string $username
77
 *   If the URL uses authentication, supply the username.
78
 * @param string $password
79
 *   If the URL uses authentication, supply the password.
80
 * @param bool $accept_invalid_cert
81
 *   Whether to accept invalid certificates.
82
 * @param integer $timeout
83
 *   Timeout in seconds to wait for an HTTP get request to finish.
84
 *
85
 * @return stdClass
86
 *   An object that describes the data downloaded from $url.
87
 */
88
function http_request_get($url, $username = NULL, $password = NULL, $accept_invalid_cert = FALSE, $timeout = NULL) {
89
  // Intra-pagedownload cache, avoid to download the same content twice within
90
  // one page download (it's possible, compatible and parse calls).
91
  static $download_cache = array();
92
  if (isset($download_cache[$url])) {
93
    return $download_cache[$url];
94
  }
95

    
96
  // Determine request timeout.
97
  $request_timeout = !empty($timeout) ? $timeout : variable_get('http_request_timeout', 30);
98

    
99
  if (!$username && valid_url($url, TRUE)) {
100
    // Handle password protected feeds.
101
    $url_parts = parse_url($url);
102
    if (!empty($url_parts['user'])) {
103
      $password = $url_parts['pass'];
104
      $username = $url_parts['user'];
105
    }
106
  }
107

    
108
  $curl = http_request_use_curl();
109

    
110
  // Only download and parse data if really needs refresh.
111
  // Based on "Last-Modified" and "If-Modified-Since".
112
  $headers = array();
113
  if ($cache = cache_get('feeds_http_download_' . md5($url))) {
114
    $last_result = $cache->data;
115
    $last_headers = array_change_key_case($last_result->headers);
116

    
117
    if (!empty($last_headers['etag'])) {
118
      if ($curl) {
119
        $headers[] = 'If-None-Match: ' . $last_headers['etag'];
120
      }
121
      else {
122
        $headers['If-None-Match'] = $last_headers['etag'];
123
      }
124
    }
125
    if (!empty($last_headers['last-modified'])) {
126
      if ($curl) {
127
        $headers[] = 'If-Modified-Since: ' . $last_headers['last-modified'];
128
      }
129
      else {
130
        $headers['If-Modified-Since'] = $last_headers['last-modified'];
131
      }
132
    }
133
    if (!empty($username) && !$curl) {
134
      $headers['Authorization'] = 'Basic ' . base64_encode("$username:$password");
135
    }
136
  }
137

    
138
  // Support the 'feed' and 'webcal' schemes by converting them into 'http'.
139
  $url = strtr($url, array('feed://' => 'http://', 'webcal://' => 'http://'));
140

    
141
  if ($curl) {
142
    $headers[] = 'User-Agent: Drupal (+http://drupal.org/)';
143
    $result = new stdClass();
144
    $result->headers = array();
145

    
146
    // Parse the URL and make sure we can handle the schema.
147
    // cURL can only support either http:// or https://.
148
    // CURLOPT_PROTOCOLS is only supported with cURL 7.19.4
149
    $uri = parse_url($url);
150
    if (!isset($uri['scheme'])) {
151
      $result->error = 'missing schema';
152
      $result->code = -1002;
153
    }
154
    else {
155
      switch ($uri['scheme']) {
156
        case 'http':
157
        case 'https':
158
          // Valid scheme.
159
          break;
160

    
161
        default:
162
          $result->error = 'invalid schema ' . $uri['scheme'];
163
          $result->code = -1003;
164
          break;
165
      }
166
    }
167

    
168
    // If the scheme was valid, continue to request the feed using cURL.
169
    if (empty($result->error)) {
170
      $download = curl_init($url);
171
      curl_setopt($download, CURLOPT_FOLLOWLOCATION, TRUE);
172
      if (!empty($username)) {
173
        curl_setopt($download, CURLOPT_USERPWD, "{$username}:{$password}");
174
        curl_setopt($download, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
175
      }
176
      curl_setopt($download, CURLOPT_HTTPHEADER, $headers);
177
      curl_setopt($download, CURLOPT_HEADER, TRUE);
178
      curl_setopt($download, CURLOPT_RETURNTRANSFER, TRUE);
179
      curl_setopt($download, CURLOPT_ENCODING, '');
180
      curl_setopt($download, CURLOPT_TIMEOUT, $request_timeout);
181
      if ($accept_invalid_cert) {
182
        curl_setopt($download, CURLOPT_SSL_VERIFYPEER, 0);
183
      }
184
      $header = '';
185
      $data = curl_exec($download);
186
      if (curl_error($download)) {
187
        throw new HRCurlException(
188
          t('cURL error (@code) @error for @url', array(
189
            '@code' => curl_errno($download),
190
            '@error' => curl_error($download),
191
            '@url' => $url,
192
          )), curl_errno($download)
193
        );
194
      }
195

    
196
      $header_size = curl_getinfo($download, CURLINFO_HEADER_SIZE);
197
      $header = substr($data, 0, $header_size - 1);
198
      $result->data = substr($data, $header_size);
199
      $headers = preg_split("/(\r\n){2}/", $header);
200
      $header_lines = preg_split("/\r\n|\n|\r/", end($headers));
201
      // Skip HTTP response status.
202
      array_shift($header_lines);
203

    
204
      while ($line = trim(array_shift($header_lines))) {
205
        list($header, $value) = explode(':', $line, 2);
206
        // Normalize the headers.
207
        $header = strtolower($header);
208

    
209
        if (isset($result->headers[$header]) && $header == 'set-cookie') {
210
          // RFC 2109: the Set-Cookie response header comprises the token Set-
211
          // Cookie:, followed by a comma-separated list of one or more cookies.
212
          $result->headers[$header] .= ',' . trim($value);
213
        }
214
        else {
215
          $result->headers[$header] = trim($value);
216
        }
217
      }
218
      $result->code = curl_getinfo($download, CURLINFO_HTTP_CODE);
219

    
220
      curl_close($download);
221
    }
222
  }
223
  else {
224
    $result = drupal_http_request($url, array('headers' => $headers, 'timeout' => $request_timeout));
225
    $result->headers = isset($result->headers) ? $result->headers : array();
226
  }
227

    
228
  $result->code = isset($result->code) ? $result->code : 200;
229

    
230
  // In case of 304 Not Modified try to return cached data.
231
  if ($result->code == 304) {
232

    
233
    if (isset($last_result)) {
234
      $last_result->from_cache = TRUE;
235
      return $last_result;
236
    }
237
    else {
238
      // It's a tragedy, this file must exist and contain good data.
239
      // In this case, clear cache and repeat.
240
      cache_clear_all('feeds_http_download_' . md5($url), 'cache');
241
      return http_request_get($url, $username, $password, $accept_invalid_cert, $request_timeout);
242
    }
243
  }
244

    
245
  // Set caches.
246
  cache_set('feeds_http_download_' . md5($url), $result);
247
  $download_cache[$url] = $result;
248

    
249
  return $result;
250
}
251

    
252
/**
253
 * Decides if it's possible to use cURL or not.
254
 *
255
 * @return bool
256
 *   TRUE if cURL is available, FALSE otherwise.
257
 */
258
function http_request_use_curl() {
259
  // Allow site administrators to choose to not use cURL.
260
  if (variable_get('feeds_never_use_curl', FALSE)) {
261
    return FALSE;
262
  }
263

    
264
  // Check availability of cURL on the system.
265
  $basedir = ini_get("open_basedir");
266
  return function_exists('curl_init') && !ini_get('safe_mode') && empty($basedir);
267
}
268

    
269
/**
270
 * Clear cache for a specific URL.
271
 */
272
function http_request_clear_cache($url) {
273
  cache_clear_all('feeds_http_download_' . md5($url), 'cache');
274
}
275

    
276
/**
277
 * Returns if the provided $content_type is a feed.
278
 *
279
 * @param string $content_type
280
 *   The Content-Type header.
281
 *
282
 * @param string $data
283
 *   The actual data from the http request.
284
 *
285
 * @return bool
286
 *   Returns TRUE if this is a parsable feed.
287
 */
288
function http_request_is_feed($content_type, $data) {
289
  $pos = strpos($content_type, ';');
290
  if ($pos !== FALSE) {
291
    $content_type = substr($content_type, 0, $pos);
292
  }
293
  $content_type = strtolower($content_type);
294
  if (strpos($content_type, 'xml') !== FALSE) {
295
    return TRUE;
296
  }
297

    
298
  // @TODO: Sometimes the content-type can be text/html but still be a valid
299
  // feed.
300
  return FALSE;
301
}
302

    
303
/**
304
 * Finds potential feed tags in the HTML document.
305
 *
306
 * @param string $html
307
 *   The html string to search.
308
 *
309
 * @return array
310
 *   An array of href to feeds.
311
 */
312
function http_request_find_feeds($html) {
313
  $matches = array();
314
  preg_match_all(HTTP_REQUEST_PCRE_LINK_TAG, $html, $matches);
315
  $links = $matches[1];
316
  $valid_links = array();
317

    
318
  // Build up all the links information.
319
  foreach ($links as $link_tag) {
320
    $attributes = array();
321
    $candidate = array();
322

    
323
    preg_match_all(HTTP_REQUEST_PCRE_TAG_ATTRIBUTES, $link_tag, $attributes, PREG_SET_ORDER);
324
    foreach ($attributes as $attribute) {
325
      // Find the key value pairs, attribute[1] is key and attribute[2] is the
326
      // value.
327
      if (!empty($attribute[1]) && !empty($attribute[2])) {
328
        $candidate[drupal_strtolower($attribute[1])] = drupal_strtolower(decode_entities($attribute[2]));
329
      }
330
    }
331

    
332
    // Examine candidate to see if it s a feed.
333
    // @TODO: could/should use http_request_is_feed ??
334
    if (isset($candidate['rel']) && $candidate['rel'] == 'alternate') {
335
      if (isset($candidate['href']) && isset($candidate['type']) && strpos($candidate['type'], 'xml') !== FALSE) {
336
        // All tests pass, its a valid candidate.
337
        $valid_links[] = $candidate['href'];
338
      }
339
    }
340
  }
341

    
342
  return $valid_links;
343
}
344

    
345
/**
346
 * Create an absolute url.
347
 *
348
 * @param string $url
349
 *   The href to transform.
350
 * @param string $base_url
351
 *   The url to be used as the base for a relative $url.
352
 *
353
 * @return string
354
 *   An absolute url
355
 */
356
function http_request_create_absolute_url($url, $base_url) {
357
  $url = trim($url);
358
  if (valid_url($url, TRUE)) {
359
    // Valid absolute url already.
360
    return $url;
361
  }
362

    
363
  // Turn relative url into absolute.
364
  if (valid_url($url, FALSE)) {
365
    // Produces variables $scheme, $host, $user, $pass, $path, $query and
366
    // $fragment.
367
    $parsed_url = parse_url($base_url);
368

    
369
    $path = dirname($parsed_url['path']);
370

    
371
    // Adding to the existing path.
372
    if ($url{0} == '/') {
373
      $cparts = array_filter(explode("/", $url));
374
    }
375
    else {
376
      // Backtracking from the existing path.
377
      $cparts = array_merge(array_filter(explode("/", $path)), array_filter(explode("/", $url)));
378
      foreach ($cparts as $i => $part) {
379
        if ($part == '.') {
380
          $cparts[$i] = NULL;
381
        }
382
        if ($part == '..') {
383
          $cparts[$i - 1] = NULL;
384
          $cparts[$i] = NULL;
385
        }
386
      }
387
      $cparts = array_filter($cparts);
388
    }
389
    $path = implode("/", $cparts);
390

    
391
    // Build the prefix to the path.
392
    $absolute_url = '';
393
    if (isset($parsed_url['scheme'])) {
394
      $absolute_url = $parsed_url['scheme'] . '://';
395
    }
396

    
397
    if (isset($parsed_url['user'])) {
398
      $absolute_url .= $parsed_url['user'];
399
      if (isset($pass)) {
400
        $absolute_url .= ':' . $parsed_url['pass'];
401
      }
402
      $absolute_url .= '@';
403
    }
404
    if (isset($parsed_url['host'])) {
405
      $absolute_url .= $parsed_url['host'] . '/';
406
    }
407

    
408
    $absolute_url .= $path;
409

    
410
    if (valid_url($absolute_url, TRUE)) {
411
      return $absolute_url;
412
    }
413
  }
414
  return FALSE;
415
}