Changeset 606 for trunk/lib

Timestamp:

Jun 22, 2017 12:02:35 PM (7 years ago)

Author:

anonymous

Message:

Update httpExists to work better with servers that balk at curl or HEAD requests by using a real GET request and pretending to be Chrome

File:

• trunk/lib/Utilities.inc.php (modified) (3 diffs)

trunk/lib/Utilities.inc.php

@@ -1446 +1446 @@
 /*
 * Returns true if the given URL resolves to a resource with a HTTP 2xx or 3xx header response.
+* The download will abort if it retrieves >= 10KB of data to avoid downloading large files.
+* We couldn't use CURLOPT_NOBODY (a HEAD request) because some services don't behave without a GET request (ahem, BBC).
+* This function may not be very portable, if the server doesn't support CURLOPT_PROGRESSFUNCTION.
 *
 * @access   public
@@ -1451 +1454 @@
 * @return   bool            True if the resource exists, false otherwise.
 * @author   Quinn Comendant <quinn@strangecode.com>
-* @version  1.0
+* @version  2.0
 * @since    02 May 2015 15:10:09
 */
@@ -1458 +1461 @@
     $ch = curl_init($url);
     curl_setopt($ch, CURLOPT_TIMEOUT, 5);
-    curl_setopt($ch, CURLOPT_NOBODY, true);
     curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
     curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
+    curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36");
+    curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); // Don't pass through data to the browser.
+    curl_setopt($ch, CURLOPT_BUFFERSIZE, 128); // Frequent progress function calls.
+    curl_setopt($ch, CURLOPT_NOPROGRESS, false); // Required to use CURLOPT_PROGRESSFUNCTION.
+    curl_setopt($ch, CURLOPT_PROGRESSFUNCTION, function($ch, $dltot, $dlcur, $ultot, $ulcur){
+        // Return a non-zero value to abort the transfer. In which case, the transfer will set a CURLE_ABORTED_BY_CALLBACK error
+        // 10KB should be enough to catch a few 302 redirect headers and get to the actual content.
+        return ($dlcur > 10*1024) ? 1 : 0;
+    });
+
     curl_exec($ch);
     $http_code = curl_getinfo($ch, CURLINFO_HTTP_CODE);