<?php
class Http_MultiRequest
{
??? //要并行抓取的url 列表
??? private $urls = array();
??? //curl 的選項
??? private $options;
??? //構造函數
??? function __construct($options = array())
??? {
??????? $this->setOptions($options);
??? }
??? //設置url 列表
??? function setUrls($urls)
??? {
??????? $this->urls = $urls;
??????? return $this;
??? }
??? //設置選項
??? function setOptions($options)
??? {
??????? $options[CURLOPT_RETURNTRANSFER] = 1;
??????? if (isset($options['HTTP_POST']))
??????? {
??????????? curl_setopt($ch, CURLOPT_POST, 1);
??????????? curl_setopt($ch, CURLOPT_POSTFIELDS, $options['HTTP_POST']);
??????????? unset($options['HTTP_POST']);
??????? }
??????? if (!isset($options[CURLOPT_USERAGENT]))
??????? {
??????????? $options[CURLOPT_USERAGENT] = 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1;)';
??????? }
??????? if (!isset($options[CURLOPT_FOLLOWLOCATION]))
??????? {
??????????? $options[CURLOPT_FOLLOWLOCATION] = 1;
??????? }
??????? if (!isset($options[CURLOPT_HEADER]))
??????? {
??????????? $options[CURLOPT_HEADER] = 0;
??????? }
??????? $this->options = $options;
??? }
??? //并行抓取所有的內容
??? function exec()
??? {
??????? if(empty($this->urls) || !is_array($this->urls))
??????? {
??????????? return false;
??????? }
??????? $curl = $data = array();
??????? $mh = curl_multi_init();
??????? foreach($this->urls as $k => $v)
??????? {
??????????? $curl[$k] = $this->addHandle($mh, $v);
??????? }
??????? $this->execMulitHandle($mh);
??????? foreach($this->urls as $k => $v)
??????? {
??????????? $data[$k] = curl_multi_getcontent($curl[$k]);
??????????? curl_multi_remove_handle($mh, $curl[$k]);
??????? }
??????? curl_multi_close($mh);
??????? return $data;
??? }
??? //只抓取一個網頁的內容.
??? function execOne($url)
??? {
??????? if (empty($url)) {
??????????? return false;
??????? }
??????? $ch = curl_init($url);
??????? $this->setOneOption($ch);
??????? $content = curl_exec($ch);
??????? curl_close($ch);
??????? return $content;
??? }
??? //內部函數,設置某個handle 的選項
??? private function setOneOption($ch)
??? {
??????? curl_setopt_array($ch, $this->options);
??? }
??? //添加一個新的并行抓取 handle
??? private function addHandle($mh, $url)
??? {
??????? $ch = curl_init($url);
??????? $this->setOneOption($ch);
??????? curl_multi_add_handle($mh, $ch);
??????? return $ch;
??? }
??? //并行執行(這樣的寫法是一個常見的錯誤,我這里還是采用這樣的寫法,這個寫法
??? //下載一個小文件都可能導致cup占用100%, 并且,這個循環會運行10萬次以上
??? //這是一個典型的不懂原理產生的錯誤.這個錯誤在PHP官方的文檔上都相當的常見.)
??? private function execMulitHandle($mh)
??? {
??????? $running = null;
??????? do {
??????????? curl_multi_exec($mh, $running);
??????? } while ($running > 0);
??? }
}
/*下面是上面的類的一個測試的例子：*/
$urls = array("http://baidu.com", "http://baidu.com", "http://baidu.com", "http://baidu.com", "http://baidu.com", "http://baidu.com", "http://www.google.com", "http://www.sina.com.cn", );
$m = new Http_MultiRequest();
$t = microtime(true);
$m->setUrls($urls);
//parallel fetch(并行抓取):
$data = $m->exec();
$parallel_time = microtime(true) - $t;
echo $parallel_time . "\n";
$t = microtime(true);
//serial fetch(串行抓取):
foreach ($urls as $url)
{
??? $data[] = $m->execOne($url);
}
$serial_time = microtime(true) - $t;
echo $serial_time . "\n";