利用curl_multi_init并行多线程处理网络数据

<code><span>for</span> (<span>$i</span>=<span>0</span>; <span>$i</span> < <span>100</span>; <span>$i</span>++)
{
      <span>$urls</span>[] = <span>"http://blog.csdn.net/lg_lin?wd="</span>.mt_rand(<span>10000</span>,<span>20000</span>);
}


<span>$t</span> = microtime(<span>true</span>);
<span>//单线程</span><span>foreach</span> (<span>$urls</span><span>as</span><span>$key</span> => <span>$value</span>)
{

    <span>$ch</span> = curl_init();
    curl_setopt(<span>$ch</span>, CURLOPT_SSL_VERIFYPEER, <span>FALSE</span>);
    curl_setopt(<span>$ch</span>, CURLOPT_RETURNTRANSFER, <span>TRUE</span>);
    curl_setopt(<span>$ch</span>, CURLOPT_TIMEOUT, <span>30</span>);
    curl_setopt(<span>$ch</span>, CURLOPT_URL, <span>$value</span>);
    <span>$response</span> =  curl_exec(<span>$ch</span>);

    curl_close(<span>$ch</span>);


    <span>$result_new</span>[<span>$key</span>] = <span>$response</span>;
}
var_dump(<span>$result_new</span>);

<span>$e</span> = microtime(<span>true</span>);
<span>echo</span><span>"For循环："</span>.(<span>$e</span>-<span>$t</span>).<span>"\n"</span>;

<span>/**
 **** 多线程****
 * 由于受网络带宽限制，一次性不能并发太多，可以分批处理
 * 总量超过100以后，效果就比单线程明显提高
*/</span><span>$p</span> = <span>5</span>;<span>//每次执行多少条</span><span>$ring</span> = ceil(count(<span>$urls</span>) / <span>$p</span>);

<span>for</span>(<span>$n</span> = <span>0</span>; <span>$n</span> < <span>$ring</span>; <span>$n</span>++)
{
    <span>$temp_url</span> = <span>array</span>();
    <span>$star</span> = <span>$n</span> * <span>$p</span>;
    <span>$end</span> = (<span>$n</span>+<span>1</span>) * <span>$p</span>;
    <span>for</span>(<span>$ii</span> = <span>$star</span>; <span>$ii</span> < <span>$end</span>; <span>$ii</span>++)
    {
        <span>if</span> (<span>isset</span>(<span>$urls</span>[<span>$ii</span>]))
        {
            <span>$temp_url</span>[] = <span>$urls</span>[<span>$ii</span>];
        }

    }

    <span>$mh</span> = curl_multi_init();
    <span>foreach</span> (<span>$temp_url</span><span>as</span><span>$i</span> => <span>$url</span>) {
        <span>$conn</span>[<span>$i</span>] = curl_init(<span>$url</span>);
        curl_setopt(<span>$conn</span>[<span>$i</span>], CURLOPT_USERAGENT, <span>$_SERVER</span>[<span>'HTTP_USER_AGENT'</span>]);
        curl_setopt(<span>$conn</span>[<span>$i</span>], CURLOPT_SSL_VERIFYPEER, <span>FALSE</span>);
        curl_setopt(<span>$conn</span>[<span>$i</span>], CURLOPT_RETURNTRANSFER, <span>TRUE</span>);
        curl_setopt(<span>$conn</span>[<span>$i</span>], CURLOPT_TIMEOUT, <span>30</span>);
        curl_setopt(<span>$conn</span>[<span>$i</span>], CURLOPT_URL, <span>$url</span>);
        curl_multi_add_handle(<span>$mh</span>,<span>$conn</span>[<span>$i</span>]);
    }

    <span>do</span>{
        curl_multi_exec(<span>$mh</span>, <span>$active</span>);
    } <span>while</span> (<span>$active</span>);

    <span>$active</span> = <span>null</span>;

    <span>foreach</span> (<span>$temp_url</span><span>as</span><span>$i</span> => <span>$url</span>) {
        <span>$data</span>[<span>$i</span>] = curl_multi_getcontent(<span>$conn</span>[<span>$i</span>]); <span>// 获得爬取的代码字符串</span>    }

    <span>foreach</span> (<span>$temp_url</span><span>as</span><span>$i</span> => <span>$url</span>) {
        curl_multi_remove_handle(<span>$mh</span>, <span>$conn</span>[<span>$i</span>]);
        curl_close(<span>$conn</span>[<span>$i</span>]);
    }

    curl_multi_close(<span>$mh</span>);


    var_dump(<span>$data</span>);

}

<span>$t</span> = microtime(<span>true</span>);

<span>echo</span><span>"多线程："</span>.(<span>$t</span>-<span>$e</span>).<span>"\n"</span>;


<span>exit</span>();</code>