CURL+DOM采集小样

个人感觉效率非常高的采集方式,但DOMdocument好像无法采集到带标签的内容,采集纯文本非常快.
大部分功能都没来得及做就被否了,只能提供大家参考用.
可以看到CURL代理访问+ZEND_DOM采集用法.
主要看CollectGoodsController.class.php就可以了.//获取店铺栏目<br>     public function getShopCate($shop_url="",$continue=0){<br>         $source=file_get_contents('Offline/shops.htm');<br>         $shops_id=1;<br>         $mall_id=1;<br>         $cate=M('goods_category')-&gt;where(array('shops_id'=&gt;$shops_id))-&gt;find();<br>         if(!empty($cate)){<br>             return fasle;//采集店铺栏目已存在<br>             //$this-&gt;error('采集店铺栏目已存在');<br>         }<br><br>         import('@.Tao.TaoHttp','','.php');<br>         $Http= new \TaoHttp();<br>         $shop_html=$Http-&gt;encoding($source);<br><br>         $shop_category_rule=D('CollectGoods')-&gt;getRule($mall_id,'shop_category');<br>         import('@.Tao.Dom.Query','','.php');<br>         $Dom= new \Zend_Dom_Query($shop_html);<br>         $shop_category=$Dom-&gt;query($shop_category_rule);<br>         if(count($shop_category)==0){<br>             return false;//采集不到店铺栏目<br>             //$this-&gt;error('采集不到店铺栏目');<br>         }<br>         $result=array();<br>         foreach ($shop_category as $key =&gt; $value) {<br>             $result[$key]['url']=$value-&gt;getAttribute('href');<br>             $result[$key]['name']=trim($value-&gt;nodeValue);<br>         }<br>         unset($result[0]);<br>         <br>         $data=array();<br>         $time=time();<br>         $cate_url=array();<br>         foreach ($result as $value) {<br>             $dataTmp=array(<br>                 'shops_id'=&gt; $shops_id,<br>                 'cate_name'=&gt; $value['name'],<br>                 'cate_url'=&gt; $value['url'],<br>                 'collect_time'=&gt;$time,<br>             );<br>             $cate_url[]=$value['url'];<br>             $data[]=$dataTmp;<br>         }<br>         M('goods_category')-&gt;addAll($data);<br>         return true;//采集店铺栏目成功<br>     }    //获取店铺一个栏目商品<br>     //http://localhost/TaoGoods/index.php?m=Taogoods&amp;c=CollectGoods&amp;a=getShopGoods&amp;cate_id=3<br>     public function getShopGoods($cate_id=0){<br>         if($cate_id==0){return false;}<br>         $goods_time=M('goods')-&gt;where(array('cate_id'=&gt;$cate_id))-&gt;getField('collect_time');<br><br>         if($goods_time){<br>             if($goods_time + 86400*$this-&gt;day &gt; time()){<br>                 $this-&gt;error('15天内请勿重复采集',U('index'));<br>             }<br>             $this-&gt;error('采集店铺栏目下货品已存在',U('index'));<br>         }<br><br>         $cate_data=M('goods_category')-&gt;find($cate_id);<br>         $shops_id=$cate_data['shops_id'];<br>         $cate_id=$cate_data['id'];<br>         $mall_id=$cate_data['mall_id'];<br><br>         import('@.Tao.TaoHttp','','.php');<br>         $Http= new \TaoHttp();<br>         $source=$Http-&gt;get($cate_data['cate_url']);<br>         $shop_html=$Http-&gt;encoding($source);<br><br>         $cate_rule=D('CollectGoods')-&gt;getRule($mall_id);<br>         import('@.Tao.Dom.Query','','.php');<br>         $Dom= new \Zend_Dom_Query($shop_html);<br>         $cate_imgs=$Dom-&gt;query($cate_rule['shop_category_goods_img']);<br>         $cate_names=$Dom-&gt;query($cate_rule['shop_category_goods_name']);<br>         $cate_sales=$Dom-&gt;query($cate_rule['shop_category_goods_sale']);<br>         $cate_cprices=$Dom-&gt;query($cate_rule['shop_category_goods_cprice']);<br>         //$cate_sprices=$Dom-&gt;query($cate_rule['shop_category_goods_sprice']);<br>         <br>         $num=count($cate_names);<br><br>         $time=time();<br>         $result=array();<br>         for ($i=0; $i              $result[$i]['goods_thumb']=$cate_imgs-&gt;bykey($i)-&gt;getAttribute('src');<br>             $result[$i]['goods_name']=$cate_names-&gt;bykey($i)-&gt;nodeValue;<br>             $result[$i]['goods_url']=$cate_names-&gt;bykey($i)-&gt;getAttribute('href');<br>             $result[$i]['goods_cprice']=$cate_cprices-&gt;bykey($i)-&gt;nodeValue;<br>             $result[$i]['goods_sale']=$cate_sales-&gt;bykey($i)-&gt;nodeValue;<br>             //$result[$i]['goods_spirce']=$cate_sprices-&gt;bykey($i)-&gt;nodeValue;<br>             $result[$i]['mall_id']=$mall_id;<br>             $result[$i]['shops_id']=$shops_id;<br>             $result[$i]['cate_id']=$cate_id;<br>             $result[$i]['collect_time']=$time;<br>         }<br><br>         if(M('goods')-&gt;addAll($result)){<br>             $this-&gt;success('采集店铺栏目下货品成功',U('index'));<br>         }    /**<br>      * 判断网页数据,转GBK等到UTF-8<br>      */ <br>     public function encoding($source){<br>         $encode = mb_detect_encoding($source, array("GBK","UTF-8","GB2312","BIG5"));<br>         if($encode=='CP936'){<br>             $source=iconv("GBK", "UTF-8//IGNORE", $source);<br>             //$meta用于DOM判断编码<br>             $meta = '<meta>';<br>             $source=$meta.$source;<br>         }<br>         return $source;<br>     }测试:只有这两个按钮能用,其他的都不能用


测试的话 可以将goods表清空 点击采集货品
sql文件在压缩包里

BY:悠悠山雨

Taogoods.zip ( 2.2 MB 下载：72 次 )

AD：真正免费，域名+虚机+企业邮箱=0元