快捷導(dǎo)航

Python制作爬蟲(chóng)抓取美女圖

更新時(shí)間：2016年01月20日 11:08:38 作者：我的代碼會(huì)飛

本文主要介紹了Python制作爬蟲(chóng)抓取美女圖，文中通過(guò)示例代碼介紹的非常詳細(xì)，對(duì)大家的學(xué)習(xí)或者工作具有一定的參考學(xué)習(xí)價(jià)值，需要的朋友們下面隨著小編來(lái)一起學(xué)習(xí)學(xué)習(xí)吧

今天我們就搞個(gè)爬蟲(chóng)把美圖都給扒下來(lái)！我只是一個(gè)學(xué)習(xí)python的菜鳥(niǎo)，技術(shù)不可恥，技術(shù)是無(wú)罪的?。?！

煎蛋：

先說(shuō)說(shuō)程序的流程：獲取煎蛋妹子圖URL，得到網(wǎng)頁(yè)代碼，提取妹子圖片地址，訪問(wèn)圖片地址并將圖片保存到本地。Ready？先讓我們看看煎蛋妹子網(wǎng)頁(yè)：

　　我們得到URL為：http://jandan.net/ooxx/page-1764#comments 1764就是頁(yè)碼，首先我們要得到最新的頁(yè)碼，然后向前尋找，然后得到每頁(yè)中圖片的url。下面我們分析網(wǎng)站代碼寫(xiě)出正則表達(dá)式！

　　根據(jù)之前文章的方法我們寫(xiě)出如下函數(shù)getNewPage：

def __getNewPage(self):
    pageCode = self.Get(self.__Url)
    type = sys.getfilesystemencoding()
    pattern = re.compile(r'<div .*?cp-pagenavi">.*?<span .*?current-comment-page">\[(.*?)\]</span>',re.S)
    newPage = re.search(pattern,pageCode.decode("UTF-8").encode(type))
    print pageCode.decode("UTF-8").encode(type)
    if newPage != None:
      return newPage.group(1)
    return 1500

　　不要問(wèn)我為什么如果失敗返回1500。。。因?yàn)榧宓鞍?500頁(yè)之前的圖片都給吃了。你也可以返回0。接下來(lái)是圖片的

def __getAllPicUrl(self,pageIndex):
    realurl = self.__Url + "page-" + str(pageIndex) + "#comments"
    pageCode = self.Get(realurl)
    type = sys.getfilesystemencoding()
    pattern = re.compile('<p>.*?<a .*?view_img_link">.*?</a>.*?<img src="(.*?)".*?</p>',re.S)
    items = re.findall(pattern,pageCode.decode("UTF-8").encode(type))
    for item in items:
      print item

　　好了，得到了圖片地址，接下來(lái)就是訪問(wèn)圖片地址然后保存圖片了：

def __savePics(self,img_addr,folder):
    for item in img_addr:
      filename = item.split('/')[-1]
      print "正在保存圖片：" + filename
      with open(filename,'wb') as file:
        img = self.Get(item)
        file.write(img)

　　當(dāng)你覺(jué)得信心滿滿的時(shí)候，一定會(huì)有一盆冷水澆到你的頭上，畢竟程序就是這樣，考驗(yàn)?zāi)愕哪托?，打磨你的自信。你測(cè)試了一會(huì)兒，然后你發(fā)現(xiàn)你重啟程序后再也無(wú)法獲取最新頁(yè)碼，你覺(jué)得我什么也沒(méi)動(dòng)啊為什么會(huì)這樣。別著急，我們將得到的網(wǎng)頁(yè)代碼打印出來(lái)看看：

　　看到了吧，是服務(wù)器感覺(jué)你不像瀏覽器訪問(wèn)的結(jié)果把你的ip給屏蔽了。真是給跪了，辛辛苦苦碼一年，屏蔽回到解放前！那么這個(gè)如何解決呢，答：換ip 找代理。接下來(lái)我們要改一下我們的HttpClient.py 將里面的opener設(shè)置下代理服務(wù)器。具體代理服務(wù)器請(qǐng)自行百度之，關(guān)鍵字：http代理。想找到一個(gè)合適的代理也不容易自己ie Internet選項(xiàng)挨個(gè)試試，測(cè)試下網(wǎng)速。

# -*- coding: utf-8 -*-
import cookielib, urllib, urllib2, socket
import zlib,StringIO
class HttpClient:
 __cookie = cookielib.CookieJar()
 __proxy_handler = urllib2.ProxyHandler({"http" : '42.121.6.80:8080'})#設(shè)置代理服務(wù)器與端口
 __req = urllib2.build_opener(urllib2.HTTPCookieProcessor(__cookie),__proxy_handler)#生成opener
 __req.addheaders = [
  ('Accept', 'application/javascript, */*;q=0.8'),
  ('User-Agent', 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)')
 ]
 urllib2.install_opener(__req)

 def Get(self, url, refer=None):
  try:
   req = urllib2.Request(url)
   #req.add_header('Accept-encoding', 'gzip')
   if not (refer is None):
    req.add_header('Referer', refer)
   response = urllib2.urlopen(req, timeout=120)
   html = response.read()
   #gzipped = response.headers.get('Content-Encoding')
   #if gzipped:
   #  html = zlib.decompress(html, 16+zlib.MAX_WBITS)
   return html
  except urllib2.HTTPError, e:
   return e.read()
  except socket.timeout, e:
   return ''
  except socket.error, e:
   return ''

　　然后，就可以非常愉快的查看圖片了。不過(guò)用了代理速度好慢。。?？梢栽O(shè)置timeout稍微長(zhǎng)一點(diǎn)兒，防止圖片下載不下來(lái)！

　　好了，rosi的下篇文章再放！現(xiàn)在是時(shí)候上一波代碼了：

# -*- coding: utf-8 -*-
import cookielib, urllib, urllib2, socket
import zlib,StringIO
class HttpClient:
 __cookie = cookielib.CookieJar()
 __proxy_handler = urllib2.ProxyHandler({"http" : '42.121.6.80:8080'})
 __req = urllib2.build_opener(urllib2.HTTPCookieProcessor(__cookie),__proxy_handler)
 __req.addheaders = [
  ('Accept', 'application/javascript, */*;q=0.8'),
  ('User-Agent', 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)')
 ]
 urllib2.install_opener(__req)

 def Get(self, url, refer=None):
  try:
   req = urllib2.Request(url)
   req.add_header('Accept-encoding', 'gzip')
   if not (refer is None):
    req.add_header('Referer', refer)
   response = urllib2.urlopen(req, timeout=120)
   html = response.read()
   gzipped = response.headers.get('Content-Encoding')
   if gzipped:
     html = zlib.decompress(html, 16+zlib.MAX_WBITS)
   return html
  except urllib2.HTTPError, e:
   return e.read()
  except socket.timeout, e:
   return ''
  except socket.error, e:
   return ''

 def Post(self, url, data, refer=None):
  try:
   #req = urllib2.Request(url, urllib.urlencode(data))
   req = urllib2.Request(url,data)
   if not (refer is None):
    req.add_header('Referer', refer)
   return urllib2.urlopen(req, timeout=120).read()
  except urllib2.HTTPError, e:
   return e.read()
  except socket.timeout, e:
   return ''
  except socket.error, e:
   return ''

 def Download(self, url, file):
  output = open(file, 'wb')
  output.write(urllib2.urlopen(url).read())
  output.close()

# def urlencode(self, data):
#  return urllib.quote(data)

 def getCookie(self, key):
  for c in self.__cookie:
   if c.name == key:
    return c.value
  return ''

 def setCookie(self, key, val, domain):
  ck = cookielib.Cookie(version=0, name=key, value=val, port=None, port_specified=False, domain=domain, domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False)
  self.__cookie.set_cookie(ck)
#self.__cookie.clear() clean cookie
# vim : tabstop=2 shiftwidth=2 softtabstop=2 expandtab

HttpClient

# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from HttpClient import HttpClient
import sys,re,os
class JianDan(HttpClient):
  def __init__(self):
    self.__pageIndex = 1500 #之前的圖片被煎蛋吞了
    self.__Url = "http://jandan.net/ooxx/"
    self.__floder = "jiandan"
  def __getAllPicUrl(self,pageIndex):
    realurl = self.__Url + "page-" + str(pageIndex) + "#comments"
    pageCode = self.Get(realurl)
    type = sys.getfilesystemencoding()
    pattern = re.compile('<p>.*?<a .*?view_img_link">.*?</a>.*?<img src="(.*?)".*?</p>',re.S)
    items = re.findall(pattern,pageCode.decode("UTF-8").encode(type))
    for item in items:
      print item
    self.__savePics(items,self.__floder)

  def __savePics(self,img_addr,folder):
    for item in img_addr:
      filename = item.split('/')[-1]
      print "正在保存圖片：" + filename
      with open(filename,'wb') as file:
        img = self.Get(item)
        file.write(img)

  def __getNewPage(self):
    pageCode = self.Get(self.__Url)
    type = sys.getfilesystemencoding()
    pattern = re.compile(r'<div .*?cp-pagenavi">.*?<span .*?current-comment-page">\[(.*?)\]</span>',re.S)
    newPage = re.search(pattern,pageCode.decode("UTF-8").encode(type))
    print pageCode.decode("UTF-8").encode(type)
    if newPage != None:
      return newPage.group(1)
    return 1500

  def start(self):
    isExists=os.path.exists(self.__floder)#檢測(cè)是否存在目錄
    print isExists
    if not isExists:
      os.mkdir(self.__floder)
    os.chdir(self.__floder)
    page = int(self.__getNewPage())
    for i in range(self.__pageIndex,page):
      self.__getAllPicUrl(i)

if __name__ == '__main__':
  jd = JianDan()
  jd.start()

JianDan

您可能感興趣的文章: