快捷導(dǎo)航

Python爬取當(dāng)當(dāng)、京東、亞馬遜圖書信息代碼實例

更新時間：2017年12月09日 11:27:34 作者：吳祥生

這篇文章主要介紹了Python爬取當(dāng)當(dāng)、京東、亞馬遜圖書信息代碼實例，具有一定借鑒價值，需要的朋友可以參考下。

注：1.本程序采用MSSQLserver數(shù)據(jù)庫存儲，請運行程序前手動修改程序開頭處的數(shù)據(jù)庫鏈接信息

2.需要bs4、requests、pymssql庫支持

3.支持多線程

from bs4 import BeautifulSoup 
import re,requests,pymysql,threading,os,traceback 
 
try: 
  conn = pymysql.connect(host='127.0.0.1', port=3306, user='root', passwd='root', db='book',charset="utf8") 
  cursor = conn.cursor() 
except: 
  print('\n錯誤：數(shù)據(jù)庫連接失敗') 
 
#返回指定頁面的html信息 
def getHTMLText(url): 
  try: 
    headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'} 
    r = requests.get(url,headers = headers) 
    r.raise_for_status() 
    r.encoding = r.apparent_encoding 
    return r.text 
  except: 
    return '' 
#返回指定url的Soup對象 
def getSoupObject(url): 
  try: 
    html = getHTMLText(url) 
    soup = BeautifulSoup(html,'html.parser') 
    return soup 
  except: 
    return '' 
#獲取該關(guān)鍵字在圖書網(wǎng)站上的總頁數(shù) 
def getPageLength(webSiteName,url): 
  try: 
    soup = getSoupObject(url) 
    if webSiteName == 'DangDang': 
      a = soup('a',{'name':'bottom-page-turn'}) 
      return a[-1].string 
    elif webSiteName == 'Amazon': 
      a = soup('span',{'class':'pagnDisabled'}) 
      return a[-1].string 
  except: 
    print('\n錯誤：獲取{}總頁數(shù)時出錯...'.format(webSiteName)) 
    return -1 
 
class DangDangThread(threading.Thread): 
  def __init__(self,keyword): 
    threading.Thread.__init__(self) 
    self.keyword = keyword 
  def run(self): 
    print('\n提示：開始爬取當(dāng)當(dāng)網(wǎng)數(shù)據(jù)...') 
    count = 1 
   
    length = getPageLength('DangDang','http://search.dangdang.com/?key={}'.format(self.keyword))#總頁數(shù) 
    tableName = 'db_{}_dangdang'.format(self.keyword) 
 
    try: 
      print('\n提示：正在創(chuàng)建DangDang表...') 
      cursor.execute('create table {} (id int ,title text,prNow text,prPre text,link text)'.format(tableName)) 
      print('\n提示：開始爬取當(dāng)當(dāng)網(wǎng)頁面...') 
      for i in range(1,int(length)): 
        url = 'http://search.dangdang.com/?key={}&page_index={}'.format(self.keyword,i) 
        soup = getSoupObject(url) 
        lis = soup('li',{'class':re.compile(r'line'),'id':re.compile(r'p')}) 
        for li in lis: 
          a = li.find_all('a',{'name':'itemlist-title','dd_name':'單品標(biāo)題'}) 
          pn = li.find_all('span',{'class': 'search_now_price'}) 
          pp = li.find_all('span',{'class': 'search_pre_price'}) 
 
          if not len(a) == 0: 
            link = a[0].attrs['href'] 
            title = a[0].attrs['title'].strip() 
          else: 
            link = 'NULL' 
            title = 'NULL' 
 
          if not len(pn) == 0: 
            prNow = pn[0].string 
          else: 
            prNow = 'NULL' 
 
          if not len(pp) == 0: 
            prPre = pp[0].string 
          else: 
            prPre = 'NULL' 
          sql = "insert into {} (id,title,prNow,prPre,link) values ({},'{}','{}','{}','{}')".format(tableName,count,title,prNow,prPre,link) 
          cursor.execute(sql) 
          print('\r提示：正在存入當(dāng)當(dāng)數(shù)據(jù),當(dāng)前處理id：{}'.format(count),end='') 
          count += 1 
          conn.commit() 
    except: 
      pass 

class AmazonThread(threading.Thread): 
  def __init__(self,keyword): 
    threading.Thread.__init__(self) 
    self.keyword = keyword 
 
  def run(self): 
    print('\n提示：開始爬取亞馬遜數(shù)據(jù)...') 
    count = 1 
    length = getPageLength('Amazon','https://www.amazon.cn/s/keywords={}'.format(self.keyword))#總頁數(shù) 
    tableName = 'db_{}_amazon'.format(self.keyword) 
     
    try: 
      print('\n提示：正在創(chuàng)建Amazon表...') 
      cursor.execute('create table {} (id int ,title text,prNow text,link text)'.format(tableName)) 
   
      print('\n提示：開始爬取亞馬遜頁面...') 
      for i in range(1,int(length)): 
        url = 'https://www.amazon.cn/s/keywords={}&page={}'.format(self.keyword,i) 
        soup = getSoupObject(url) 
        lis = soup('li',{'id':re.compile(r'result_')}) 
        for li in lis: 
          a = li.find_all('a',{'class':'a-link-normal s-access-detail-page a-text-normal'}) 
          pn = li.find_all('span',{'class': 'a-size-base a-color-price s-price a-text-bold'}) 
          if not len(a) == 0: 
            link = a[0].attrs['href'] 
            title = a[0].attrs['title'].strip() 
          else: 
            link = 'NULL' 
            title = 'NULL' 
 
          if not len(pn) == 0: 
            prNow = pn[0].string 
          else: 
            prNow = 'NULL' 
 
          sql = "insert into {} (id,title,prNow,link) values ({},'{}','{}','{}')".format(tableName,count,title,prNow,link) 
          cursor.execute(sql) 
          print('\r提示：正在存入亞馬遜數(shù)據(jù),當(dāng)前處理id：{}'.format(count),end='') 
          count += 1 
          conn.commit() 
    except: 
      pass 

class JDThread(threading.Thread): 
  def __init__(self,keyword): 
    threading.Thread.__init__(self) 
    self.keyword = keyword 
  def run(self): 
    print('\n提示：開始爬取京東數(shù)據(jù)...') 
    count = 1 
 
    tableName = 'db_{}_jd'.format(self.keyword) 
     
    try: 
      print('\n提示：正在創(chuàng)建JD表...') 
      cursor.execute('create table {} (id int,title text,prNow text,link text)'.format(tableName)) 
      print('\n提示：開始爬取京東頁面...') 
      for i in range(1,100): 
        url = 'https://search.jd.com/Search?keyword={}&page={}'.format(self.keyword,i) 
        soup = getSoupObject(url) 
        lis = soup('li',{'class':'gl-item'}) 
        for li in lis: 
          a = li.find_all('div',{'class':'p-name'}) 
          pn = li.find_all('div',{'class': 'p-price'})[0].find_all('i') 
 
          if not len(a) == 0: 
            link = 'http:' + a[0].find_all('a')[0].attrs['href'] 
            title = a[0].find_all('em')[0].get_text() 
          else: 
            link = 'NULL' 
            title = 'NULL' 
           
          if(len(link) > 128): 
            link = 'TooLong' 
 
          if not len(pn) == 0: 
            prNow = '￥'+ pn[0].string 
          else: 
            prNow = 'NULL' 
          sql = "insert into {} (id,title,prNow,link) values ({},'{}','{}','{}')".format(tableName,count,title,prNow,link) 
          cursor.execute(sql) 
          print('\r提示：正在存入京東網(wǎng)數(shù)據(jù),當(dāng)前處理id：{}'.format(count),end='') 
          count += 1 
          conn.commit() 
    except : 
      pass 
def closeDB(): 
  global conn,cursor 
  conn.close() 
  cursor.close() 
 
def main(): 
  print('提示：使用本程序，請手動創(chuàng)建空數(shù)據(jù)庫：Book，并修改本程序開頭的數(shù)據(jù)庫連接語句') 
  keyword = input("\n提示：請輸入要爬取的關(guān)鍵字：") 
 
  dangdangThread = DangDangThread(keyword) 
  amazonThread = AmazonThread(keyword) 
  jdThread = JDThread(keyword) 
   dangdangThread.start() 
  amazonThread.start() 
  jdThread.start() 
  dangdangThread.join() 
  amazonThread.join() 
  jdThread.join() 
   closeDB() 
   print('\n爬取已經(jīng)結(jié)束，即將關(guān)閉....') 
  os.system('pause') 
   
main()

示例截圖：

關(guān)鍵詞：Android下的部分運行結(jié)果（以導(dǎo)出至Excel）