var superagent = require('superagent'); 
var cheerio = require('cheerio');
var async = require('async');
var fs = require('fs');
var request = require('request');
var page=1; //獲取發(fā)型師處有分頁(yè)功能，所以用該變量控制分頁(yè)
var num = 0;//爬取到的信息總條數(shù)
var storeid = 1;//門(mén)店ID
console.log('爬蟲(chóng)程序開(kāi)始運(yùn)行......');

function fetchPage(x) {   //封裝函數(shù)
  startRequest(x); 
}
function startRequest(x) {

  superagent
    .post('http://tweixin.yueyishujia.com/v2/store/designer.json')
    .send({ 
      // 請(qǐng)求的表單信息Form data
      page : x, 
      storeid : storeid
    })
      // Http請(qǐng)求的Header信息
    .set('Accept', 'application/json, text/javascript, */*; q=0.01')
    .set('Content-Type','application/x-www-form-urlencoded; charset=UTF-8')
    .end(function(err, res){     
      // 請(qǐng)求返回后的處理
      // 將response中返回的結(jié)果轉(zhuǎn)換成JSON對(duì)象
      if(err){
        console.log(err);
      }else{
        var designJson = JSON.parse(res.text);
        var deslist = designJson.data.designerlist;
        if(deslist.length > 0){
          num += deslist.length;
          // 并發(fā)遍歷deslist對(duì)象
          async.mapLimit(deslist, 5, 
            function (hair, callback) {
            // 對(duì)每個(gè)對(duì)象的處理邏輯
               console.log('...正在抓取數(shù)據(jù)ID：'+hair.id+'----發(fā)型師:'+hair.name);
               saveImg(hair,callback);
            }, 
            function (err, result) {
              console.log('...累計(jì)抓取的信息數(shù)→→' + num);
            }
          );
          page++;
          fetchPage(page);
        }else{
          if(page == 1){
            console.log('...爬蟲(chóng)程序運(yùn)行結(jié)束~~~~~~~');
            console.log('...本次共爬取數(shù)據(jù)'+num+'條...');
            return;
          }
          storeid += 1;
          page = 1;
          fetchPage(page);
        }
      }
    });
} 
fetchPage(page);
function saveImg(hair,callback){
  // 存儲(chǔ)圖片
  var img_filename = hair.store.name+'-'+hair.name + '.png';

  var img_src = 'http://photo.yueyishujia.com:8112' + hair.avatar; //獲取圖片的url

  //采用request模塊，向服務(wù)器發(fā)起一次請(qǐng)求，獲取圖片資源
  request.head(img_src,function(err,res,body){
    if(err){
      console.log(err);
    }else{
        request(img_src).pipe(fs.createWriteStream('./image/' + img_filename));   //通過(guò)流的方式，把圖片寫(xiě)到本地/image目錄下，并用發(fā)型師的姓名和所屬門(mén)店作為圖片的名稱。
        console.log('...存儲(chǔ)id='+hair.id+'相關(guān)圖片成功！');
    }
  });
  // 存儲(chǔ)照片相關(guān)信息
  var html = '姓名：'+hair.name+'<br>職業(yè)：'+hair.jobtype+'<br>職業(yè)等級(jí)：'+hair.jobtitle+'<br>簡(jiǎn)介：'+hair.simpleinfo+'<br>個(gè)性簽名：'+hair.info+'<br>剪發(fā)價(jià)格：'+hair.cutmoney+'元<br>店名：'+hair.store.name+'<br>地址：'+hair.store.location+'<br>聯(lián)系方式：'+hair.telephone+'<br>頭像：<img src='+img_src+' style="width:200px;height:200px;">';
  fs.appendFile('./data/' +hair.store.name+'-'+ hair.name + '.html', html, 'utf-8', function (err) {
    if (err) {
      console.log(err);
    }
  });
  callback(null, hair);
}