NODE.JS采集练手

1'use strict'
2const request = require('request')
3const co = require('co')
4const cheerio = require('cheerio')
5const xlsx = require('node-xlsx')
6const fs = require('fs')
7let base = 'https://aso100.com/app/rank/appid/1188599882/country/cn'
8let base_url = 'https://aso100.com/app/rank/appid/1188599882/country/'
9let urls = []
10let dataArray = []
11
12//通过url，获取网页内容
13let getUrl = (url) => {
14  return new Promise((resolve, reject) => {
15    request(url, function (err, response, body) {
16      if (err) {
17        return reject(err)
18      }
19      return err ? reject(err) : resolve(body)
20    })
21  })
22}
23
24//通过url获取需要的字段内容，并存入dataArray
25let getDetail = function* (url) {
26  let data = yield getUrl(url)
27  let $ = cheerio.load(data)
28  let title = $('.appinfo-title').text()
29  let name = $('.appinfo-country .name').text()
30  console.log(title + name)
31
32  dataArray.push([title, name])
33}
34
35//休眠函数，避免速度太快
36let sleep = (t) => {
37  return new Promise((resolve, reject) => {
38    setTimeout(function () {
39      resolve()
40    }, t)
41  })
42}
43
44//获取base目录中所有需要采集的url地址，并执行采集，最后导出excel
45let getUrlArray = function* () {
46  let html = yield getUrl(base)
47  let $ = cheerio.load(html)
48  let ele = $('.select-container').find('a')
49  ele.each(function () {
50    if ($(this).attr('data-country')) {
51      urls.push(base_url + $(this).attr('data-country'))
52    }
53  })
54
55  //获取内容
56  for (let i = 0; i < urls.length; i++) {
57    yield sleep(20000)
58    yield getDetail(urls[i])
59  }
60
61  //导出excel
62  let buffer = xlsx.build([{ name: 'mySheetName', data: dataArray }])
63  fs.writeFileSync('test.csv', buffer, 'binary')
64}
65
66//通过co执行Generator
67let run = (fn) => {
68  co(fn).catch((err) => {
69    console.log(err)
70  })
71}
72
73//开始采集
74run(getUrlArray())
Copy