為什么結(jié)果只顯示“正在爬取http://idcbgp.cn/learn/id”*6 ?
結(jié)果:
正在爬取http://idcbgp.cn/learn/637 正在爬取http://idcbgp.cn/learn/348 正在爬取http://idcbgp.cn/learn/259 正在爬取http://idcbgp.cn/learn/197 正在爬取http://idcbgp.cn/learn/134 正在爬取http://idcbgp.cn/learn/75
代碼:
var?http?=?require('http') var?cheerio?=?require('cheerio') var?Promise?=?require('bluebird') var?baseUrl?=?'http://idcbgp.cn/learn/' var?videoIds?=?[637,348,259,197,134,75] var?fetchCourseArray?=?[] videoIds.forEach(function?(id)?{ ????fetchCourseArray.push(getPageAsync(baseUrl?+?id)) }) Promise ????.all([]) ????.then(function?(pages)?{ ????????var?coursesData?=?[] ????????pages.forEach(function?(html)?{ ????????????var?courses?=?filterChapters(html) ????????????coursesData.push(courses) ????????}) ????????coursesData.sort(function?(a,?b)?{ ????????????return?a.number?<?b.number ????????}) ????????printCourseInfo(coursesData) ????}) function?getPageAsync(url)?{ ????return?new?Promise(function?(resolve,?reject)?{ ????????console.log('正在爬取'?+?url) ????????http.get(url,?function?(res)?{ ????????????var?html?=?'' ????????????res.on('data',function?(data)?{ ????????????????html?+=?data ????????????}) ????????????res.on('end',function?()?{ ????????????????resolve(html) ????????????}) ????????}).on('error',function?(e)?{ ????????????reject(e) ????????????console.log('獲取頁面數(shù)據(jù)出錯(cuò)!') ????????}) ????}) } function?filterChapters(html)?{ ????var?$?=?cheerio.load(html) ????var?chapters?=?$('.chapter') ????var?title?=?$('.course-infos?.hd?.l').text() ????var?number?=?parseInt($('.course-infos?.statics?.static-item')[3].children('.meta-value?strong').text().trim(),10) ????var?courseData?=?{ ????????title:?title, ????????number:?number, ????????videos:?[] ????} ????chapters.each(function?(item)?{ ????????var?chapter?=?item ????????var?chapterTitle?=?chapter.find('strong').text() ????????var?videos?=?chapter.find('.video').children('li') ????????var?chapterData?=?{ ????????????chapterTitle:?chapterTitle, ????????????videos:?[] ????????} ????????videos.each(function?(item)?{ ????????????var?video?=?item.find('.studyvideo') ????????????var?videoTitle?=?video.text() ????????????var?id?=?video.attr('href').split('video/')[1] ????????????chapterData.videos.push({ ????????????????title:?videoTitle, ????????????????id:?id ????????????}) ????????}) ????????courseData.videos.push(chapterData) ????}) ????return?courseData } function?printCourseInfo(coursesData)?{ ????coursesData.forEach(function?(courseData)?{ ????????console.log(courseData.number?+?'?人學(xué)過?'?+?courseData.title?+?'\n') ????}) ????coursesData.forEach(function?(courseData)?{ ????????console.log('###?'?+?courseData.title?+?'\n') ????????courseData.videos.forEach(function?(item)?{ ????????????console.log('###?'?+?item.title?+?'\n') ????????????var?chapterTitle?=?courseData.chapterTitle ????????????console.log(chapterTitle?+?'\n') ????????????item.videos.forEach(function?(video)?{ ????????????????console.log('?【'?+?video.id?+?'】'?+?video.title?+?'\n') ????????????}) ????????}) ????}) }
2016-05-25
修改: