[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8},"actionData","errors","root",{"_11":-5},"routes/user.$name.collections",{"_9":10,"_11":12,"_13":14,"_15":16,"_17":18,"_19":16},"topics",[34],"user",{"_20":21,"_13":14,"_22":23,"_24":14,"_25":26,"_27":28,"_29":16,"_30":31,"_32":33},"loginname","caddyliu","page",1,"limit",20,"total","id","25489","avatar_url","https://avatars.githubusercontent.com/u/16445105?v=3","githubUsername","create_at","2016-10-23 09:48:31.21","score",15,"topic_count","reply_count",2,"is_star",false,{"_20":35,"_36":37,"_38":39,"_40":41,"_42":43,"_44":45,"_46":47,"_48":33,"_30":49,"_50":51,"_25":52,"_53":54},"20000","author_id","13952","tab","share","content","# 使用node-crawler和jsdom完成爬虫\r\n\r\n## 初衷\r\n\r\n从天蚕土豆的《大主宰》Huge Dominate\r\n\r\n起点上是付费的，不经常看，偶尔看看而已，付费没啥意思。于是找免费的\r\n\r\nhttp://www.biquku.com/0/330/\r\n\r\n即使说无弹窗，可是还是有很多广告。。。\r\n\r\n作为一个有追求的程序员能忍么？\r\n\r\n## node-crawler说明\r\n\r\n爬取《大主宰》的所有章节列表\r\n\r\n![1](https://github.com/i5ting/simplereader/raw/master/docs/images/1.png)\r\n\r\n\r\n```\r\nvar Crawler = require(\"crawler\");\r\nvar jsdom = require('jsdom');\r\n\r\nvar c = new Crawler({\r\n jQuery: jsdom,\r\n maxConnections : 100,\r\n forceUTF8:true,\r\n // incomingEncoding: 'gb2312',\r\n // This will be called for each crawled page\r\n callback : function (error, result, $) {\r\n var urls = $('#list a');\r\n console.log(urls)\r\n }\r\n});\r\n\r\nc.queue('http://www.biquku.com/0/330/');\r\n```\r\n\r\n执行\r\n\r\n```\r\nnode examples/1/hello-crawler.js\r\n```\r\n\r\n## 解析章节信息\r\n\r\n代码\r\n\r\n```\r\nvar Crawler = require(\"crawler\");\r\nvar jsdom = require('jsdom');\r\n\r\nvar current_book = { }\r\n\r\nvar c = new Crawler({\r\n jQuery: jsdom,\r\n maxConnections : 100,\r\n forceUTF8:true,\r\n // incomingEncoding: 'gb2312',\r\n // This will be called for each crawled page\r\n callback : function (error, result, $) {\r\n var urls = $('#list a');\r\n // console.log(urls)\r\n \r\n \r\n current_book.title = $('#maininfo h1').text()\r\n current_book.author = $('#info p').eq(0).text()\r\n current_book.update_time = $('#info p').eq(2).text()\r\n current_book.latest_chapter = $('#info p').eq(3).html()\r\n current_book.intro = $('#intro').html()\r\n current_book.chapters = [];\r\n\r\n for(var i = 0; i< urls.length; i++){\r\n var url = urls[i]\r\n \r\n var _url = $(url).attr('href')+\"\";\r\n var num = _url.replace('.html','');\r\n var title = $(url).text();\r\n\r\n\r\n current_book.chapters.push({\r\n num: num,\r\n title: title,\r\n url: _url\r\n })\r\n }\r\n \r\n console.log(current_book)\r\n }\r\n});\r\n\r\nc.queue('http://www.biquku.com/0/330/');\r\n```\r\n\r\n这里主要是拿jQuery解析dom。然后赋值current_book\r\n\r\n执行\r\n\r\n```\r\nnode examples/1/chaper-list.js\r\n{ title: '大主宰',\r\n author: '作者：天蚕土豆',\r\n update_time: '更新时间：2016-07-10',\r\n latest_chapter: '最新章节：第一千两百六十二章大陆洗礼',\r\n intro: '\\n\\t\\t\\t\\t\\t

大千世界，位面交汇，万族林立，群雄荟萃，一位位来自下位面的天之至尊，在这无尽世界，演绎着令人向往的传奇，追求着那主宰之路。\\n 无尽火域，炎帝执掌，万火焚苍穹。\\n 武境之内，武祖之威，震慑乾坤。\\n 西天之殿，百战之皇，战威无可敌。\\n 北荒之丘，万墓之地，不死之主镇天地。\\n ......\\n 少年自北灵境而出，骑九幽冥雀，闯向了那精彩绝伦的纷纭世界，主宰之路，谁主沉浮？\\n 大千世界，万道争锋，吾为大主宰。\\n ..................\\n

\\n\\t\\t\\t\\t\\t

各位书友要是觉得《大主宰》还不错的话请不要忘记向您QQ群和微博里的朋友推荐哦！

\\n\\t\\t\\t\\t',\r\n chapters: \r\n [ { num: '153064', title: '第一章已发。', url: '153064.html' },\r\n { num: '153065', title: '第一章北灵院', url: '153065.html' },\r\n { num: '153066', title: '第二章被踢出灵路的少年', url: '153066.html' },\r\n { num: '153067', title: '第三章牧域', url: '153067.html' },\r\n { num: '153068', title: '第四章大浮屠诀', url: '153068.html' },\r\n```\r\n\r\n想要的信息都准备好了，下面就开始爬某一章吧\r\n\r\n## 爬取某一章\r\n\r\n看一下\r\n\r\n```\r\n第一章北灵院\r\nhttp://www.biquku.com/0/330/153065.html\r\n```\r\n\r\n和之前的地址的差别是什么呢？\r\n\r\n- http://www.biquku.com/0/330 是一样的\r\n- 153065.html章节地址\r\n\r\n再回顾一下\r\n\r\n```\r\n chapters: \r\n [ { num: '153064', title: '第一章已发。', url: '153064.html' },\r\n```\r\n\r\n也就是说 url 即是章节地址，剩下的就拼接起来就好了。\r\n\r\n\r\n```\r\nfunction one(chapter){\r\n console.log(chapter)\r\n c.queue([{\r\n uri: 'http://www.biquku.com/0/330/' + chapter.num + '.html',\r\n jQuery: jsdom,\r\n forceUTF8:true,\r\n // The global callback won't be called\r\n callback: function (error, result, $) {\r\n var content = $('#content').html();\r\n console.log(content)\r\n }\r\n }]);\r\n}\r\n```\r\n\r\n模拟执行代码\r\n\r\n```\r\nvar chapter = { num: '4063307', title: '第一千两百五十二章现世！', url: '4063307.html' }\r\n\r\none(chapter);\r\n```\r\n\r\n执行\r\n\r\n```\r\nnode examples/1/chaper-one.js\r\n{ num: '4063307', title: '第一千两百五十二章现世！', url: '4063307.html' }\r\n    第一千两百五十二章

    轰轰！

    灵战子的低吼声，犹如雷鸣一般在这天地之间回荡，一股股磅礴浩瀚的灵力，也是犹如洪流一般，不断的从其体内呼啸而出，引得空间震荡。◇↓，

    此时的灵战子，双目精光涌动，神采飞扬，再没了先前的那种虚弱之感，显然，借助着那所谓的“战祭”，他直接是在顷刻间就将自身状态恢复到了巅峰。

    之前消耗的灵力，也是再度充盈了他的身躯。\r\n```\r\n\r\n## 总结\r\n\r\n爬取一本书的流程\r\n\r\n- 先取列表\r\n- 再去章节\r\n\r\n技能\r\n\r\n- node-crawler 爬取，发送http请求，是基于request模块的\r\n- 结合jsdom，使用类似于jquery的dom操作，解析结果\r\n\r\nnode-crawler有2种用法\r\n\r\n- c.queue方法如无callback，走全局的callback，这是获取列表的时候的用法\r\n- c.queue方法如有callback，走自己的callback，这是获取章节的用法\r\n\r\n我们的做法\r\n\r\n- 最小化问题，先关注爬取一本书的流程\r\n- 爬取一本书的流程中的列表\r\n- 爬取一本书的流程中的章节\r\n\r\n在这个过程中，我们可以很好的学习node-crawler和jquery的dom操作，知识点整体来说比较少，更加容易学习。\r\n\r\n下面，爬取的信息咋办呢？见下一章。\r\n\r\n## 更多\r\n\r\n参见 https://github.com/i5ting/githubrank\r\n\r\n\r\n全文完\r\n\r\n欢迎关注我的公众号【node全栈】\r\n\r\n![node全栈.png](//static.cnodejs.org/FtALxsauUkYDGdzcuA5y6BaIdUMC)\r\n\r\n联系我，更多交流\r\n\r\n![xiaoweiquan.jpeg](//static.cnodejs.org/FrCbjyzuNmHF1u3j8t26t-cGWPnB)","title","Node.js写爬虫系列","last_reply_at","2017-08-25 07:38:20.497","good",true,"top",66,"visit_count",33453,"2016-08-30 06:38:07.475","author",{"_13":55,"_22":56},"i5ting","https://avatars.githubusercontent.com/u/3118295?v=4"]