仓库源文，站点原文

title: NodeJS写个爬虫，把文章放到kindle中阅读 categories:

• JavaScript
• 前端杂烩
• 网络交互 tags:
• tech
• cnblogs
• NodeJS
• 爬虫 date: 2014-04-28 01:41:00 fromUrl: http://www.cnblogs.com/hustskyking/archive/2014/04/28/spider-with-node.html from: 博客园

var http = require("http");

var url = "http://www.alloyteam.com/";
var data = "";

// 创建一个请求
var req = http.request(url, function(res){
    // 设置显示编码
    res.setEncoding("utf8");
    // 数据是 chunked 发送，意思就是一段一段发送过来的
    // 我们使用 data 给他们串接起来
    res.on('data', function(chunk){
        data += chunk;
    });
    // 响应完毕时间出发，输出 data
    res.on('end', function(){
        // dealData(data);
        console.log(data);
    });
});

// 发送请求
req.end();

// function dealData
var reg = /<ul\s+class="articlemenu">\s+<li>\s+<a[^>]*>.*?<\ a="">\s+<a href="(.*?)" [^="">]*>(.*?)<\ a="">[\s\S]*?<div\s+class="text">([\s\S]*?)<\ div="">/g;
var res = [];
while(match = reg.exec(data)) {
    res.push({
        "url": match[1],
        "title": match[2],
        "excerpt": match[3]
    });
}

</></div\s+class="text"></></a></></a[^></li></ul\s+class="articlemenu">

reg.exec(data);

[{
    "url: url,
    "title": title,
    "excerpt" excerpt
}];

<h4>3. 数据的过滤</h4>

var excerpt = excerpt.replace(/(<.*?>)((.*?)(<.*?>))?/g, "$3");

</.?></.?>

excerpt = excerpt.slice(0, 120);

<h4>4. 存到数据库（或者文件）</h4>

[title](url)
> excerpt

var str = "";
for(var i = 0, len = data.length; i < len; i++){
    str += "[" + data[i].title + "](" + data[i].url + ")\n" + data[i].excerpt.replace("\n\s*\n?", ">\n") + "\n\n";
}

fs.writeFile('index.md', str, function (err) {
    if (err) throw err;
    console.log('数据已保存～');
});

var http = require("http");
var fs = require("fs");

var url = "http://www.alloyteam.com/";
var data = "";

var req = http.request(url, function(res){
    res.setEncoding("utf8");
    res.on('data', function(chunk){
        data += chunk;
    });
    res.on('end', function(){
        dealData(data);
    });
});

req.on('error', function(e){
    throw e;
});

req.end();
console.log("数据下载中...");

function dealData(data){
    var reg = /<ul\s+class="articlemenu">\s+<li>\s+<a[^>]*>.*?<\ a="">\s+<a href="(.*?)" [^="">]*>(.*?)<\ a="">[\s\S]*?<div\s+class="text">([\s\S]*?)<\ div="">/g;
    var res = [];
    while(match = reg.exec(data)) {
        res.push({
            "url": match[1],
            "title": match[2],
            "excerpt": match[3].replace(/(<.*?>)((.*?)(<.*?>))?/g, "$3").slice(0,120)
        });
    }
    writeFile(res)
 }

function writeFile(data){
    var str = "";
    for(var i = 0, len = data.length; i < len; i++){
        str += "[" + data[i].title + "](" + data[i].url + ")\n>" + data[i].excerpt.replace(/\n\s*\n?/g, "\n>") + "\n\n";
    }
    fs.writeFile('index.md', str, function (err) {
       if (err) throw err;
       console.log('数据已保存～');
    });
}

node spider.js

<p>就可以在同级目录下看到 index.md 文件了。至于如何放到 kindle 中，先了解下 <a href="http://www.idpf.org/epub/20/spec/OPF_2.0.1_draft.htm" target="_blank">OPF</a> 格式，然后使用 Amazon 的 <a href="http://www.amazon.com/gp/feature.html?ie=UTF8<docId=1000765211" target="_blank">KindleGen</a> 工具打包就行啦。</p>

<h3>三. 参考资料</h3>