ip资源池，定时更新

Author: 熊长江

db/db.js

@@ -0,0 +1,13 @@
+const mongoose = require('mongoose');
+
+mongoose.connect('mongodb://localhost:27017/test');
+
+let db = mongoose.connection;
+
+db.on('error', console.error.bind(console, 'mongoDB 链接错误:'));
+
+db.once('open', function(callback){
+    console.log('MongoDB 打开!');
+});
+
+module.exports = mongoose;

db/models_ip.js

@@ -0,0 +1,30 @@
+let mongoose = require('./db.js');
+
+let ipSchema = mongoose.Schema({
+    country: {type: String},
+    ip: {type: String},
+    port: {type: String},
+    area: {type: String},
+    types: {type: String},
+    protocol: {type: String},
+    speed: {type: String},
+    time: {type: String},
+});
+
+let ipModel = mongoose.model('ipSchema',ipSchema);
+
+module.exports = ipModel;
+
+
+// let ipOne = new ipModel({name: 'fff'});
+
+// ipOne.save(function(err){
+//     if(err) return console.error(err);
+//     console.log('写入成功！');
+
+//     ipModel.find(function(err, docs){
+//     if(err) return console.error(err);
+//     console.log(docs);
+//     });
+// });
+

package.json

@@ -19,6 +19,8 @@
   "dependencies": {
     "cheerio": "^1.0.0-rc.2",
     "express": "^4.15.3",
+    "mongoose": "^4.11.3",
+    "node-schedule": "^1.2.3",
     "request": "^2.81.0"
   }
 }

proxy.js

@@ -1,7 +1,11 @@
 const http = require('http');
 const fs = require('fs');
+
 let request = require('request');
 let cheerio = require('cheerio');
+let schedule = require('node-schedule');
+
+let ipModel = require('./db/models_ip');
 
 let ipListURL = 'http://www.xicidaili.com/nn/';
 let pageSize = 0;
@@ -18,6 +22,11 @@ function startRequest(ipListURL) {
         });
 
         res.on('end', function () {
+            ipModel.remove({}, function(err){
+                if(err) return console.error(err);
+                console.log('清空数据');
+            });
+
             pageSize++;
             $ = cheerio.load(html);
 
@@ -31,11 +40,34 @@ function startRequest(ipListURL) {
                     let testip = request.defaults({ 'proxy': proxy }).get('http://ip.chinaz.com/getip.aspx', { timeout: 3000 }, function (err, response, body) {
                         if (body && body.substring(0, 4) == '{ip:') {
                             proxy = proxy + '\n';
-                            fs.appendFile('./data/' + 'ipList' + '.txt', proxy, 'utf-8', function (err) {
-                                if (err) {
-                                    console.log(err);
-                                }
+
+                            let ipOne = new ipModel({
+                                country: td.eq(0).find('img').attr('alt'),
+                                ip: td.eq(1).text(),
+                                port: td.eq(2).text(),
+                                area: td.eq(3).find('a').text(),
+                                types: td.eq(4).text(),
+                                protocol: td.eq(5).text(),
+                                speed: td.eq(6).find('.bar').attr('title'),
+                                time: td.eq(7).find('.bar').attr('title'),
                             });
+
+                            ipOne.save(function(err){
+                                if(err) return console.error(err);
+                                console.log('写入成功！');
+
+                                ipModel.find(function(err, docs){
+                                    if(err) return console.error(err);
+                                    console.log(docs);
+                                });
+                            });
+
+
+                            // fs.appendFile('./data/' + 'ipList' + '.txt', proxy, 'utf-8', function (err) {
+                            //     if (err) {
+                            //         console.log(err);
+                            //     }
+                            // });
                         }
 
                         // console.log(body);
@@ -71,6 +103,9 @@ function startRequest(ipListURL) {
         });
     });
 }
-startRequest(ipListURL);
+var everyDay = schedule.scheduleJob('0 0 0 * * ?', function(){
+    console.log('everyDay 0:00:00');
+    startRequest(ipListURL);
+});
 
 module.exports = ipListArray;