Nodejs Multiple Http Get Example

Recently one of buddy asked me to create an app that would loop through the links from the sitemap.xml and make HTTP GET request. I said, sure why not, but don’t ask me what he wanted to do with it. In case somebody has similar needs then below is the sample code.

var http = require('http');
var URL = require('url');
var parseString = require('xml2js').parseString;
var _ = require('lodash');
var sitemapURL = 'http://www.example.com/sitemap.xml';
http.get(sitemapURL, function(req){
var xml = '';
req.on('data', function (data) {
	xml += data;
if (xml.length > 1e6)
    req.connection.destroy();
});
req.on('end', function () {
		parseString(xml, function (err, result) {
		    _.each(result.urlset.url,function(linkObj, key){
				  	var urlObj = URL.parse(linkObj.loc[0]);
				  	var options = {'host': urlObj.host,'path': urlObj.path, agent:false,'start':process.hrtime()};
					http.get(options, function(res) {
						console.log(linkObj.loc[0] + ' ' + res.statusCode + ' ' + parseInt(process.hrtime(options.start)[1] / 10000000, 10) + 'ms');
					}).on("error", function(e){
						console.log(linkObj.loc[0] + ': ERROR ' + e.message);
					});
		    });
		});
	});
}).on("error", function(e){
  console.log("Got error: " + e.message);
});
 
 

Momin