Nodejs Multiple Http Get Example
Recently one of buddy asked me to create an app that would loop through the links from the sitemap.xml and make HTTP GET request. I said, sure why not, but don’t ask me what he wanted to do with it. In case somebody has similar needs then below is the sample code.
var http = require('http'); var URL = require('url'); var parseString = require('xml2js').parseString; var _ = require('lodash'); var sitemapURL = 'http://www.example.com/sitemap.xml'; http.get(sitemapURL, function(req){ var xml = ''; req.on('data', function (data) { xml += data; if (xml.length > 1e6) req.connection.destroy(); }); req.on('end', function () { parseString(xml, function (err, result) { _.each(result.urlset.url,function(linkObj, key){ var urlObj = URL.parse(linkObj.loc[0]); var options = {'host': urlObj.host,'path': urlObj.path, agent:false,'start':process.hrtime()}; http.get(options, function(res) { console.log(linkObj.loc[0] + ' ' + res.statusCode + ' ' + parseInt(process.hrtime(options.start)[1] / 10000000, 10) + 'ms'); }).on("error", function(e){ console.log(linkObj.loc[0] + ': ERROR ' + e.message); }); }); }); }); }).on("error", function(e){ console.log("Got error: " + e.message); });