I want to make an http call for each element in an array, and combine all the responses into a single array, like so:
result = [] ; i = 0
for item in array
options = get_options(item)
request options, (err, res, body)->
result[i++] = body
// when all calls are done, console.log the 'result' array
I'm looking at the async module, but I'm not really sure how to use it.
Try this, should work. You may want to try mapLimit to limit the no of parallel requests or mapSeries to do it in order.
var async = require('async');
var request = require('request');
async.map(['url1', 'url2'], function(item, callback){
request(item, function (error, response, body) {
callback(error, body);
});
}, function(err, results){
console.log(results);
});
for your example I need to see your get_options function. is it synchronous? if yes you can do this
var async = require('async');
var request = require('request');
var options;
async.map([item1, item2], function(item, callback){
options = get_options(item);
request(options, function (error, response, body) {
callback(error, body);
});
}, function(err, results){
console.log(results);
});
If your get_options is asynchronous and accepts callback do following:
var async = require('async');
var request = require('request');
async.map([item1, item2], function(item, callback){
get_options(item, function(err, option){
if(err) {
callback(err);
}
else{
request(options, function (error, response, body) {
callback(error, body);
});
}
});
}, function(err, results){
console.log(results);
});
using async library's map function you can
var makeRequest = function(item, callback){
var options = get_options(item);
// handle http errors and getting access to body properly
http.get(options,res){
callback(null, res)
}
}
async.map(['item1','item2','item3'], makeRequest(item, callback), function(err, results){
//check errors do stuff with results
});
If you're interested in using promises here is an answer for that as well:
var Q = require('q');
var http = require('http');
var httpGet = function(url) {
var data = "", defer = Q.defer();
http.get(url, function (res) {
var content_length = parseInt(res.headers['content-length'], 10);
var total_downloaded = 0;
if (res.statusCode !== 200) {
defer.reject("HTTP Error " + res.statusCode + " for " + url);
return;
}
res.on("error", defer.reject);
res.on("data", function (chunk) {
data += chunk;
total_downloaded += chunk.length;
var percentage = Math.floor((total_downloaded / content_length) * 100);
defer.notify(percentage);
});
res.on("end", function () { defer.resolve(data); });
});
return defer.promise;
}
This uses the Q promises library. You would use it like this:
var promise = httpGet("http://example.com/");
promise.then(function (data) {
// Do something with data.
}, function (reason) {
// Do something with the error
}, function (progress) {
// Do something with the progress update (% so far)
});
var httpGet = function()... preferred over function httpGet()... in this instance? I'm familiar with the difference theoretically but never found much a difference in the field.