Dealing with asynchronous functions. Custom callback?

Question

I have the code below and am trying to access the all_records array once the _.each function has completed. However as it is asynchronous I was wondering if was possible to force a callback onto the underscores each?

var request = require('request'),
    cheerio = require('cheerio'),
    _       = require('underscore');

var all_records = [];

_.each([0,100], function(start) {

  var base_url = "http://www.example.com/search?limit=100&q=foobar&start=";
  var url = base_url + start;

  request(url, function(err, res, body) {
    var $ = cheerio.load(body),
      links = $('#results .row');
    $(links).each(function(i, link) {
      var $link = $(link);
      var record = {
        title: $link.children('.title').text().trim()
      };
      all_records.push(record);
    });
  });
});

// Need to run this once _.each has completed final iteration.
console.log(all_records);

Benjamin Gruenbaum · Accepted Answer · 2013-03-30 17:02:28Z

2

Here is a simple solution using a simple synchronization method:

var count = 101;//there are 101 numbers between 0 and 100 including 0 and 100
_.each([0,100], function(start) {

  var base_url = "http://www.example.com/search?limit=100&q=foobar&start=";
  var url = base_url + start;

  request(url, function(err, res, body) {
    var $ = cheerio.load(body),
      links = $('#results .row');
    $(links).each(function(i, link) {
      var $link = $(link);
      var record = {
        title: $link.children('.title').text().trim()
      };
      all_records.push(record);
      count--;
      if(count===0){//101 iterations done
         console.log(all_records);
      }
    });
  });
});

A more elegant solution can be accomplied by using async's .parallel method.

var requests = []; //an array for all the requests we will be making

for(var i=0;i<=100;i++){
   requests.push((function(done){ //create all the requests
       //here you put the code for a single request. 
       //After the push to all_records you make a single done() call 
       //to let async know the function completed
   }).bind(null,i));//the bind is that so each function gets its own value of i   
}
async.parallel(requests,function(){
    console.log(all_records);
});

answered Mar 30, 2013 at 17:02

Benjamin Gruenbaum

278k90 gold badges524 silver badges524 bronze badges

Sign up to request clarification or add additional context in comments.

3 Comments

fatnic Over a year ago

async was the way to go but I ended up using async.forEach in the end.

Benjamin Gruenbaum Over a year ago

Great, if this solution solved your problem, please consider accepting this answer. Also, I think you meant async.each and not foreach.

fatnic Over a year ago

Yes, it is each according to the docs. Although forEach works also.

fatnic · Accepted Answer · 2013-03-30 19:47:18Z

0

async.each ended up being the easiest to implement.

async.each([0,100], function(start) {
  var base_url = "http://www.example.com/search?limit=100&q=foobar&start=";
  var url = base_url + start;

  request(url, function(err, res, body) {
    var $ = cheerio.load(body),
      links = $('#results .row');
    $(links).each(function(i, link) {
      var $link = $(link);
      var record = {
        title: $link.children('.title').text().trim()
      };
      all_records.push(record);
    });
  });
}, function(err){
  console.log(all_records);
});

edited Mar 30, 2013 at 19:47

answered Mar 30, 2013 at 19:41

fatnic

5096 silver badges12 bronze badges

Collectives™ on Stack Overflow

Dealing with asynchronous functions. Custom callback?

2 Answers 2

3 Comments

Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

2 Answers 2

3 Comments

Comments

Your Answer

Sign up or log in

Post as a guest

Related