Dealing with asynchronous functions. Custom callback?

Question

Dealing with asynchronous functions. Custom callback?

I have the code below and am trying to access the all_records array once the _.each function has completed. However as it is asynchronous I was wondering if was possible to force a callback onto the underscores each?

var request = require('request'),
    cheerio = require('cheerio'),
    _       = require('underscore');

var all_records = [];

_.each([0,100], function(start) {

  var base_url = "http://www.example.com/search?limit=100&q=foobar&start=";
  var url = base_url + start;

  request(url, function(err, res, body) {
    var $ = cheerio.load(body),
      links = $('#results .row');
    $(links).each(function(i, link) {
      var $link = $(link);
      var record = {
        title: $link.children('.title').text().trim()
      };
      all_records.push(record);
    });
  });
});

// Need to run this once _.each has completed final iteration.
console.log(all_records);

node.js
asynchronous

Answer 1

Here is a simple solution using a simple synchronization method:

var count = 101;//there are 101 numbers between 0 and 100 including 0 and 100
_.each([0,100], function(start) {

  var base_url = "http://www.example.com/search?limit=100&q=foobar&start=";
  var url = base_url + start;

  request(url, function(err, res, body) {
    var $ = cheerio.load(body),
      links = $('#results .row');
    $(links).each(function(i, link) {
      var $link = $(link);
      var record = {
        title: $link.children('.title').text().trim()
      };
      all_records.push(record);
      count--;
      if(count===0){//101 iterations done
         console.log(all_records);
      }
    });
  });
});

A more elegant solution can be accomplied by using async's .parallel method.

var requests = []; //an array for all the requests we will be making

for(var i=0;i<=100;i++){
   requests.push((function(done){ //create all the requests
       //here you put the code for a single request. 
       //After the push to all_records you make a single done() call 
       //to let async know the function completed
   }).bind(null,i));//the bind is that so each function gets its own value of i   
}
async.parallel(requests,function(){
    console.log(all_records);
});

Answer 2

async.each ended up being the easiest to implement.

async.each([0,100], function(start) {
  var base_url = "http://www.example.com/search?limit=100&q=foobar&start=";
  var url = base_url + start;

  request(url, function(err, res, body) {
    var $ = cheerio.load(body),
      links = $('#results .row');
    $(links).each(function(i, link) {
      var $link = $(link);
      var record = {
        title: $link.children('.title').text().trim()
      };
      all_records.push(record);
    });
  });
}, function(err){
  console.log(all_records);
});