Building an object from multiple asynchronous sources in node.js/express.js

Question

Building an object from multiple asynchronous sources in node.js/express.js

I am having a tough time finding a solution to my problem online and was hoping someone on here might be able to help me. I have an express route that does a few API requests for different JSON objects. I would like to build a JSON response for my client side view but all of my attempts so far either yield previous request data or no data at all.

So my question to you JavaScript pros using node/express js. How do you sync up multiple sources of JSON objects into one single object to be returned to the client side in one response? Is there a library or some callback magic that you use?

Thanks in advance for any and all help!

javascript
json
node.js
asynchronous
express

Answer 1

Async.js(https://github.com/caolan/async) can help with tasks like this. A quick example:

app.get('/resource', function(req, res){
    // object to send back in the response
    var results = {};

    // helper function to make requests and handle the data to results object
    var getData = function(url, propertyName, next){
      http.get(url, function(err, data){
        // let async.js know if there is an error
        if(err) return next(err);
        // save data to results object
        results[propertyName] = data;
        // async.js needs us to notify it when each operation is complete - so we call the callback without passing any data back to
        next();
      });
    };

    // array of operations to execute in series or parallel
    var operations = [];

    operations.push(function(next){
        getData('http://url-one', 'users', next);
    });

    operations.push(function(next){
    getData('http://url-two', 'posts', next);
    });

    // async.js has a few options on how to execute your operations - here we use series
    async.series(operations, function(err){
        if(err){
            throw err;
        }
        // if we get to this point, all of the operations have exectued and called 
        // next() without any errors - send the response with a populated results object.
        res.send(results);
    });

});

I haven't actually tried this code yet but it should give you a good idea on how to do it.

Answer 2

Async is one of the more popular libraries for this purpose. There are many other async and promise libraries that can help with this. There are different methods that have different behaviors depending on what you need. I think series method is what you need, but check the documentation carefully.

var async = require('async');
var request = require('request');


app.get('endpoint',function(req, res){
    async.series([
        function(callback){request.get('url',callback)},
        function(callback){request.get('url2',callback)},
        function(callback){request.get('url'3,callback)},  
    ],
    function(err,results){
        //handle error

        //results is an array of values returned from each one
        var processedData = {
            a: results[0],
            b: results[1],
            c: results[2]
        };
        res.send(processedDAta)
    })

})

You could also do this yourself (and is good practice for learning how to organize your node.js code).. callbackhell is a good write up of using named functions and modules to organize callbacks.

Here is a one possible way to do it. (Not tested)

app.get('/endpoint',function(req, res){
    var dataSources = ['url1', 'url2',url3];
    var requestData = [];

    processRequestData = function(){
    //do you stuff here
    res.send(processedData);
    };
    dataSources.forEach(function(dataSource){
        request.get(dataSource,function(err,result){
            //handle error

            request.data.push(result);
            if(requestData.length == dataSources.length){
                processDataRequest();
            }
        })
    })
});