I have a function in a node module that is a set of three nested callbacks, and that works splendidly, giving me the linearity I need, since each nested callback depends on data from the previous callback. The problem is the second function's callback needs to bubble up and call its parent function recursively. It communicates with an external API. Here is the actual code, with the variables renamed to obfuscate my super top sekrit business logic:
exports.account_usage = function (req, res) {
var domainID = req.body.domains,
startDate = req.body.date_start,
endDate = req.body.date_end,
accountItems = {},
usage = {},
domainStats = {},
page = 0;
//req.cs is a module that communicates with an external API to gather usage data
req.cs.exec("listAccountItems", {
"listall": "true",
"domainid": domainID
},
function (error, result) {
accountItems = result.item;
console.log("listAccountItems callback");
//Get Usage Records
req.cs.exec("listUsageRecords", {
"startdate": startDate,
"enddate": endDate,
"domainid": domainID,
"page": page,
"pagesize": 2000 //try not to DDOS the server. only fetch 2000 records at a time
}, function (error, result) {
usage = req._.extend(usage, result.usagerecord); //this is underscore
console.log("Usage Records: " + usage.length);
page++;
//right here, if result.length === 2000, I need to call
// listUsageRecords until result.length < 2000
//got list of Usage,
//now process usage here
//filter usage item 1
var bytesrecords1 = req._.filter(usage, function (usage) {
return (usage.usagetype === 4);
});
//sum
var bytes1 = req._.reduce(bytesrecords1, function (memo, record) {
return memo + parseInt(record.rawusage, 10);
}, 0);
console.log("Bytes1: " + bytes1);
domainStats.bytes1 = (((bytes1 / 1000) / 1000) / 1000).toFixed(4);
//filter usage item 2
var bytesrecords2 = req._.filter(usage, function (usage) {
return (usage.usagetype === 5);
});
//sum
var bytes2 = req._.reduce(bytesrecords2, function (memo, record) {
return memo + parseInt(record.rawusage, 10);
}, 0);
console.log("Bytes2: " + bytes2);
domainStats.bytes2 = (((bytes2 / 1000) / 1000) / 1000).toFixed(4);
req._.each(accountItems, function (account) {
//get runnning hours
var recs = req._.filter(usage, function (usage) {
return (usage.accountid === account.id && usage.usagetype === 1);
});
account.usage = req._.reduce(recs, function (memo, record) {
return memo + parseInt(record.rawusage, 10);
}, 0);
//filter all the recs for each usage type, 1-14
console.log("Account Usage: " + account.usage);
console.log("Account ID: " + account.name);
});
console.log("ready to render");
res.render('usage', {
"title": "Usage Report",
"domain": domainStats
});
});
});
};
Actual code is also in this fiddle: [http://jsfiddle.net/3wTQA/1/][1] I have used the Google until my fingers have bled and I don't know how to keep the inner callback from continuing instead of recursing. The paging from the external API is required by the API to prevent a DDOS on the remote system in the case of large datasets that need to be fetched.
EDIT: Here is the actual code, annotated and sanitized, with a few examples of the data I'm trying to extract: http://jsfiddle.net/3wTQA/1/
Ok, I got it working. I used the async.whilst function to stay inside the callback until all my data was fetched. here is how I did it:
exports.account_usage = function (req, res) {
var domainID = req.body.domains,
startDate = req.body.date_start,
endDate = req.body.date_end,
accountItems = {},
usage = {},
domainStats = {},
page = 0;
//req.cs is a module that communicates with an external API to gather usage data
req.cs.exec("listAccountItems", {
"listall": "true",
"domainid": domainID
},
function (error, result) {
accountItems = result.item;
console.log("listAccountItems callback");
//Get Usage Records
async.whilst(
function(){return count === pagesize},
function (callback){
req.cs.exec("listUsageRecords", {
"startdate": startDate,
"enddate": endDate,
"domainid": domainID,
"page": page,
"pagesize": 2000 //try not to DDOS the server. only fetch 2000 records at a time
}, function (error, result) {
usage.usagerecord = usage.usagerecord.concat(result.usagerecord);
count = result.usagerecord.length;
console.log("Usage Records: " + usage.length);
page++;
callback();
//now process usage here
},
function (err) {
//filter usage item 1
var bytesrecords1 = req._.filter(usage, function (usage) {
return (usage.usagetype === 4);
});
//sum
var bytes1 = req._.reduce(bytesrecords1, function (memo, record) {
return memo + parseInt(record.rawusage, 10);
}, 0);
console.log("Bytes1: " + bytes1);
domainStats.bytes1 = (((bytes1 / 1000) / 1000) / 1000).toFixed(4);
//filter usage item 2
var bytesrecords2 = req._.filter(usage, function (usage) {
return (usage.usagetype === 5);
});
//sum
var bytes2 = req._.reduce(bytesrecords2, function (memo, record) {
return memo + parseInt(record.rawusage, 10);
}, 0);
console.log("Bytes2: " + bytes2);
domainStats.bytes2 = (((bytes2 / 1000) / 1000) / 1000).toFixed(4);
req._.each(accountItems, function (account) {
//get runnning hours
var recs = req._.filter(usage, function (usage) {
return (usage.accountid === account.id && usage.usagetype === 1);
});
account.usage = req._.reduce(recs, function (memo, record) {
return memo + parseInt(record.rawusage, 10);
}, 0);
//filter all the recs for each usage type, 1-14
console.log("Account Usage: " + account.usage);
console.log("Account ID: " + account.name);
});
console.log("ready to render");
res.render('usage', {
"title": "Usage Report",
"domain": domainStats
});
});
});
}),
};
I think you can break out a pages function within the closure containing biglistofresults.
biglistofresults = {};
function pages(id, page) {
page = page || 0;
module.exec("anotherAPIcall", {"id": id, "page": page },
function (error, result) {
if (result.length === 2000) { //there is probably more data
biglistofresults = _.extend(biglistofresults, result);
pages(id, page + 1);
}
);
}
module.exec("externalAPIcall", {"listall": "true", "domainid": domainID},
function (error, result) {
_.map(result, pages);
});