I have a case where I need to implement a mailing platform using amazon SES. I have decided to do this using Node.js to achieve the desired concurrency.
Within the node js application, I would fetch all my contacts from a redis storage. What I specifically require is that I need to restrict the application from calling Amazon's api at a particular rate ,say x emails per second, or else I will be throttled by Amazon. Can any one please suggest me on how achieve this rate limiting using Node.js. I have tried the limiter package but couldn't follow the exact working. There isn't enough documentation available as well. Any help would be greatly appreciated.
I have a similar setup that uses mongodb. Messages are queued into a collection by various apps and processed by a separate node app.
var mongoose = require('mongoose'),
_ = require('underscore') ;
var BATCH_SIZE = 50;
var INTERVAL = 60000;
mongoose.connect('mongodb://localhost/outbox');
var Schema = mongoose.Schema,
ObjectId = Schema.ObjectId;
var MessageSchema = new Schema({
to: {type:String},
subject: {type:String},
body: {type:String},
queued: {type:Date}
});
var Message = mongoose.model('Message',MessageSchema);
function processQueue(){
var query = Message.find({}).sort({queued: -1}).limit(BATCH_SIZE);
query.exec(function(err,messages){
if (err) return handleError(err);
if (!messages.length){
console.log('Queue is empty');
}else{
_.each(messages,function(msg){
sendTheSESMessage(msg, function(success){
if (success){
//remove from queue
msg.remove();
}else{
//handle it
}
})
});
}
});
}
setInterval(processQueue, INTERVAL);
EDIT 2: Underscore has already implemented something like this, take a look at the throttle function.
EDIT: I have made a module out of the code below, visit: https://github.com/gammasoft/rate-limited
This is a very simple implementation of something that might help you.
Params:
1. params: array with email addresses to be sent via SES
2. fn: function that will send a single email at a time
3. timeout: minimum time between each call to fn, in miliseconds, eg.: for 100 emails/sec then you pass 1000/100 (1 sec divided by 100 emails)
4. callback: function to be called when everything ends
CODE
// rate-limit.js
var async = require("async");
module.exports = function(params, fn, timeout, callback){
var length = params.length;
var wait = function(cb){
if(--length === 0) return cb();
else setTimeout(function(){
cb();
}, timeout);
};
async.eachSeries(params, async.compose(wait, fn), function(err){
callback(err);
});
};
module.exports.timesPerSecond = function(times){
if(times === 0) throw new Error("Should not be zero");
return 1000/times;
};
module.exports.timesPerMinute = function(times){
if(times === 0) throw new Error("Should not be zero");
return (60 * 1000)/times;
};
USAGE
//test.js
var rateLimited = require("./rate-limit.js");
var timesPerMinute = rateLimited.timesPerMinute;
console.time("time");
rateLimited([1, 2, 3, 4, 5, 6], function(name, cb){
console.log(name);
cb();
}, timesPerMinute(5), function(err){
if(err) throw err;
console.timeEnd("time");
});