Node.js Express application taking too much time to process outbound HTTP requests under load testing

I have an express (node.js v0.10.29) proxy on a linux machine (Ubuntu 12.04 64bit, 3.75GB Memory, 1 Core) which is making some outbound HTTP requests for each incoming request. While load testing I found out that the response time becomes very slow (around 30 seconds for a request that making 4 outbound requests when firing 1000). After some investigation I made sure that the outbound requests are the bottleneck and eliminated the machine limitation (the cpu and memory are not getting higher then 20%, and I increased the number of open files to 10000). First I was using request module for the outbound request, tried changing it to http module, and for both of them tried increasing the globalAgent.maxSockets, using agent = false, using my own agent with any number of maxSockets, setting request.setNoDelay(true), using cluster, but nothing made any change on the results of my load testing.

what can be the problem?

Here is my latest code for the HTTP request:

var http = require('http');
var agent = new http.Agent();
agent.maxSockets = 100;

var doPost = function(reqUrl, body, next, onError) {
    var stringBody = JSON.stringify(body);
    var headers = {
        'Content-Type': 'application/json',
        'Accept-Encoding': 'gzip',
        'Content-Length': stringBody.length
    };
    var parsedUrl = url.parse(reqUrl);
    var options = {
        host: parsedUrl.host,
        path: parsedUrl.path,
        method: 'POST',
        headers: headers,
        //agent: false
        agent: agent
    };

    doHttpRequest(options, stringBody, next, onError);
};


function doHttpRequest(options, body, next, onError, HttpContext){
    var req = http.request(options, function(res) {
         var chunks = [];
         res.on('data', function (chunk) {
            chunks.push(chunk);
         });

         res.on('end', function(){
            var buffer = Buffer.concat(chunks);
            var encoding = res.headers['content-encoding'];
            if (encoding == 'gzip') {
                zlib.gunzip(buffer, function(error, decoded) {
                    var jsonRes = JSON.parse(decoded && decoded.toString());
                    next(jsonRes);
                });
            } else if (encoding == 'deflate') {
                zlib.inflate(buffer, function(error, decoded) {
                    var jsonRes = JSON.parse(decoded && decoded.toString());
                    next(jsonRes);  
                });
            } else {
                next(null, buffer.toString());
            }
        });
    });

    req.setNoDelay(true);
    req.write(body);
    req.end();
    req.on('error', function(e) {
        log(e);
    });
}

the "next" method will call the "doPost" function a few times (in this case 4 times).

I am seeing the same behaviour, I have simple proxy module created. When I call a endpoint directly its taking 50ms, but via proxy server (which internal make another req and pipe to original req) it taking double ~100ms. I have also tried all options what you mentioned here.

I am glad I am not alone having this issue, will be digging into further. Will let you know if found something.

UPDATE

After setting keepAlive=true on agent, I was able to get result for proxied request with ~5ms more than direct call.