How to avoid the data of request stream loss after doing some authentication on node.js?

Question

How to avoid the data of request stream loss after doing some authentication on node.js?

How dose request stream works with node.js (express or restify) ?

I mean when a client who try to upload a audio/mpeg or other binary file to the node server, the request should be a Writable Stream on the server. and then we could use request.pipe() to do some operations with the other stream.
ex: get the file from request, and then put the file to amazon s3 by knox.

my question is:

is the data of stream which from client just exists on server memory ? or node.js just already put the data on some local temp folder?

cus I'm thinking about which way is the best for put file to amazon s3 from client.

my solution is:

request.pipe , using the file body from client in express or restify, if I set some authentications which using sync function to verify user, then request can't not trigger stream event like 'data' , 'end', or the data would be loss, the length won't match with content-type.

here is my simple code, in this case, request stream events won't trigger or data just broken.

var express = require('express'),
app = express(),
passport = require('passport'),
BasicStrategy = require('passport-http').BasicStrategy;

var users = [
    { id: 1, username: 'bob', password: 'secret', email: 'bob@example.com' }
    , { id: 2, username: 'joe', password: 'birthday', email: 'joe@example.com' }
];

function findByUsername(username, fn) {
    for (var i = 0, len = users.length; i < len; i++) {
        var user = users[i];
        if (user.username === username) {
            return fn(null, user);
        }
    }
    return fn(null, null);
}

passport.use(new BasicStrategy(
    function(username, password, done) {
        process.nextTick(function () {
            findByUsername(username, function(err, user) {
                if (err) { return done(err); }
                if (!user) { return done(null, false); }
                if (user.password != password) { return done(null, false); }
                return done(null, user);
            })
        });
    }));

app.configure(function() {
    app.use(express.logger());
    app.use(passport.initialize());
    app.use(app.router);
});    

app.post('/upload', 
    passport.authenticate('basic', { session: false }),
    function(req, res, next) {

        var dataLength = 0;

        req.on('data', function(chunk) {
            console.log('loading');
            dataLength += chunk.length;

        }).on('end', function() {
            console.log('load end');
            console.log('contentLength: %s', req.headers['content-length']);
            console.log('dataLength:  : %s', dataLength);
            res.send(200);

        }); 
    });

app.listen(8080, function() {
    console.log('server is running');
});

formidable, using multi-part body from client it's more stable for me, but I always using it from restify or express default bodyParser

Is there any ways to avoid request miss stream events or loss data after authenticate? should I focus on server middleware?

node.js
express
passport.js
restify

Answer 1

Lets see, where to begin:

Yes, the request is a ReadStream that will emit data events as you said.
The first chunk of data in the stream is stored in memory, there are no temporary files.

The general issue you are running into is that you need to capture the data emitted from the req and re-emit it once the authentication is completed. Connect provides helpers to do this for you. It looks like passport-http doesn't currently use them.

If you install the pause module it can handle this for you.

Try something like this:

var pause = require('pause');

// Create our own middleware constructor.
var bufferedAuthenticate = function(){
    // Set up standard authenticate handler passing arguments through.
    var authMiddleware = passport.authenticate.apply(passport, arguments);

    // Pass our own middleware instead that wraps passport-http.
    return function(req, res, next){
        // Pause the request before authenticating.
        var obj = pause(req);
        authMiddleware(req, res, function(err){
          next(err);

          // Emit any cached data events that fired while authenticating.
          obj.resume();
        });
    };
};

app.post('/upload',
    bufferedAuthenticate('basic', { session: false }),
    function(req, res, next) {
        var dataLength = 0;

        req.on('data', function(chunk) {
            console.log('loading');
            dataLength += chunk.length;

        }).on('end', function() {
            console.log('load end');
            console.log('contentLength: %s', req.headers['content-length']);
            console.log('dataLength:  : %s', dataLength);
            res.send(200);
        }); 
    }
);