I'm trying to create a Node.js HTTP file streaming server. I want to do the following:
sender --HTTP POST--> Node.js HTTP Server <--HTTP GET-- receiver
I have this already working with the following code:
var app = express();
// ...
app.post('/upload', function (req, res) {
// Get the `req` and `res` arguments of the HTTP GET request
var downloadRequest = getDownloadClientRequest();
// parse a file upload
var form = new formidable.IncomingForm();
form.onPart = function (part) {
if (!part.filename) {
// let formidable handle all non-file parts
form.handlePart(part);
return;
}
downloadRequest.res.writeHead(200, {
'Content-Disposition': 'attachment; filename=' + part.filename
});
part.on('data', function(data) {
downloadRequest.res.write(data);
});
part.on('end', function() {
downloadRequest.res.end();
});
};
form.parse(req, function (err, fields, files) {
res.writeHead(200);
res.write('Successfully streamed');
res.end();
});
});
app.get('/download', function (req, res) {
storeDownloadClientRequest(req, res);
});
What obviously happens is that if the uploading client is much faster than the downloading client, the upload completes very fast and my Node.js process keeps the whole file in memory until the downloading client finishes it.
How can I throttle the uploader's speed? I guess I can pause his stream every now and then but how do I know if the downloader's stream/buffer is filling up?
PS: I throttled the downloading client with wget --limit-rate=100k, if it helps.
part.pipe(downloadRequest) should automatically handle pausing/resuming when buffer is full.