My Node.js server need to receive large amounts of compressed JSON (gzip) data in POST requests. I've added some code to gunzip
the payload of the POST, but the outcome is always incorrect and varies depending on the size of the JSON posted.
var http = require('http');
var zlib = require('zlib');
var port = process.env.port || 1337;
http.createServer(function(req, res) {
if (req.method == 'POST') {
if( req.headers['content-encoding'] == 'gzip' ) {
var body = [];
req.on('data', function (data) {
console.log('Receiving data...');
body.push(data);
});
req.on('end', function () {
console.log('Data received! Uncompressing...');
var buffer = Buffer.concat(body);
zlib.gunzip(buffer, function(err, result) {
if (!err) {
console.log('Unzipped!!')
var r = result.toString();
console.log(r)
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(r);
} else {
console.log(err);
res.writeHead(500, { 'Content-Type': 'text/plain' });
res.end('Something went wrong!');
}
});
});
}
else {
console.log('Everything OK with uncompressed requests!');
res.writeHead(200, { 'Content-Type': 'text/plain' });
res.end('Some JSON was received!\n');
}
req.on('error', function(err){
console.log(err);
res.writeHead(500, { 'Content-Type': 'text/plain' });
res.end('Something went wrong!');
});
}
else {
res.writeHead(200, { 'Content-Type': 'text/plain' });
res.end('Hello World\n');
}
}).listen(1337);
If I send normal, uncompressed JSON data everything works.
The result with compression:
As pointed out in a comment, this is not incorrect behavior, as the payload is so small that is not compressed.{"id":1, "name":"Gabe"}
-> returns 500 with Error: incorrect header check] errno: -3, code: 'Z_DATA_ERROR' when trying to uncompress the data.
{"id":1,"name":"Gabe","age":33,"address":"555 Road, City, Province","occupation":"I exchange code for money"}
-> the request ends abruptly with a 504 just after the request.on('data'...
, although the process continues and shows the correct uncompressed data, as seen below.
{"id":1,"name":"Gabe","age":666,"address":"666 Road, City, Province","occupation":"I sleep during the day and write code at night... not"}
-> It's stuck receiving the data and does nothing more.
To test, I've configured Fiddler to send compressed data whenever the Content-Encoding header is set with Rules -> Customize Rules:
if (oSession.requestBodyBytes != null && oSession.requestBodyBytes.Length>50 && oSession.oRequest.headers.Exists("Content-Encoding")){
oSession.requestBodyBytes = Utilities.GzipCompress(oSession.requestBodyBytes);
oSession["Content-Length"] = oSession.requestBodyBytes.Length.ToString();
oSession["Content-Encoding"] = "gzip";
}
A sample request:
POST http://localhost:1337/ HTTP/1.1
User-Agent: Fiddler
Content-Encoding: gzip
Host: localhost:1337
{"id":1,"name":"Gabe","age":00,"address":"555 Road, City, Province","occupation":"I exchange code for money"}
I tried listening to errors and using Domains, but I don't see any error generated. I've also tried with Express both doing it manually and using the body-parser module which supports GZIP and Deflate and the behavior is the same. Tested with Node 0.10.18 and 0.10.30, without Express and finally with Express 3 and 4.
So, what going on here? Am I so incredibly blind that I'm missing a colon somewhere? Is this a bug in Node? Is it the way I'm sending the request with Fiddler?