Been trying to figure out why uploading to Amazon S3 is amazingly slow using the putObject command (node.js library). The code below reads an entire directory of files and puts them to S3 asynchronously.
//Read a directory of files
fs.readdir(dir,function(err,files){
//Read each file within the folder
for(var i=0; i < files.length; i++){
var file = files[i];
//Read the File
fs.readFile(path.join(dir,file), function(err, data){
//Create a new buffer
var buffer = new Buffer(data, 'base64');
//Add the pdf to S3
s3.putObject({
'Bucket':bucket,
'Key':path.join(key,file),
'Body':buffer,
'ContentType':mime(file)
},function(err, data) {
//Wait for all the other files to be done
// and perform a callback
});
});
}
});
Tested with a number of different folders with similar results.
Uploading the same files using the AWS web interface takes around 3 sec to complete (or less). Why is using the node.js API so slow??
As per Amazon documentation I've even tried spawning multiple children to handle each upload independently. No changes in upload speed.