I'm using async to download uri's on each request and close when a count is reached, however I discovered that i'm still running into the same problem of the files not being downloaded properly and exiting before it reaches it's max. What's the best way to resolve this? As always, thanks for the help.
async.each(images, function (image, callback) {
var uri = image.attribs.src;
var random = (Math.random() * (3 - 1) + 1) | 0;
var stream = fs.createWriteStream(outFile);
request(uri).pipe(stream);
stream.on('finish', callback);
console.log('[' + chalk.green('✓') + ']' + ' wrote image ' + count);
count++;
if (count === commander.max) {
process.exit(0);
}
sleep.sleep(random);
}, function (error) {
if (error) {
console.log('[' + chalk.red('×') + ']' + ' failed on image ' + count);
}
});
You are not actually waiting for each request to complete, nor are you ever invoking the callback. You need to listen for the finish event on the writable stream that you are sending to the pipe. So, you will want your code to look something like this:
async.each(images, function (image, callback) {
var uri = image.attribs.src;
var random = (Math.random() * (3 - 1) + 1) | 0;
var writable = fs.createWriteStream(outFile);
request(uri).pipe(writable);
writable.on('finish', callback);
console.log('[' + chalk.green('✓') + ']' + ' wrote image ' + count);
count++;
if (count === commander.max) {
process.exit(0);
}
sleep.sleep(random);
}, function (error) {
if (error) {
console.log('[' + chalk.red('×') + ']' + ' failed on image ' + count);
}
});
See more information here: http://nodejs.org/api/stream.html#stream_event_finish