I simply can't figure this out. I have a folder with files. I want to create a json with information of these files. To be precise the filename, a regexed version of the filename and additional data I get from a ajax call.
The individual parts work perfectly. In fact, if I run my code in sync, I will get the result I want. Something like this:
[
{
"filename" : "first file",
"regexname" : "first file",
"ajaxresult" : "first file"
},
{
"filename" : "second file",
"regexname" : "second file",
"ajaxresult" : "second file"
},
]
But I want to do it async because the XHR module I use doesn't work properly when trying to do everything 'in sync' and it's of course blocking especially with a lot of files. My 'in sync' code looks like this (a bit simplified to make it shorter )
fs.readdir(datapath, function(err, files) {
var data = new Array();
if(err) throw err;
files.forEach(function(file){
var filename = file
, regex= filename.replace(/regex/)/i, "")
, filepath = './public/config/dataconfig.js'
, ajaxResults = getFile(url)
, mapFiles = null
mapFiles = { filename:filename, regexname:regex, ajaxcall:ajaxResults}
data[data.length] = mapfiles;
}
var JSONdata= JSON.stringify(data, null, 4);
fs.writeFile(filepath, JSONdata, function(e) {
if (!e) {
console.log('Done')
}else{
console.log('Failed', e)
};
});
});
I tried wrapping the ajaxcall in a promise, but I lose the filename and regexname inside the promise. like so:
fs.readdir(datapath, function(err, files) {
var data = new Array();
if(err) throw err;
files.forEach(function(file){
var filename = file
, regex= filename.replace(/regex/)/i, "")
, filepath = './public/config/dataconfig.js'
, mapFiles = null
// Make promise for async call with a url composed of the regex
var promise = getFile(url);
promise.then( function(result) {
mapFiles = { filename:filename, regexname:regex, ajaxcall:result}
data[data.length] = mapfiles;
var JSONdata= JSON.stringify(data, null, 4);
fs.writeFile(filepath, JSONdata, function(e) {
if (!e) {
console.log('Done')
}else{
console.log('Failed', e)
};
});
});
});
This gives me the following result:
[
{
"filename" : "first file",
"regexname" : "first file",
"ajaxresult" : "first file"
},
{
"filename" : "first file",
"regexname" : "first file",
"ajaxresult" : "second file"
},
]
So my final thought was to use something like the async module for node. writing the json after I get callbacks back from the function. but I can't get that to work either.
Thanks in advance!
[EDIT - what I forgot to mention ]
A thing I forgot to mention is that the AjaxCall url is built with the regex. So something like this:
url = 'http://www.test.com/'+regex
So this (as I see it) would mean I do the regex before calling the ajaxcall but I also need the regex after the call to write in the Json. I hope this makes sence
[EDIT 2 - A bit more information]
My XHR call looks like this (without my faulty promise solution. In that case I return the promise with the result):
// Get info
function getFile(url) {
xhr = new XMLHttpRequest();
xhr.open("GET", url);
xhr.onreadystatechange = handler
var ajaxResult = null
//Callback
function handler (){
// If status is ready
if (this.readyState == 4 && this.status == 200) {
ajaxResult = eval(this.responseText);
console.log('XHR OK')
};
};
xhr.send(null);
return ajaxResult[0];
};
Here’s a quick example using async, as you mentioned:
doStuffWithFiles('folder', 'output.json', function () {
console.log('Done!');
});
function doStuffWithFiles (dirPath, outputFilePath, callback) {
fs.readdir(dirPath, function (error, files) {
async.map(
files,
function (file, callback) {
doAjaxStuff(file, function (error, ajaxResults) {
callback(null, {
filename: file,
regexStuff: file.replace(/stuff/, ''),
ajaxResults: ajaxResults
});
});
},
function (error, results) {
var data = JSON.stringify(results);
fs.writeFile(outputFilePath, data, function (error) {
callback();
});
}
);
});
}
The key part is async.map which acts like the standard map function except it requires you to return your results as arguments to the callback passed to your first function. Once all the callbacks have returned, async.map calls the second function, passing it the list of results.
Solution based on promises, that will work properly, may look like:
readdir(datapath).map(function (file) {
return getFile(url)(function (ajaxresult) {
return writeFile('./public/config/dataconfig.js', JSON.stringifiy({
filename: file,
regexname: file.replace(/regex/i, ""),
ajaxcall: ajaxResult
}, null, 4));
});
}).end(function () {
console.log("All done");
}, function (e) {
console.log("Failed", e);
});
In this example I used deferred promise implementation, to make it work you need to make sure that all async functions return promises. Typical callback functions can be converted easily with deferred.promisify:
var promisify = require('deferred').promisify;
var readdir = promisify(fs.readdir);
var writeFile = promisify(fs.writeFile);
if your problem is getting the filename and regex, how about binding them to your callback like
var callback = function(result) {
mapFiles = { filename:this.filename, regexname:this.regex, ajaxcall:result}
data[data.length] = mapfiles;
var JSONdata= JSON.stringify(data, null, 4);
fs.writeFile(filepath, JSONdata, function(e) {
if (!e) {
console.log('Done')
}else{
console.log('Failed', e)
};
}
then
promise.then(callback.bind({'filename':filename,'regex':regex}))