With the code below I have been trying to download a URL, save to file and than check if there are any html tags in that file (the tags I want to check are in the checks.json). I need to run this from the commandline and for that I think i need the asynch, callback features of the javascript language.
And I am lost at this point. I cannot make it work. My problem is that from the commandline I always get the error that the file does not exists.
I know it needs to wait until it is downloaded and then run the check. But it seems every time I run the code, that function is not called. And I don't know why.
So what I would need to understand is:
How I can run this from the commandline, download the URL, save it to file, check with checks.json and print the result to console.
Thank you.
var fs = require('fs');
var program = require('commander');
var cheerio = require('cheerio');
var rest = require('restler');
var HTMLFILE_DEFAULT = "index.html";
var CHECKSFILE_DEFAULT = "checks.json";
var URLFILE_DEFAULT = "downloaded.html";
var assertFileExists = function(infile) {
var instr = infile.toString();
if(!fs.existsSync(instr)) {
console.log("%s does not exist. Exiting.", instr);
process.exit(1); // http://nodejs.org/api/process.html#process_process_exit_code
}
return instr;
};
var cheerioHtmlFile = function(htmlfile) {
return cheerio.load(fs.readFileSync(htmlfile));
};
var loadChecks = function(checksfile) {
return JSON.parse(fs.readFileSync(checksfile));
};
var checkHtmlFile = function(htmlfile, checksfile) {
$ = cheerioHtmlFile(htmlfile);
var checks = loadChecks(checksfile).sort();
var out = {};
for(var ii in checks) {
var present = $(checks[ii]).length > 0;
out[checks[ii]] = present;
}
return out;
};
var clone = function(fn) {
// Workaround for commander.js issue.
// http://stackoverflow.com/a/6772648
return fn.bind({});
};
var downAnd2File = function() {
rest.get('http://www.wired.com/').on('complete', function(result) {
if (result instanceof Error) {
sys.puts('Error: ' + result.message);
this.retry(5000); // try again after 5 sec
} else
{
fs.writeFile(__dirname + '/downloaded.html', result, function(err) {
if (err) throw err;
console.log('Saved!');
});
}
});
downAnd2File(checkHtmlFile);
}
if(require.main == module) {
program
.option('-c, --checks <check_file>', 'Path to checks.json', clone(assertFileExists), CHECKSFILE_DEFAULT)
.option('-f, --file <html_file>', 'Path to index.html', clone(assertFileExists), HTMLFILE_DEFAULT)
.option('-u, --url <html_file>', 'Path to downloaded url', clone(assertFileExists), URLFILE_DEFAULT) ///////////////
.parse(process.argv);
var down2FileAndCheck = downAnd2File(checkHtmlFile(program.url, program.checks));
var checkJson = checkHtmlFile(program.file, program.checks);
var outJson = JSON.stringify(checkJson, null, 4);
console.log(outJson);
} else {
exports.checkHtmlFile = checkHtmlFile;
}
You don't seem to understand how callbacks work in node.js. You will need to do some reading about it on stackoverflow or elsewhere.
I rewrote your program (but didn't test it), study it and ask me if you have any questions.
// halts if the provided filename doesn't exist
function assertFileExists(filename) {
if (!fs.existsSync(filename)) {
console.log("%s does not exist. Exiting.", filename);
process.exit(1);
}
return filename;
}
// loads checks from a file
function loadChecks(checksfile) {
return JSON.parse(fs.readFileSync(checksfile)).sort();
}
// checks html
function checkHtml(html, checks) {
$ = cheerio.load(html);
var out = {};
for(var ii in checks) {
var present = $(checks[ii]).length > 0;
out[checks[ii]] = present;
}
return out;
}
// loads html from a file and checks it
// for exports only
function checkHtmlFile(filename, checks) {
return checkHtml(fs.readFileSync(filename), checks);
}
// downloads html from the internet
// callback is called with two arguments: err, html
// where err is null if there is no error
function download(url, callback) {
var resp = rest.get(url);
resp.on('complete', function(result) {
if (result instanceof Error) {
// callback(result);
sys.puts('Error: ' + result.message);
this.retry(5000); // try again after 5 sec
return;
}
callback(null, result);
});
}
if (require.main == module) {
program
.option('-c, --checks <check_file>', 'Path to checks.json', assertFileExists, CHECKSFILE_DEFAULT)
.option('-f, --file <html_file>', 'Path to index.html', assertFileExists, HTMLFILE_DEFAULT)
.option('-u, --url <url>', 'Path to downloaded url') ///////////////
.parse(process.argv);
// this function loads checks & checks html
function check(err, html) {
if (err) {
console.log('Error getting html: ' + err);
process.exit(1);
}
var checks = loadChecks(program.checks);
var checkJson = checkHtml(html, checks);
var outJson = JSON.stringify(checkJson, null, 4);
console.log(outJson);
}
if (program.url) {
// download the provided url and then check the html
download(program.url, check);
} else if (program.file) {
// load html from a file and then check it
fs.readFile(program.file, check);
}
} else {
exports.loadChecks = loadChecks; // for loading checks
exports.checkHtmlFile = checkHtmlFile; // for checking a file
}
As Mak mentioned already, you don't really need to download and save the html you just fetched.
Here's a minor refactor over the original code. The idea is to fetch the html (either from the url or the saved file). Once the result/data is ready, call the method to run the check on the content.
function checkHtml(err, html){
if (err){
console.log('Error: ' + err);
process.exit(1);
}
var checkJson = checkHtmlFile(html, program.checks);
var outJson = JSON.stringify(checkJson, null, 4);
console.log(outJson);
}
if(require.main == module){
program
.option('-c, --checks <check_file>', 'Check path', clone(assertFileExists), CHECKS_DEF)
.option('-f, --file <html_file>', 'File path', clone(assertFileExists), HTML_DEF)
.option('-u, --url <url_pointer>', 'Url link that needs to be graded')
.parse(process.argv);
if (program.url){
rest.get(program.url)
.on('complete', function(result){
checkHtml((html instanceof Error), result);
});
} else {
fs.readFile(program.file, checkHtml);
}
}
Hope this helps.
Just in case your problem is happening before any JS code is executed:
You might want to make the file executable
chmod +x <filename>
and add this statement on top of you file:
#!/usr/bin/env node
Sorry if it's already the case