Written File from Node Scrape Incorrect

Question

Written File from Node Scrape Incorrect

I am trying to pull the name, address, and url from the madeinnyc tech map for a personal project in learning how to use mapbox. In the process I wanted to scrape the listings and export them in a json file that I could use for the project. The issue I am running into is that the information is not being correctly written to the json file. When I log the scrapped data in the console, I am receiving the correct format and all of the data, but the written file from the scrapped data is incorrect and only received one random piece of data. I think the current setup is scraping individual pieces and overwriting it multiple times, because I am getting multiple File Success logs in my console. Is my writeFile method incorrect?

Here is the info logged to the console console.log(metadata) (correct data, condensed since you get the idea of the rest of the listings):

{ name: 'Todayships',
  address: '4447 Seton Ave 2',
  url: 'http://todayships.com/' }
{ name: 'Togather',
  address: '45 Main St Suite 404',
  url: 'http://www.togather.com' }
{ name: 'Tomorrow Networks, LLC',
  address: '1270 Avenue of the Americas 2920',
  url: 'http://www.tomorrownetworks.com' }
{ name: 'Topi',
  address: '20 W 20 Street 2nd Floor',
  url: 'http://topi.com' }

output.json

{
    "name": "Berlinica Publishing LLC",
    "address": "255 West 43rd Street Suite 1012",
    "url": "http://www.berlinica.com"
}s.com"
}ackground.html"
}drianyc.com/ersp.html"
}

scrape.js

var request = require('request');
var cheerio = require('cheerio');
var fs = require('fs');

request('http://mappedinny.com/', function (error, response, html) {
  if (!error && response.statusCode == 200) {
    var $ = cheerio.load(html);

    $('li').each(function(i, element){

        var li = $(this)


        var name = li.attr('data-name');
        var address = li.attr('data-address');
        var url = li.attr('data-url');


        var metadata = {
            name : name,
            address : address,
            url : url
        };
        console.log(metadata);

        fs.writeFile('output.json', JSON.stringify(metadata, null, 4), function(err){
                console.log('File Success');
        });

    });
  }
});

javascript
jquery
json
node.js

Answer 1

The problem is that you're asynchronously writing to the same file in a synchronous loop (your each()).

If your intention is to write all results to the same file, you might try:

var results = [];

$('li').each(function(i, element){

  var li = $(this)

  var name = li.attr('data-name');
  var address = li.attr('data-address');
  var url = li.attr('data-url');


  var metadata = {
    name : name,
    address : address,
    url : url
  };
  console.log(metadata);

  results.push(metadata);

});

fs.writeFile('output.json', JSON.stringify(results, null, 4), function(err){
  if (err)
    console.log('File Error: ' + err);
  else
    console.log('File Success');
});