Poll REST service with Node.js

Question

Poll REST service with Node.js

I'm working on a service that would poll Foursquare for certain check-ins every minute or so and save/update the results in a NoSQL database. Is the best approach to wrap an http.request with setInterval and then aggregate the chunked response using the data event emitter? I plan to use the end emitter to parse the JSON and push into a NoSQL DB when the request is complete. Thoughts?

node.js

Answer 1

There might be better ways, but I ended up just using event emitters to process the REST response as follows:

var fourSquareGet = {
    host: 'api.foursquare.com',
    port: 443,
    path: '/v2/venues/search?ll=33.88,-119.19&query=burger*',
    method: 'GET'
};
setInterval(function () {
    var reqGet = https.request(fourSquareGet, function (res) {
        var content;

        res.on('data', function (chunk) {
            content += chunk;
        });
        res.on('end', function () {
            // remove 'undefined that appears before JSON for some reason
            content = JSON.parse(content.substring(9, content.length));
            db.checkins.save(content.response.venues, function (err, saved) {
                if (err || !saved) throw err;
            });
            console.info("\nSaved from Foursquare\n");
        });
    });

    reqGet.end();
    reqGet.on('error', function (e) {
        console.error(e);
    });
}, 25000);

However, I'm not sure why I had parse out "undefined" from the JSON I received from foursquare.

Answer 2

I've fixed the answer by @occasl, and updated for clarity:

var https = require('https');

setInterval(function () {

    var rest_options = {
        host: 'api.example.com',
        port: 443,
        path: '/endpoint',
        method: 'GET'
    };

    var request = https.request(rest_options, function(response) {
        var content = "";

        // Handle data chunks
        response.on('data', function(chunk) {
            content += chunk;
        });

        // Once we're done streaming the response, parse it as json.
        response.on('end', function() {
            var data = JSON.parse(content);

            //TODO: Do something with `data`.
        });
    });

    // Report errors
    request.on('error', function(error) {
        console.log("Error while calling endpoint.", error);
    });

    request.end();
}, 5000);

Answer 3

When I was running into a similar issue, I employed a similar technique and it worked out well. Here's where I got the idea from. Hopefully this will help a bit.