Slow tailable cursor on capped collections

Question

Slow tailable cursor on capped collections

I'm using MubSub to allow users to subscribe to a certain query and get pushed updates as soon as they're available. This library uses capped collections to obtain a tailable cursor. The problem I'm having is that when I have just one tailable cursor everything goes fine. It takes about a few ms to acquire a cursor. But as I add more subscriptions (and thus open more cursors), the receiving of the cursor can sometimes take up to 8 seconds. I've tried adding indexes but that didn't help at all.

Here are the stats of my collection:

{
        "ns" : "mDB.myCollection",
        "count" : 395669,
        "size" : 325551880,
        "avgObjSize" : 822.7884418541761,
        "storageSize" : 1000001536,
        "numExtents" : 1,
        "nindexes" : 3,
        "lastExtentSize" : 1000001536,
        "paddingFactor" : 1,
        "flags" : 1,
        "totalIndexSize" : 81678240,
        "indexSizes" : {
                "subscriptionIndex" : 32704000,
                "_id_" : 11593568,
                "subscriptionQueryAsc" : 37380672
        },
        "capped" : 1,
        "max" : 2147483647,
        "ok" : 1
}

This is the piece of code that is taking too long to execute:

this.collection.then(handle(true, function(collection) {
    var latest = null;
    // The next statement takes a few ms for the first cursor, 
    // then 5+ seconds for more cursors
    collection.find({}).sort({ $natural: -1 }).limit(1).nextObject(handle(function(doc) {
        if (doc) latest = doc._id;

        (function poll() {
            if (latest) query._id = { $gt: latest };

            var options = { tailable: true, awaitdata: true, numberOfRetries: -1 };
            var cursor = collection.find(query, options).sort({ $natural: 1 });

            (function more() {
                cursor.nextObject(handle(function(doc) {
                    if (!doc) return setTimeout(poll, self.wait);

                    callback(doc);
                    latest = doc._id;
                    more();
                }));
            })();
        })();
    }));
}));

Is this a known problem, or am I just doing something wrong?

node.js
mongodb
publish-subscribe

Answer 1

I fixed this problem by deleting the following lines in the code pasted above:

collection.find({}).sort({ $natural: -1 }).limit(1).nextObject(handle(function(doc) {

That particular statement made the code very slow, probably because it is fetching all ({}) documents, and somehow the number of cursors slow the process down. I did something like this:

this.collection.then(handle(true, function(collection) {
    var latest = null;
    if (doc) latest = doc._id;

    (function poll() {
        if (latest) query._id = { $gt: latest };

        var options = { tailable: true, awaitdata: true, numberOfRetries: -1 };
        var cursor = collection.find(query, options).sort({ $natural: 1 });

        (function more() {
            cursor.nextObject(handle(function(doc) {
                if (!doc) return setTimeout(poll, self.wait);

                callback(doc);
                latest = doc._id;
                more();
            }));
        })();
    })();
}));

I don't fully understand why the author of MubSub did this because it doesn't matter what the _id of the last document is. This is because the documents are inserted in a capped collection, which preserves the insertion order.