Check for same data doesn t work as expected

Question

Check for same data doesn t work as expected

I have a large MongoDB database, since trying count document make it fall, I needed to make a function for this:

Query is:

M_logs[from]
      .find()
      .select('referer')
      .where('time')
      .gt(lower_bound)
      .lt(upper_bound);

Which give me (reduced, real data are >1k documents):

[ { _id: 53db8f924a1cb7d34a0001e3, referer: '' },
  { _id: 53dbe3ef4a1cb7655b008f4d, referer: '' },
  { _id: 53dbe3ef4a1cb7655b008f4e, referer: '' },
  { _id: 53dbe3ef4a1cb7655b008f4f,
    referer: 'http://www.url1.com'
  { _id: 53dbe3ef4a1cb7655b008f50,
    referer: 'http://url1.com' },
  { _id: 53dbe3ef4a1cb7655b008f51,
    referer: 'http://www.url1.com' }
  { _id: 53dbe3ef4a1cb7655b008f52,
    referer: 'http://www.url1.com' },
  { _id: 53dbe3ef4a1cb7655b008f53,
    referer: 'http://www.url1.com'
  { _id: 53ed5bc64a1cb7f78c00361e,
    referer: 'http://url1.com' },
  { _id: 53ef80384a1cb7019c0000c5,
    referer: 'http://url2'}
]

As you can see, some logs are buggy and empty, some have a url prefixed with www., some not. As I need to show how many visitors each url gave us, I need to parse them to get only 'url1.com', 'url2.com', and ignore empty fields. And count how many time each appear.

Code is:

function referer_process(result, referer, index, j, callback) {

    var ur,
        host;

    result.forEach(function (element) {
        ur = url.parse(element.referer, false, false);

        if (ur.host) {
            if (ur.host.search('www.') === 0) {
                host = ur.host.substring(4);
            } else {
                host = ur.host;
            }
            if (!index[host]) {
                console.log('.' + host + '. ' + host.length);
                index[host] = j;
                j = j + 1;
                referer[index[host]] = {name: host, y: 1};
            } else {
                referer[index[host]].y = referer[index[host]].y + 1;
            }
        }
    });

    callback(referer, index, j);

}

Problem arise when we see the result, if result have the first two rows with the same referer (once parsed), one of them is assigned to a different count, but everything else goes fine.

Example:

url1.com: 5
url1.com: 1
url2.com: 1

I don t understand how it can happen, forEach is synchronous, so the index must have been created when it encounter url1 for the second time!

How can this happen? Where can I search for a solution?

node.js
mongoose

Answer 1

What I would suggest you to do is to use mapReduce in mongo shell instead for these kinds of problem. Here's how you use it:

TIME_UPPER_BOUND = ...
TIME_LOWER_BOUND = ...

var map = function() {
    var host = this.referer;
    if (host && host.substring(0, 4) == 'http') {
        host = host.substring(7);
    }
    if (host && host.substring(0, 3) == 'www') {
        host = host.substring(4);
    }
    emit(host, 1);
};

var reduce = function(key, values) {
    return Array.sum(values);
};

var option = {
    query: {time: {$gt: TIME_LOWER_BOUND, $lt: TIME_UPPER_BOUND}},  
    out: {inline: 1},
};

db.refers.mapReduce(map, reduce, option).results;

With the data you provided above, this will output:

[
    {
        "_id" : "",
        "value" : 3
    },
    {
        "_id" : "url1.com",
        "value" : 6
    },
    {
        "_id" : "url2",
        "value" : 1
    }
]

Pretty clean. Remember to replace refers with your collection name above: db.<collection>.mapReduce. You can find more information on mapReduce here: http://docs.mongodb.org/manual/core/map-reduce/