I have node.js file that is connecting to the mongoDB and getting the number of the documents in the collection then it's running in for loop and updating each document with a value them I'm getting from an external server, the thing is that i have a lot of connections open at a time and the program is crashing, my question is how can i use 1 connection at a time and multiple connections this is my code:
var request = require('request');
var cheerio = require ('cheerio');
var fs = require('fs');
var MongoClient = require('mongodb').MongoClient;
//var config = JSON.parse(fs.readFileSync("config.json"));
//var host = config.host;
//var port = config.port;
// Variable Deceleration
//var dbName = "ystocks";
var port = "27017";
var host = "localhost";
var collection = "stocks"
var dbName = "yahooStocks";
function updateStock(){
// getting the number of docs in the collection
getNumOfDocs("stocks", host, port, dbName, function(err, count) {
if (err) {
return console.log(err.message);
}
console.log('number of documents', count);
for(i = 0 ; i <= 20; i++)
{
// Finding each document by _id with a callbak
GetDocumentByID("stocks", host, port, dbName, i, function(err, findOne) {
if (err) {
return console.log(err.message);
}
console.log('Here is the doc that was found', findOne.ticker);
var stockName = findOne.ticker;
console.log("\n* stock name : " +stockName + "\n");
var yUrl = "http://finance.yahoo.com/q/ks?s=" + stockName;
getStockValue(stockName, yUrl);
}); // end of GetDocumentByID
}//end of for loop
}); // end of getNumOfDocs call
}// end of update function
////passed arguments
//stockName = for printing stock name to console
//yUrl = the specific url for the current stock ticker
function getStockValue(stockName, yUrl){
request(yUrl, stockName, function (error, response, body) {
var date = setTimeToLocal(new Date(), 3);
if (!error && response.statusCode == 200) {
var $ = cheerio.load(body);
// the keys - We get them from a certain class attribute
var span = $('.time_rtq_ticker>span');
var stockValue = $(span).text();
console.log("checking the type of the stockValue " + typeof(stockValue));
//*************************************
// parsing the value to int in case it was a String
var parsedValue = parseFloat(stockValue);
console.log("checking the type of the stockValue " + typeof(parsedValue) + " " + parsedValue);
// Calling the setStockValue function which will update the stock value
setStockValue(stockName, parsedValue);
//writehead ???
//response.writeHead(200, {'Content-Type':'text/plain'});
console.log("Response received from -> " + yUrl);
console.log(date);
console.log("Stock - " + stockName + " --> " + stockValue );
//response.write is for printing in browser when we have a server running
//response.write("Welcome to stockwatch\n");
//response.write("Stock - " + ticker + " current value--> " + stockValue + "\n" );
//TODO #1
//ALEX : lines 61-72 -> should be replaced with a function call that
// connects to the DB and updates the specified stock
//i think it sould look like this : funcName( tickerName , currenValue ){...}
var json = {stockName : "", price : "", date : ""};
json.price = stockValue;
json.stockName = stockName;
json.date = date;
var tempName = json.stockName +'.json';
fs.writeFile( tempName ,JSON.stringify(json, null, 4) , function(err) {
if(!err) {
console.log("File successfully written");
}
});//end of filewrite
//end of TODO #1
//response.end("res end");
}//end of !error && response.statusCode == 200
else if (response.statusCode == 404){
console.log("Response failed from " + yUrl + " --> error code: " + response.statusCode);
}//end of statusCode == 400
});// end of request
}//end of getStockValue
// updating a stock with a provided ticker and value and the MongoClient
// edited by Alex Brodov on : AUG 21 2014
function setStockValue(ticker, value) {
// open the connection the DB server
MongoClient.connect("mongodb://localhost:27017/yahooStocks", function (error, db){
if(error) throw error;
var query = {'ticker' : ticker};
operator ={'$set' : {'value' : value}};
// find one document in our collection
db.collection('stocks').update(query, operator, function(error, updated){
if (error) throw error;
//print the result
console.dir("Successfully updated " + updated + " document");
// close the DB
db.close();
});// end of update
}); // Connection to the DB
} // end of setStockValue
// Gets the local date and the desired offset time
// set
function setTimeToLocal(date, offset ) {
// getting the local tome in millseconds
var localtime = date.getTime();
// getting the local offset in millseconds
var localOffset = date.getTimezoneOffset()*60000;
var utc = localOffset + localtime;
// Jerusalem offset
// var offset = 3;
// Jerusalem time in millseconds
var jerusalem = utc + (offset*3600000);
var d = new Date(jerusalem);
console.log("Jerusalem Local Time: " + d.toLocaleString());
return d;
} // end of SetTimeToLocal
/**
* Gets the number of documents from a collection
* With a provided collection name, DB host, DB port, DB name and a callback function
* Edited by Alex
*/
function getNumOfDocs (collectionName, host, port, dbName, callback) {
var mongoConnection = MongoClient.connect("mongodb://" + host + ":" + port + "/" + dbName, function (error, db){
if(error) return callback(err);
db.collection(collectionName).count({}, function(error, numOfDocs){
if(error) return callback(err);
db.close();
callback(null, numOfDocs);
}); // end of count
}); // end of connection to DB
} // end of getNumOfDocs
/**
* Gets a document
* With a provided collection name, DB host, DB port, DB name, the document ID and a callback
* Edited by Alex Brodov on Aug 24th 2014
*/
function GetDocumentByID (collectionName, host, port, dbName, docID, callback) {
// open the connection to DB server
MongoClient.connect("mongodb://" + host + ":" + port + "/" + dbName, function (error, db){
if(error) return callback(error);
var query = {'_id' : docID};
// find one document in our collection
db.collection(collectionName).findOne(query, function(error, doc) {
if (error) return callback(error);
console.log(doc);
// close the DB
db.close();
callback(null, doc);
});// end of findOne
}); // Connection to the DB
}
var refreshInterval = setInterval(updateStock, 30000);
In a MongoDB/Node application you should do the MongoClient.connect() once, at the startup, and then reuse the db object directly as much as you need. The MongoClient is in fact a connection pool and efficient.
So you should review you code, to connect only once, and it should solve the problem (faster program and less connections)
remove all the mongoclientconnect, callback and db.close from your programm
they add the following "around your code"
MongoClient.connect("mongodb://localhost:27017/yahooStocks", function (error, db){ if(error) throw error; // all your methods here // ... // .. var refreshInterval = setInterval(updateStock, 3000); });