My Node.js app makes two types of http requests server side before sending back processed information to the client:
Even if I see that the 10 URLs are requested, only 8 responses are generated. Here is the code:
app.get('/search', function(clientReq, clientRes) {
console.log(clientReq.query.r);
// first type of request to retrieve the json data containing 10 URLs
var searchReq = https.request({ host: 'myhost.com', path: '/?q=' + clientReq.query.r }, function(searchRes) {
console.log('searching...');
var searchOutput = '';
var communications = [];
var waiting = 0;
searchRes.on('data', function(d) {
searchOutput += d;
});
searchRes.on('end', function() {
var obj = JSON.parse(searchOutput);
for(var i = 0; i < obj.results.length; i++){
waiting++;
console.log(encodeURIComponent(obj.results[i].uniqueId)); // prints 10 URLs
// second type of request, fetches a web page listed in searchOutput
var fetchReq = https.request({ host: 'myhost.com', path: '/?uniqueId='+ encodeURIComponent(obj.results[i].uniqueId) }, function(fetchRes) {
var htmlOutput = '';
console.log("starting to read the fetch response"); // only logs 8 times
fetchRes.on('data', function(d) {
htmlOutput += d;
if (htmlOutput.toString().indexOf("<div>") !== -1) {
var communication = "";
var $ = cheerio.load(htmlOutput.toString().substring(htmlOutput.toString().indexOf("<body "),htmlOutput.toString().indexOf("<div>")));
$('p').each(function (index) {
communication += $(this).text();
});
communications.push({"x":communication});
fetchRes.destroy();
waiting--;
complete();
}
});
fetchRes.on('end', function() {
console.log('this is the end of the fetch response'); // only logs 8 times
});
fetchRes.on('error', function(e) {
console.error(e);
});
});
fetchReq.end(console.log("fetchReq ended")); // prints 10 times
fetchReq.on('error', function(e) {
console.error(e);
});
};
function complete() {
if(waiting === 0) {
clientRes.send(communications);
}
}
});
});
searchReq.end();
searchReq.on('error', function(e) {
console.error(e);
});
});
I use fetchRes.destroy(); as the HTML can be huge and I only need the top of it.
This is the output in my console, starting with the 10th fetchReq ended:
fetchReq ended
starting to read the fetch response
this is the end of the fetch response
starting to read the fetch response
this is the end of the fetch response
starting to read the fetch response
this is the end of the fetch response
starting to read the fetch response
this is the end of the fetch response
starting to read the fetch response
this is the end of the fetch response
starting to read the fetch response
this is the end of the fetch response
starting to read the fetch response
this is the end of the fetch response
starting to read the fetch response
this is the end of the fetch response
As you can see, I only got 8 responses. The end result is that the communications is never sent back to the browser as waiting does not reach 0.