Why do I get 8 responses from 10 external http requests in Node.js

My Node.js app makes two types of http requests server side before sending back processed information to the client:

  • The first type of server side request gets a list of links to fetch form an external API (only 1 request);
  • The second type of server side request fetches the html of a web page (repeated 10 times as the list returned by the API contains 10 URLs).

Even if I see that the 10 URLs are requested, only 8 responses are generated. Here is the code:

app.get('/search', function(clientReq, clientRes) {
    console.log(clientReq.query.r);
    // first type of request to retrieve the json data containing 10 URLs
    var searchReq = https.request({ host: 'myhost.com', path: '/?q=' + clientReq.query.r }, function(searchRes) {
        console.log('searching...');
        var searchOutput = '';
        var communications = [];
        var waiting = 0;

        searchRes.on('data', function(d) {
            searchOutput += d;
        });

        searchRes.on('end', function() {
            var obj = JSON.parse(searchOutput);
            for(var i = 0; i < obj.results.length; i++){
                waiting++;
                console.log(encodeURIComponent(obj.results[i].uniqueId)); // prints 10 URLs
                // second type of request, fetches a web page listed in searchOutput
                var fetchReq = https.request({ host: 'myhost.com', path: '/?uniqueId='+ encodeURIComponent(obj.results[i].uniqueId) }, function(fetchRes) {
                    var htmlOutput = '';
                    console.log("starting to read the fetch response"); // only logs 8 times
                    fetchRes.on('data', function(d) {
                        htmlOutput += d;
                        if (htmlOutput.toString().indexOf("<div>") !== -1) {
                            var communication = "";
                            var $ = cheerio.load(htmlOutput.toString().substring(htmlOutput.toString().indexOf("<body "),htmlOutput.toString().indexOf("<div>")));
                            $('p').each(function (index) {
                                communication += $(this).text();
                            });
                            communications.push({"x":communication});
                            fetchRes.destroy();
                            waiting--;
                            complete();
                        }
                    });
                    fetchRes.on('end', function() {
                        console.log('this is the end of the fetch response'); // only logs 8 times
                    });   

                    fetchRes.on('error', function(e) {
                        console.error(e);
                    });
                });
                fetchReq.end(console.log("fetchReq ended")); // prints 10 times
                fetchReq.on('error', function(e) {
                    console.error(e);
                });  
            };    
            function complete() {
                if(waiting === 0) {
                    clientRes.send(communications);
                }
            }
        });
    });

    searchReq.end();

    searchReq.on('error', function(e) {
        console.error(e);
    });
});

I use fetchRes.destroy(); as the HTML can be huge and I only need the top of it.

This is the output in my console, starting with the 10th fetchReq ended:

fetchReq ended
starting to read the fetch response
this is the end of the fetch response
starting to read the fetch response
this is the end of the fetch response
starting to read the fetch response
this is the end of the fetch response
starting to read the fetch response
this is the end of the fetch response
starting to read the fetch response
this is the end of the fetch response
starting to read the fetch response
this is the end of the fetch response
starting to read the fetch response
this is the end of the fetch response
starting to read the fetch response
this is the end of the fetch response

As you can see, I only got 8 responses. The end result is that the communications is never sent back to the browser as waiting does not reach 0.