I'm trying to scrape an internal site which requires a login, so when I try to access a restricted page, I get redirected to a login page. How do I get past this? My initial try to get request to log in doesn't seem to work.
router.get('/scrape', function(req, res) {
request.get('https://blah/cas/login?service=BLAHlogin', {
'auth': {
'user': 'bob',
'pass': 'bobby',
'sendImmediately': true
}
});
var url= 'http://blah';
var addonURL = "http://blah";
request(url, function(error, response, html) {
if(!error) {
console.log(url);
var $ = cheerio.load(html);
$(".evlist-event div.event-title").each(function() {
console.log("FIRST TIME THROUOOOUGUOGHOGHO");
var data = $(this);
console.log(data.children().last().prev().attr("href"));
var eventURL = addonURL + data.children().last().prev().attr("href");
request(eventURL, function(error, response, html){
if (!error) {
//NONE OF THIS SCRAPES SINCE GOES TO LOG IN PAGE
console.log(eventURL);
var $ = cheerio.load(html);
console.log($("#event_body").html());
console.log($(".title").html());
}
else
{
}
});
});
}
else
{
console.log(error);
}
})
});
Any suggestions appreciated, th
anks!