Casperjs Running Out Of Memory

I'm running the following script with CasperJS and after about 1/3rd of the way through the array it starts running out of swap space and the machine becomes extremely slow. What

Solution 1:

Well turns out this is a very well-known issue with PhantomJS. 3+ years as an open bug and apparently it has something to do with QT Webkit. Nonetheless, i was able to solve it by closing each page during the loop and re-opening a new Phantom page. It's a bit of a hacky work-around, but the memory consumption is far less. However, after about 200 pages, it still has a pretty high memory usage (1GB+). So, i break up my scripts into blocks of 200 and just start the next one upon completion. Here is the finished product that completes successfully without too much memory usage. It uses less on MacOS than Windows for some reason.

casper.start(url,function(){
    this.echo('continuing captures...');
}).each(searchPages,function(casper,index){
    loadSearch(this,index);
});

functionloadSearch(casper,index){
    var currentTime = newDate();
    var month = currentTime.getMonth() + 1;
    var day = currentTime.getDate() + 1;
    var year = currentTime.getFullYear();
    var dateStart = month + "/" + day + "/" + year;
    var fortnightAway = newDate(+newDate + 12096e5);
    var dateEnd = fortnightAway.getMonth() + 1 + "/" + fortnightAway.getDate() + "/" + fortnightAway.getFullYear();

    casper.page.close();
    casper.page = require('webpage').create();

    casper.thenOpen(url,function(){
        var myfile = "data-"+year + "-" + month + "-" + day+".html";
        this.evaluate(function(j) {
            document.querySelector('select[name="searchParameters.localeId"]').selectedIndex = j;
        },index);
        this.evaluate(function(start) {
            $("#leaveDate").val(start);
        },dateStart);
        this.evaluate(function(end) {
            $("#returnDate").val(end);
        },dateEnd);
        this.evaluate(function() {
            $("#OSB_btn").click();
        });
        this.waitForSelector('#destinationForPackage', function() {
            if (this.exists('#destinationForPackage')){
                var name = casper.evaluate(function() {
                    return $("#destinationForPackage option[value='" + $("#destinationForPackage").val() + "']").text()
                });
                if (name != "Going To"){
                    if (name == null){
                        console.log("it's null");
                    }else{
                        name = name.replace("/","_");
                        name = name.replace("/","_");
                        casper.capture('Captures/Searches/search_' + name + '.jpg');
                        console.log("Capturing search_" + name);
                    }
                }
            }else{
                console.log("Search failed to load. Retrying");
                loadSearch(casper,index);
            }

        },function(){
            console.log("Search page timed-out. Retrying");
            loadSearch(casper,index);
        },20000);
    });
}

Solution 2:

There might be a better solution to the original issue, but for a quick fix on running out of memory, try setTimeout to make the recursive call without winding up the stack...

setTimeout(() =>loadSearch(casper,index), 0);

(This idea assumes that the memory issue is the result of too much recursive depth over a long wait time).