Is there any solution that will allow me to process this many characters into a string with Node.js?

Question

Is there any solution that will allow me to process this many characters into a string with Node.js?

My question:

The problem:

I need to compute a huge (array, and then) string of character combinations in Node.js, like so:

var fs = require('fs');

var legalChars = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i',
'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 
'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', 
'8', '9', '_'];

var fiveChars = [legalChars, 
    legalChars, 
    legalChars, 
    legalChars, 
    legalChars];

//output all possible combinations of the arrays within
//fiveChars
function allPossibleCases(arr) {
    if (arr.length === 0) {
        return [];
    } else if (arr.length === 1) {
        return arr[0];
    } else {
        var result = [];
        var allCasesOfRest = allPossibleCases(arr.slice(1));
        // recur with the rest of array
        for (var c in allCasesOfRest) {
            for (var i = 0; i < arr[0].length; i++) {
                result.push(arr[0][i] + allCasesOfRest[c]);
                var val = arr[0][i] + allCasesOfRest[c];
            }
        }
        return result; 
    }
}

// join array of results into string
var result = allPossibleCases(fiveChars).join('", "');

// this essentially writes the string in the form of a Node module 
//containing a giant array
fs.writeFile("./data/fiveChars.js", 
'fiveChar = ["' + result + '"]; \r\n module.exports = fiveChar;',
function(err) {
    if(err) {
        console.log(err);
    } else {
        console.log("The file was saved!");
    }
});

I want to continue adding character rows up to 16 rows:

var sixteenChars = [legalChars, legalChars, legalChars, legalChars, legalChars, legalChars, legalChars, legalChars, legalChars, legalChars, legalChars, legalChars, legalChars, legalChars, legalChars, legalChars];

The problem is that Node cant cope with this all at once.

FATAL ERROR: JS Allocation failed - process out of memory Aborted

I don't want to download more RAM, just looking for a way to handle this calculation process over time rather than filling the memory allocation up.

javascript
node.js
out-of-memory

Answer 1

Since I like using highland these days, I thought I'd post a solution here as an exercise, that I think should scale memory-wise to a lot of combinations. It might be too slow for you I don't know because what you want to generate is 37^16 combinations which is a lot.

This solution will create a file containing all solutions separated by a new line for the 5 chars case. Maybe you can try it and see if it fits your box ;-). Going to the 16 chars case will just need adding new flatMap calls.

Keep me posted ! you can tail -f the file to see it growing.

var fs = require('fs');
var _ = require('highland');

var legalChars = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i',
'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u',
'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7',
'8', '9', '_'];


var suffixer = function(alphabet) {
  var len = alphabet.length;
  return function(prefix) {
    var i = 0;
    return _(function(push, next) {
      if (i == len) {
        push(null, _.nil);
      } else {
        push(null, prefix + alphabet[i]);
        i++;
      }
      next();
    })
  }
}

var s = _(['\n'])
        .flatMap(suffixer(legalChars))
        .flatMap(suffixer(legalChars))
        .flatMap(suffixer(legalChars))
        .flatMap(suffixer(legalChars))
        .flatMap(suffixer(legalChars))

s.pipe(fs.createWriteStream('alpha.txt'));

Now why you would want to generate so many combinations is another question !