Merge branch 'master' into development

Update synaptic.js

.gitignore

@@ -14,3 +14,5 @@ node_modules
 # Demo.
 demo.js
 
+# Degub
+debug.html

bower.json

@@ -0,0 +1,33 @@
+{
+  "name": "synaptic",
+  "version": "1.0.4",
+  "homepage": "https://github.com/cazala/synaptic",
+  "authors": [
+    "Juan Cazala <juancazala@gmail.com>"
+  ],
+  "description": "architecture-free neural network library for node.js and the browser",
+  "main": "./dist/synaptic.min.js",
+  "moduleType": [
+    "amd",
+    "globals",
+    "node"
+  ],
+  "keywords": [
+    "neural",
+    "network",
+    "deep",
+    "learning",
+    "machine",
+    "learning",
+    "lstm",
+    "perceptron"
+  ],
+  "license": "MIT",
+  "ignore": [
+    "**/.*",
+    "node_modules",
+    "bower_components",
+    "test",
+    "tests"
+  ]
+}

gulpfile.js

@@ -44,7 +44,6 @@ gulp.task('debug', function () {
     .bundle()
     .pipe(source('synaptic.js'))
     .pipe(buffer())
-    .pipe(prepend(license))
     .pipe(append(globals))
     .pipe(gulp.dest('./dist'));
 });
@@ -55,7 +54,7 @@ gulp.task('test', function () {
         .pipe(mocha());
 });
 
-// watch for changed and re-build (debug)
+// watch for changes and re-build (debug)
 gulp.task('dev', function () {
    gulp.watch('./src/*.js', ['debug']);
 });

package.json

@@ -1,7 +1,7 @@
 {
   "name": "synaptic",
-  "version": "0.1.7",
-  "description": "Architecture-free neural network library",
+  "version": "1.0.4",
+  "description": "architecture-free neural network library",
   "main": "./src/synaptic",
   "scripts": {
     "test": "mocha test"
@@ -38,5 +38,7 @@
     "url": "https://github.com/cazala/synaptic/issues"
   },
   "homepage": "http://synaptic.juancazala.com",
-  "engines" : { "node" : "^0.10.0" }
+  "engines": {
+    "node": ">=0.10"
+  }
 }

src/architect.js

@@ -1,7 +1,7 @@
 // import
-var Layer = require('./layer'),
-    Network = require('./network'),
-    Trainer = require('./trainer');
+var Layer   = require('./layer')
+,   Network = require('./network')
+,   Trainer = require('./trainer')
 
 /*******************************************************************************************
                                         ARCHITECT
@@ -15,7 +15,7 @@ var Architect = {
 
     var args = Array.prototype.slice.call(arguments); // convert arguments to Array
     if (args.length < 3)
-      throw "Error: not enough layers (minimum 3) !!";
+      throw new Error("not enough layers (minimum 3) !!");
 
     var inputs = args.shift(); // first argument
     var outputs = args.pop(); // last argument
@@ -53,28 +53,28 @@ var Architect = {
 
     var args = Array.prototype.slice.call(arguments); // convert arguments to array
     if (args.length < 3)
-      throw "Error: not enough layers (minimum 3) !!";
+      throw new Error("not enough layers (minimum 3) !!");
 
     var last = args.pop();
     var option = {
       peepholes: Layer.connectionType.ALL_TO_ALL,
-      hiddentohidden: false,
-      outtohidden: false,
-      outtogates: false,
-      intoout: true,
+      hiddenToHidden: false,
+      outputToHidden: false,
+      outputToGates: false,
+      inputToOutput: true,
     };
     if (typeof last != 'number') {
       var outputs = args.pop();
       if (last.hasOwnProperty('peepholes'))
         option.peepholes = last.peepholes;
-      if (last.hasOwnProperty('hiddentohidden'))
-        option.hiddentohidden = last.hiddentohidden;
-      if (last.hasOwnProperty('outtohidden'))
-        option.outtohidden = last.outtohidden;
-      if (last.hasOwnProperty('outtogates'))
-        option.outtogates = last.outtogates;
-      if (last.hasOwnProperty('intoout'))
-        option.intoout = last.intoout;
+      if (last.hasOwnProperty('hiddenToHidden'))
+        option.hiddenToHidden = last.hiddenToHidden;
+      if (last.hasOwnProperty('outputToHidden'))
+        option.outputToHidden = last.outputToHidden;
+      if (last.hasOwnProperty('outputToGates'))
+        option.outputToGates = last.outputToGates;
+      if (last.hasOwnProperty('inputToOutput'))
+        option.inputToOutput = last.inputToOutput;
     } else
       var outputs = last;
 
@@ -129,20 +129,20 @@ var Architect = {
       var self = memoryCell.project(memoryCell);
 
       // hidden to hidden recurrent connection
-      if (option.hiddentohidden)
+      if (option.hiddenToHidden)
         memoryCell.project(memoryCell, Layer.connectionType.ALL_TO_ELSE);
 
       // out to hidden recurrent connection
-      if (option.outtohidden)
+      if (option.outputToHidden)
         outputLayer.project(memoryCell);
 
       // out to gates recurrent connection
-      if (option.outtogates) {
+      if (option.outputToGates) {
         outputLayer.project(inputGate);
         outputLayer.project(outputGate);
         outputLayer.project(forgetGate);
       }
-      
+
       // peepholes
       memoryCell.project(inputGate, option.peepholes);
       memoryCell.project(forgetGate, option.peepholes);
@@ -159,7 +159,7 @@ var Architect = {
     }
 
     // input to output direct connection
-    if (option.intoout)
+    if (option.inputToOutput)
       inputLayer.project(outputLayer);
 
     // set the layers of the neural network
@@ -270,5 +270,4 @@ for (var architecture in Architect) {
 }
 
 // export
-if (module) module.exports = Architect; 
-
+if (module) module.exports = Architect;

src/layer.js

@@ -1,5 +1,9 @@
+// export
+if (module) module.exports = Layer;
+
 // import
-var Neuron = require('./neuron');
+var Neuron  = require('./neuron')
+,   Network = require('./network')
 
 /*******************************************************************************************
                                             LAYER
@@ -9,7 +13,7 @@ function Layer(size, label) {
   this.size = size | 0;
   this.list = [];
   this.label = label || null;
-  this.connectedto = [];
+  this.connectedTo = [];
 
   while (size--) {
     var neuron = new Neuron();
@@ -26,7 +30,7 @@ Layer.prototype = {
 
     if (typeof input != 'undefined') {
       if (input.length != this.size)
-        throw "INPUT size and LAYER size must be the same to activate!";
+        throw new Error("INPUT size and LAYER size must be the same to activate!");
 
       for (var id in this.list) {
         var neuron = this.list[id];
@@ -48,7 +52,7 @@ Layer.prototype = {
 
     if (typeof target != 'undefined') {
       if (target.length != this.size)
-        throw "TARGET size and LAYER size must be the same to propagate!";
+        throw new Error("TARGET size and LAYER size must be the same to propagate!");
 
       for (var id = this.list.length - 1; id >= 0; id--) {
         var neuron = this.list[id];
@@ -65,14 +69,14 @@ Layer.prototype = {
   // projects a connection from this layer to another one
   project: function(layer, type, weights) {
 
-    if (layer instanceof require('./network'))
+    if (layer instanceof Network)
       layer = layer.layers.input;
 
     if (layer instanceof Layer) {
       if (!this.connected(layer))
         return new Layer.connection(this, layer, type, weights);
     } else
-      throw "Invalid argument, you can only project connections to LAYERS and NETWORKS!";
+      throw new Error("Invalid argument, you can only project connections to LAYERS and NETWORKS!");
 
 
   },
@@ -82,7 +86,7 @@ Layer.prototype = {
 
     if (type == Layer.gateType.INPUT) {
       if (connection.to.size != this.size)
-        throw "GATER layer and CONNECTION.TO layer must be the same size in order to gate!";
+        throw new Error("GATER layer and CONNECTION.TO layer must be the same size in order to gate!");
 
       for (var id in connection.to.list) {
         var neuron = connection.to.list[id];
@@ -95,7 +99,7 @@ Layer.prototype = {
       }
     } else if (type == Layer.gateType.OUTPUT) {
       if (connection.from.size != this.size)
-        throw "GATER layer and CONNECTION.FROM layer must be the same size in order to gate!";
+        throw new Error("GATER layer and CONNECTION.FROM layer must be the same size in order to gate!");
 
       for (var id in connection.from.list) {
         var neuron = connection.from.list[id];
@@ -108,7 +112,7 @@ Layer.prototype = {
       }
     } else if (type == Layer.gateType.ONE_TO_ONE) {
       if (connection.size != this.size)
-        throw "The number of GATER UNITS must be the same as the number of CONNECTIONS to gate!";
+        throw new Error("The number of GATER UNITS must be the same as the number of CONNECTIONS to gate!");
 
       for (var id in connection.list) {
         var gater = this.list[id];
@@ -248,8 +252,8 @@ Layer.connection = function LayerConnection(fromLayer, toLayer, type, weights) {
       this.size = this.list.push(connection);
     }
   }
-  
-  fromLayer.connectedto.push(this);
+
+  fromLayer.connectedTo.push(this);
 }
 
 // types of connections
@@ -270,7 +274,3 @@ Layer.gateType.ONE_TO_ONE = "ONE TO ONE";
     return connections++;
   }
 })();
-
-// export
-if (module) module.exports = Layer;
-

src/network.js

@@ -1,6 +1,9 @@
+// export
+if (module) module.exports = Network;
+
 // import
-var Neuron = require('./neuron'),
-    Layer = require('./layer');
+var Neuron  = require('./neuron')
+,   Layer   = require('./layer')
 
 /*******************************************************************************************
                                          NETWORK
@@ -27,8 +30,8 @@ Network.prototype = {
       for (var layer in this.layers.hidden)
         this.layers.hidden[layer].activate();
       return this.layers.output.activate();
-    } 
-    else 
+    }
+    else
     {
       if (this.optimized == null)
         this.optimize();
@@ -48,8 +51,8 @@ Network.prototype = {
       reverse.reverse();
       for (var layer in reverse)
         reverse[layer].propagate(rate);
-    } 
-    else 
+    }
+    else
     {
       if (this.optimized == null)
         this.optimize();
@@ -69,7 +72,7 @@ Network.prototype = {
     if (unit instanceof Layer)
       return this.layers.output.project(unit, type, weights);
 
-    throw "Invalid argument, you can only project connections to LAYERS and NETWORKS!";
+    throw new Error("Invalid argument, you can only project connections to LAYERS and NETWORKS!");
   },
 
   // let this network gate a connection
@@ -142,7 +145,6 @@ Network.prototype = {
       hardcode += "F[" + optimized.variables[i].id + "] = " + (optimized.variables[
         i].value || 0) + "; ";
     hardcode += "var activate = function(input){\n";
-    hardcode += "influences = [];";
     for (var i in optimized.inputs)
       hardcode += "F[" + optimized.inputs[i] + "] = input[" + i + "]; ";
     for (var currentLayer in optimized.activation_sentences) {
@@ -356,21 +358,6 @@ Network.prototype = {
       neurons.push(copy);
     }
 
-    if (!ignoreTraces)
-      for (var i in neurons) {
-        var copy = neurons[i];
-
-        for (var input in neuron.trace.elegibility)
-          copy.trace.elegibility[input] = neuron.trace.elegibility[input];
-
-        for (var gated in neuron.trace.extended) {
-          copy.trace.extended[gated] = {};
-          for (var input in neuron.trace.extended[gated])
-            copy.trace.extended[ids[gated]][input] = neuron.trace.extended[
-              gated][input];
-        }
-      }
-
     // get connections
     for (var i in list) {
       var neuron = list[i].neuron;
@@ -391,8 +378,7 @@ Network.prototype = {
           from: ids[neuron.ID],
           to: ids[neuron.ID],
           weight: neuron.selfconnection.weight,
-          gater: neuron.selfconnection.gater ? ids[neuron.selfconnection.gater
-            .ID] : null,
+          gater: neuron.selfconnection.gater ? ids[neuron.selfconnection.gater.ID] : null,
         });
     }
 
@@ -401,37 +387,37 @@ Network.prototype = {
       connections: connections
     }
   },
-  
+
   // export the topology into dot language which can be visualized as graphs using dot
   /* example: ... console.log(net.toDotLang());
               $ node example.js > example.dot
               $ dot example.dot -Tpng > out.png
   */
-  toDot: function(edgeconnection) {
-    if (! typeof edgeconnection)
-      edgeconnection = false;
+  toDot: function(edgeConnection) {
+    if (! typeof edgeConnection)
+      edgeConnection = false;
     var code = "digraph nn {\n    rankdir = BT\n";
     var layers = [this.layers.input].concat(this.layers.hidden, this.layers.output);
     for (var layer in layers) {
-      for (var to in layers[layer].connectedto) { // projections
-        var connection = layers[layer].connectedto[to];
-        var layerto = connection.to;
+      for (var to in layers[layer].connectedTo) { // projections
+        var connection = layers[layer].connectedTo[to];
+        var layerTo = connection.to;
         var size = connection.size;
         var layerID = layers.indexOf(layers[layer]);
-        var layertoID = layers.indexOf(layerto);
+        var layerToID = layers.indexOf(layerTo);
         /* http://stackoverflow.com/questions/26845540/connect-edges-with-graph-dot
          * DOT does not support edge-to-edge connections
          * This workaround produces somewhat weird graphs ...
         */
-        if ( edgeconnection) {
+        if ( edgeConnection) {
           if (connection.gatedfrom.length) {
-            var fakeNode = "fake" + layerID + "_" + layertoID;
+            var fakeNode = "fake" + layerID + "_" + layerToID;
             code += "    " + fakeNode +
               " [label = \"\", shape = point, width = 0.01, height = 0.01]\n";
             code += "    " + layerID + " -> " + fakeNode + " [label = " + size + ", arrowhead = none]\n";
-            code += "    " + fakeNode + " -> " + layertoID + "\n";
+            code += "    " + fakeNode + " -> " + layerToID + "\n";
           } else
-            code += "    " + layerID + " -> " + layertoID + " [label = " + size + "]\n";
+            code += "    " + layerID + " -> " + layerToID + " [label = " + size + "]\n";
           for (var from in connection.gatedfrom) { // gatings
             var layerfrom = connection.gatedfrom[from].layer;
             var type = connection.gatedfrom[from].type;
@@ -439,12 +425,12 @@ Network.prototype = {
             code += "    " + layerfromID + " -> " + fakeNode + " [color = blue]\n";
           }
         } else {
-          code += "    " + layerID + " -> " + layertoID + " [label = " + size + "]\n";
+          code += "    " + layerID + " -> " + layerToID + " [label = " + size + "]\n";
           for (var from in connection.gatedfrom) { // gatings
             var layerfrom = connection.gatedfrom[from].layer;
             var type = connection.gatedfrom[from].type;
             var layerfromID = layers.indexOf(layerfrom);
-            code += "    " + layerfromID + " -> " + layertoID + " [color = blue]\n";
+            code += "    " + layerfromID + " -> " + layerToID + " [color = blue]\n";
           }
         }
       }
@@ -473,7 +459,7 @@ Network.prototype = {
     // build network activation
     for (var neuron in data.activate) { // shouldn't this be layer?
       for (var sentence in data.activate[neuron])
-        activation += data.activate[neuron][sentence] + "\n";
+        activation += data.activate[neuron][sentence].join('') + "\n";
     }
 
     // build outputs
@@ -538,12 +524,12 @@ Network.prototype = {
   },
 
   // returns a copy of the network
-  clone: function(ignoreTraces) {
-    return Network.fromJSON(this.toJSON(ignoreTraces));
+  clone: function() {
+    return Network.fromJSON(this.toJSON());
   }
 }
 
-// rebuild a network that has been stored in a json using the method toJson()
+// rebuild a network that has been stored in a json using the method toJSON()
 Network.fromJSON = function(json) {
 
   var neurons = [];
@@ -558,14 +544,13 @@ Network.fromJSON = function(json) {
     var config = json.neurons[i];
 
     var neuron = new Neuron();
-    neuron.trace.elegibility = config.trace.elegibility;
-    neuron.trace.extended = config.trace.extended;
+    neuron.trace.elegibility = {};
+    neuron.trace.extended = {};
     neuron.state = config.state;
     neuron.old = config.old;
     neuron.activation = config.activation;
     neuron.bias = config.bias;
-    neuron.squash = config.squash in Neuron.squash ? Neuron.squash[config.squash] :
-      Neuron.squash.LOGISTIC;
+    neuron.squash = config.squash in Neuron.squash ? Neuron.squash[config.squash] : Neuron.squash.LOGISTIC;
     neurons.push(neuron);
 
     if (config.layer == 'input')
@@ -593,7 +578,3 @@ Network.fromJSON = function(json) {
 
   return new Network(layers);
 }
-
-// export
-if (module) module.exports = Network;
-

src/neuron.js

@@ -1,3 +1,6 @@
+// export
+if (module) module.exports = Neuron;
+
 /******************************************************************************************
                                          NEURON
 *******************************************************************************************/
@@ -76,7 +79,7 @@ Neuron.prototype = {
       }
       influences[neuron.ID] = influence;
     }
-    
+
     for (var i in this.connections.inputs) {
       var input = this.connections.inputs[i];
 
@@ -117,7 +120,7 @@ Neuron.prototype = {
     // output neurons get their error from the enviroment
     if (isOutput)
       this.error.responsibility = this.error.projected = target - this.activation; // Eq. 10
-    
+
     else // the rest of the neuron compute their error responsibilities by backpropagation
     {
       // error responsibilities from all the connections projected from this neuron
@@ -299,7 +302,7 @@ Neuron.prototype = {
 
   // hardcodes the behaviour of the neuron into an optimized function
   optimize: function(optimized, layer) {
-    
+
     optimized = optimized || {};
     var that = this;
     var store_activation = [];
@@ -460,19 +463,20 @@ Neuron.prototype = {
           buildSentence(derivative, ' = 1', store_activation);
           break;
         case Neuron.squash.HLIM:
-          buildSentence(activation, ' = +(', state, ' > 0)',
-            store_activation);
+          buildSentence(activation, ' = +(', state, ' > 0)', store_activation);
           buildSentence(derivative, ' = 1', store_activation);
+        case Neuron.squash.RELU:
+          buildSentence(activation, ' = ', state, ' > 0 ? ', state, ' : 0', store_activation);
+          buildSentence(derivative, ' = ', state, ' > 0 ? 1 : 0', store_activation);
           break;
       }
 
-      influences = [];
       for (var id in this.trace.extended) {
         // calculate extended elegibility traces in advance
-        
+
         var xtrace = this.trace.extended[id];
         var neuron = this.neighboors[id];
-        var influence = getVar('aux');
+        var influence = getVar('influences[' + neuron.ID + ']');
         var neuron_old = getVar(neuron, 'old');
         var initialized = false;
         if (neuron.selfconnection.gater == this)
@@ -487,19 +491,14 @@ Neuron.prototype = {
             [incoming].from, 'activation');
 
           if (initialized)
-            buildSentence(influence, ' += ', incoming_weight, ' * ',
-              incoming_activation, store_trace);
+            buildSentence(influence, ' += ', incoming_weight, ' * ', incoming_activation, store_trace);
           else {
-            buildSentence(influence, ' = ', incoming_weight, ' * ',
-              incoming_activation, store_trace);
+            buildSentence(influence, ' = ', incoming_weight, ' * ', incoming_activation, store_trace);
             initialized = true;
           }
         }
-        
-        influences.push(neuron.ID);
-        buildSentence("influences[" + (influences.length - 1) + "] = ", influence, store_trace);
       }
-      
+
       for (var i in this.connections.inputs) {
         var input = this.connections.inputs[i];
         if (input.gater)
@@ -535,7 +534,7 @@ Neuron.prototype = {
           // extended elegibility trace
           var xtrace = this.trace.extended[id];
           var neuron = this.neighboors[id];
-          var influence = getVar('aux');
+          var influence = getVar('influences[' + neuron.ID + ']');
           var neuron_old = getVar(neuron, 'old');
 
           var trace = getVar(this, 'trace', 'elegibility', input.ID, this.trace
@@ -550,14 +549,14 @@ Neuron.prototype = {
             if (neuron.selfconnection.gater)
               buildSentence(xtrace, ' = ', neuron_self_gain, ' * ',
                 neuron_self_weight, ' * ', xtrace, ' + ', derivative, ' * ',
-                trace, ' * ', "influences[" + influences.indexOf(neuron.ID) + "]", store_trace);
+                trace, ' * ', influence, store_trace);
             else
               buildSentence(xtrace, ' = ', neuron_self_weight, ' * ',
                 xtrace, ' + ', derivative, ' * ', trace, ' * ',
-                "influences[" + influences.indexOf(neuron.ID) + "]", store_trace);
+                influence, store_trace);
           else
             buildSentence(xtrace, ' = ', derivative, ' * ', trace, ' * ',
-              "influences[" + influences.indexOf(neuron.ID) + "]", store_trace);
+              influence, store_trace);
         }
       }
       for (var connection in this.connections.gated) {
@@ -742,7 +741,7 @@ Neuron.prototype = {
 Neuron.connection = function Connection(from, to, weight) {
 
   if (!from || !to)
-    throw "Connection Error: Invalid neurons";
+    throw new Error("Connection Error: Invalid neurons");
 
   this.ID = Neuron.connection.uid();
   this.from = from;
@@ -775,7 +774,12 @@ Neuron.squash.IDENTITY = function(x, derivate) {
   return derivate ? 1 : x;
 };
 Neuron.squash.HLIM = function(x, derivate) {
-  return derivate ? 1 : +(x > 0);
+  return derivate ? 1 : x > 0 ? 1 : 0;
+};
+Neuron.squash.RELU = function(x, derivate) {
+  if (derivate)
+    return x > 0 ? 1 : 0;
+  return x > 0 ? x : 0;
 };
 
 // unique ID's
@@ -795,7 +799,3 @@ Neuron.squash.HLIM = function(x, derivate) {
     }
   }
 })();
-
-// export
-if (module) module.exports = Neuron;
-

src/trainer.js

@@ -1,3 +1,6 @@
+// export
+if (module) module.exports = Trainer;
+
 /*******************************************************************************************
                                         TRAINER
 *******************************************************************************************/
@@ -8,7 +11,8 @@ function Trainer(network, options) {
   this.rate = options.rate || .2;
   this.iterations = options.iterations || 100000;
   this.error = options.error || .005
-  this.cost = options.cost || Trainer.cost.CROSS_ENTROPY;
+  this.cost = options.cost || null;
+  this.crossValidate = options.crossValidate || null;
 }
 
 Trainer.prototype = {
@@ -18,8 +22,10 @@ Trainer.prototype = {
 
     var error = 1;
     var iterations = bucketSize = 0;
-    var abort_training = false;
+    var abort = false;
     var input, output, target, currentRate;
+    var cost = options && options.cost || this.cost || Trainer.cost.MSE;
+    var crossValidate = false, testSet, trainSet;
 
     var start = Date.now();
 
@@ -47,6 +53,13 @@ Trainer.prototype = {
         console.log('Deprecated: use schedule instead of customLog')
         this.schedule = options.customLog;
       }
+      if (this.crossValidate) {
+        crossValidate = true;
+        if (options.crossValidate.testSize)
+          this.crossValidate.testSize = options.crossValidate.testSize;
+        if (options.crossValidate.testError)
+          this.crossValidate.testError = options.crossValidate.testError;
+      }
     }
 
     currentRate = this.rate;
@@ -54,33 +67,42 @@ Trainer.prototype = {
       bucketSize = Math.floor(this.iterations / this.rate.length);
     }
 
+    if(crossValidate) {
+      var numTrain = Math.ceil((1 - this.crossValidate.testSize) * set.length);
+      trainSet = set.slice(0, numTrain);
+      testSet = set.slice(numTrain);
+    }
 
-    while (!abort_training && iterations < this.iterations && error > this.error) {
+    while ((!abort && iterations < this.iterations && error > this.error)) {
+      if (crossValidate && error <= this.crossValidate.testError) {
+        break;
+      }
+
+      var currentSetSize = set.length;
       error = 0;
 
       if(bucketSize > 0) {
         var currentBucket = Math.floor(iterations / bucketSize);
-        currentRate = this.rate[currentBucket];
+        currentRate = this.rate[currentBucket] || currentRate;
       }
 
-      for (var train in set) {
-        input = set[train].input;
-        target = set[train].output;
-
-        output = this.network.activate(input);
-        this.network.propagate(currentRate, target);
-
-        error += this.cost(target, output);
+      if (crossValidate) {
+        this._trainSet(trainSet, currentRate, cost);
+        error += this.test(testSet).error;
+        currentSetSize = 1;
+      } else {
+        error += this._trainSet(set, currentRate, cost);
+        currentSetSize = set.length;
       }
 
       // check error
       iterations++;
-      error /= set.length;
+      error /= currentSetSize;
 
       if (options) {
         if (this.schedule && this.schedule.every && iterations %
           this.schedule.every == 0)
-          abort_training = this.schedule.do({
+          abort = this.schedule.do({
             error: error,
             iterations: iterations,
             rate: currentRate
@@ -102,6 +124,48 @@ Trainer.prototype = {
     return results;
   },
 
+  // preforms one training epoch and returns the error (private function used in this.train)
+  _trainSet: function(set, currentRate, costFunction) {
+    var errorSum = 0;
+    for (var train in set) {
+      input = set[train].input;
+      target = set[train].output;
+
+      output = this.network.activate(input);
+      this.network.propagate(currentRate, target);
+
+      errorSum += costFunction(target, output);
+    }
+    return errorSum;
+  },
+
+  // tests a set and returns the error and elapsed time
+  test: function(set, options) {
+
+    var error = 0;
+    var abort = false;
+    var input, output, target;
+    var cost = options && options.cost || this.cost || Trainer.cost.MSE;
+
+    var start = Date.now();
+
+    for (var test in set) {
+      input = set[test].input;
+      target = set[test].output;
+      output = this.network.activate(input);
+      error += cost(target, output);
+    }
+
+    error /= set.length;
+
+    var results = {
+      error: error,
+      time: Date.now() - start
+    }
+
+    return results;
+  },
+
   // trains any given set to a network using a WebWorker
   workerTrain: function(set, callback, options) {
 
@@ -110,7 +174,8 @@ Trainer.prototype = {
     var iterations = bucketSize = 0;
     var input, output, target, currentRate;
     var length = set.length;
-    var abort_training = false;
+    var abort = false;
+    var cost = options && options.cost || that.cost || Trainer.cost.MSE;
 
     var start = Date.now();
 
@@ -125,34 +190,36 @@ Trainer.prototype = {
         };
       }
       if (options.iterations)
-        this.iterations = options.iterations;
+        that.iterations = options.iterations;
       if (options.error)
-        this.error = options.error;
+        that.error = options.error;
       if (options.rate)
-        this.rate = options.rate;
+        that.rate = options.rate;
       if (options.cost)
-        this.cost = options.cost;
+        that.cost = options.cost;
       if (options.schedule)
-        this.schedule = options.schedule;
+        that.schedule = options.schedule;
       if (options.customLog)
+      {
         // for backward compatibility with code that used customLog
         console.log('Deprecated: use schedule instead of customLog')
-        this.schedule = options.customLog;
+        that.schedule = options.customLog;
+      }
     }
 
     // dynamic learning rate
-    currentRate = this.rate;
-    if(Array.isArray(this.rate)) {
-      bucketSize = Math.floor(this.iterations / this.rate.length);
+    currentRate = that.rate;
+    if(Array.isArray(that.rate)) {
+      bucketSize = Math.floor(that.iterations / that.rate.length);
     }
 
     // create a worker
-    var worker = this.network.worker();
+    var worker = that.network.worker();
 
     // activate the network
     function activateWorker(input)
     {
-        worker.postMessage({ 
+        worker.postMessage({
             action: "activate",
             input: input,
             memoryBuffer: that.network.optimized.memory
@@ -163,9 +230,9 @@ Trainer.prototype = {
     function propagateWorker(target){
         if(bucketSize > 0) {
           var currentBucket = Math.floor(iterations / bucketSize);
-          currentRate = this.rate[currentBucket];
+          currentRate = that.rate[currentBucket] || currentRate;
         }
-        worker.postMessage({ 
+        worker.postMessage({
             action: "propagate",
             target: target,
             rate: currentRate,
@@ -188,10 +255,11 @@ Trainer.prototype = {
 
                 // log
                 if (options) {
-                  if (this.schedule && this.schedule.every && iterations % this.schedule.every == 0)
-                    abort_training = this.schedule.do({
+                  if (that.schedule && that.schedule.every && iterations % that.schedule.every == 0)
+                    abort = that.schedule.do({
                       error: error,
-                      iterations: iterations
+                      iterations: iterations,
+                      rate: currentRate
                     });
                   else if (options.log && iterations % options.log == 0) {
                     console.log('iterations', iterations, 'error', error);
@@ -200,7 +268,7 @@ Trainer.prototype = {
                     shuffle(set);
                 }
 
-                if (!abort_training && iterations < that.iterations && error > that.error)
+                if (!abort && iterations < that.iterations && error > that.error)
                 {
                     activateWorker(set[index].input);
                 } else {
@@ -219,8 +287,8 @@ Trainer.prototype = {
 
         if (e.data.action == "activate")
         {
-            error += that.cost(set[index].output, e.data.output);
-            propagateWorker(set[index].output); 
+            error += cost(set[index].output, e.data.output);
+            propagateWorker(set[index].output);
             index++;
         }
     }
@@ -235,7 +303,7 @@ Trainer.prototype = {
   XOR: function(options) {
 
     if (this.network.inputs() != 2 || this.network.outputs() != 1)
-      throw "Error: Incompatible network (2 inputs, 1 output)";
+      throw new Error("Incompatible network (2 inputs, 1 output)");
 
     var defaults = {
       iterations: 100000,
@@ -276,6 +344,7 @@ Trainer.prototype = {
     var rate = options.rate || .1;
     var log = options.log || 0;
     var schedule = options.schedule || {};
+    var cost = options.cost || this.cost || Trainer.cost.CROSS_ENTROPY;
 
     var trial = correct = i = j = success = 0,
       error = 1,
@@ -351,10 +420,7 @@ Trainer.prototype = {
           this.network.propagate(rate, output);
         }
 
-        var delta = 0;
-        for (var j in prediction)
-          delta += Math.pow(output[j] - prediction[j], 2);
-        error += delta / this.network.outputs();
+        error += cost(output, prediction);
 
         if (distractorsCorrect + targetsCorrect == length)
           correct++;
@@ -399,6 +465,7 @@ Trainer.prototype = {
     var criterion = options.error || .05;
     var rate = options.rate || .1;
     var log = options.log || 500;
+    var cost = options.cost || this.cost || Trainer.cost.CROSS_ENTROPY;
 
     // gramar node
     var Node = function() {
@@ -557,12 +624,7 @@ Trainer.prototype = {
         read = sequence.charAt(++i);
         predict = sequence.charAt(i + 1);
 
-        var delta = 0;
-        for (var k in output)
-          delta += Math.pow(target[k] - output[k], 2)
-        delta /= output.length;
-
-        error += delta;
+        error += cost(target, output);
       }
       error /= sequence.length;
       iteration++;
@@ -579,6 +641,78 @@ Trainer.prototype = {
       test: test,
       generate: generate
     }
+  },
+
+  timingTask: function(options){
+
+    if (this.network.inputs() != 2 || this.network.outputs() != 1)
+      throw new Error("Invalid Network: must have 2 inputs and one output");
+
+    if (typeof options == 'undefined')
+      var options = {};
+
+    // helper
+    function getSamples (trainingSize, testSize){
+
+      // sample size
+      var size = trainingSize + testSize;
+
+      // generate samples
+      var t = 0;
+      var set  = [];
+      for (var i = 0; i < size; i++) {
+        set.push({ input: [0,0], output: [0] });
+      }
+      while(t < size - 20) {
+          var n = Math.round(Math.random() * 20);
+          set[t].input[0] = 1;
+          for (var j = t; j <= t + n; j++){
+              set[j].input[1] = n / 20;
+              set[j].output[0] = 0.5;
+          }
+          t += n;
+          n = Math.round(Math.random() * 20);
+          for (var k = t+1; k <= (t + n) &&  k < size; k++)
+              set[k].input[1] = set[t].input[1];
+          t += n;
+      }
+
+      // separate samples between train and test sets
+      var trainingSet = []; var testSet = [];
+      for (var l = 0; l < size; l++)
+          (l < trainingSize ? trainingSet : testSet).push(set[l]);
+
+      // return samples
+      return {
+          train: trainingSet,
+          test: testSet
+      }
+    }
+
+    var iterations = options.iterations || 200;
+    var error = options.error || .005;
+    var rate = options.rate || [.03, .02];
+    var log = options.log === false ? false : options.log || 10;
+    var cost = options.cost || this.cost || Trainer.cost.MSE;
+    var trainingSamples = options.trainSamples || 7000;
+    var testSamples = options.trainSamples || 1000;
+
+    // samples for training and testing
+    var samples = getSamples(trainingSamples, testSamples);
+
+    // train
+    var result = this.train(samples.train, {
+      rate: rate,
+      log: log,
+      iterations: iterations,
+      error: error,
+      cost: cost
+    });
+
+    return {
+      train: result,
+      test: this.test(samples.test)
+    }
   }
 };
 
@@ -598,9 +732,11 @@ Trainer.cost = {
     for (var i in output)
       mse += Math.pow(target[i] - output[i], 2);
     return mse / output.length;
+  },
+  BINARY: function(target, output){
+    var misses = 0;
+    for (var i in output)
+      misses += Math.round(target[i] * 2) != Math.round(output[i] * 2);
+    return misses;
   }
 }
-
-// export
-if (module) module.exports = Trainer;
-

test/synaptic.js

@@ -12,29 +12,43 @@ var Perceptron = synaptic.Architect.Perceptron,
 
 // utils
 
-var noRepeat = function(range, avoid) {
+function noRepeat (range, avoid) {
   var number = Math.random() * range | 0;
-  var used = false;
-  for (var i in avoid)
-    if (number == avoid[i])
-      used = true;
-  return used ? noRepeat(range, avoid) : number;
-};
+  for (var i in avoid){
+    if (number == avoid[i]){
+      return noRepeat(range,avoid);
+    }
+  }
+  return number;
+}
 
-var equal = function(prediction, output) {
+function equal (prediction, output) {
   for (var i in prediction)
     if (Math.round(prediction[i]) != output[i])
       return false;
   return true;
-};
+}
 
-var generateRandomArray = function(size){
+function generateRandomArray (size){
     var array = [];
     for (var j = 0; j < size; j++)
         array.push(Math.random() + .5 | 0);
     return array;
 }
 
+function compare (a, b) {
+  var mse = 0;
+  for (var k in a)
+    mse += Math.pow(a[k] - b[k], 2);
+  mse /= a.length;
+
+  return mse < 1e-10;
+}
+
+function equalWithError (output, expected, error) {
+  return Math.abs(output - expected) <= error;
+}
+
 // specs
 
 describe('Basic Neural Network', function() {
@@ -78,7 +92,7 @@ describe('Basic Neural Network', function() {
     var test01 = Math.round(network.activate([0, 1]));
     assert.equal(test01, 0, "[0,1] did not output 0");
 
-    var test10 = Math.round(network.activate([0, 1]));
+    var test10 = Math.round(network.activate([1, 0]));
     assert.equal(test10, 0, "[1,0] did not output 0");
 
     var test11 = Math.round(network.activate([1, 1]));
@@ -124,7 +138,7 @@ describe('Basic Neural Network', function() {
     var test01 = Math.round(network.activate([0, 1]));
     assert.equal(test01, 1, "[0,1] did not output 1");
 
-    var test10 = Math.round(network.activate([0, 1]));
+    var test10 = Math.round(network.activate([1, 0]));
     assert.equal(test10, 1, "[1,0] did not output 1");
 
     var test11 = Math.round(network.activate([1, 1]));
@@ -196,6 +210,111 @@ describe("Perceptron - XOR", function() {
   });
 });
 
+describe("Perceptron - SIN", function() {
+  var mySin = function(x) {
+    return (Math.sin(x)+1)/2;
+  };
+
+  var sinNetwork = new Perceptron(1, 12, 1);
+
+  var trainingSet = Array.apply(null, Array(800)).map(function () {
+    var inputValue = Math.random() * Math.PI * 2;
+    return {
+      input: [inputValue],
+      output: [mySin(inputValue)]
+    };
+  });
+
+  var results = sinNetwork.trainer.train(trainingSet, {
+    iterations: 2000,
+    log: false,
+    error: 1e-6,
+    cost: Trainer.cost.MSE,
+  });
+
+  var test0 = sinNetwork.activate([0])[0];
+  var expected0 = mySin(0);
+  it("input: [0] output: " + test0 + ", expected: " + expected0, function() {
+    var eq = equalWithError(test0, expected0, .035);
+    assert.equal(eq, true, "[0] did not output " + expected0);
+  });
+
+  var test05PI = sinNetwork.activate([.5*Math.PI])[0];
+  var expected05PI = mySin(.5*Math.PI);
+  it("input: [0.5*Math.PI] output: " + test05PI + ", expected: " + expected05PI, function() {
+    var eq = equalWithError(test05PI, expected05PI, .035);
+    assert.equal(eq, true, "[0.5*Math.PI] did not output " + expected05PI);
+  });
+
+  var test2 = sinNetwork.activate([2])[0];
+  var expected2 = mySin(2);
+  it("input: [2] output: " + test2 + ", expected: " + expected2, function() {
+    var eq = equalWithError(test2, expected2, .035);
+    assert.equal(eq, true, "[2] did not output " + expected2);
+  });
+
+  var errorResult = results.error;
+  it("Sin error: " + errorResult, function() {
+    var lessThanOrEqualError = errorResult <= .001;
+    assert.equal(lessThanOrEqualError, true, "Sin error not less than or equal to desired error.");
+  });
+});
+
+describe("Perceptron - SIN - CrossValidate", function() {
+
+  var mySin = function(x) {
+    return (Math.sin(x)+1)/2;
+  };
+
+  var sinNetwork = new Perceptron(1, 12, 1);
+
+  var trainingSet = Array.apply(null, Array(800)).map(function () {
+    var inputValue = Math.random() * Math.PI * 2;
+    return {
+      input: [inputValue],
+      output: [mySin(inputValue)]
+    };
+  });
+
+  var results = sinNetwork.trainer.train(trainingSet, {
+    iterations: 2000,
+    log: false,
+    error: 1e-6,
+    cost: Trainer.cost.MSE,
+    crossValidate: {
+      testSize: .3,
+      testError: 1e-6
+    }
+  });
+
+  var test0 = sinNetwork.activate([0])[0];
+  var expected0 = mySin(0);
+  it("input: [0] output: " + test0 + ", expected: " + expected0, function() {
+    var eq = equalWithError(test0, expected0, .035);
+    assert.equal(eq, true, "[0] did not output " + expected0);
+  });
+
+  var test05PI = sinNetwork.activate([.5*Math.PI])[0];
+  var expected05PI = mySin(.5*Math.PI);
+  it("input: [0.5*Math.PI] output: " + test05PI + ", expected: " + expected05PI, function() {
+    var eq = equalWithError(test05PI, expected05PI, .035);
+    assert.equal(eq, true, "[0.5*Math.PI] did not output " + expected05PI);
+  });
+
+  var test2 = sinNetwork.activate([2])[0];
+  var expected2 = mySin(2);
+  it("input: [2] output: " + test2 + ", expected: " + expected2, function() {
+    var eq = equalWithError(test2, expected2, .035);
+    assert.equal(eq, true, "[2] did not output " + expected2);
+  });
+
+  var errorResult = results.error;
+  it("CrossValidation error: " + errorResult, function() {
+    var lessThanOrEqualError = errorResult <= .001;
+    assert.equal(lessThanOrEqualError, true, "CrossValidation error not less than or equal to desired error.");
+  });
+});
+
 describe("LSTM - Discrete Sequence Recall", function() {
 
   var targets = [2, 4];
@@ -293,8 +412,25 @@ describe("LSTM - Discrete Sequence Recall", function() {
   }
 });
 
+describe("LSTM - Timing Task", function() {
+  var network = new synaptic.Architect.LSTM(2,7,1);
+  var result = network.trainer.timingTask({
+    log: false,
+    trainSamples: 4000,
+    testSamples: 500
+  });
+
+  it("should complete the training in less than 200 iterations", function() {
+    assert(result.train.iterations <= 200);
+  });
+
+  it("should pass the test with an error smaller than 0.05", function() {
+    assert(result.test.error < .05);
+  });
+});
+
 describe("Optimized and Unoptimized Networks Equivalency", function() {
-  var optimized = new Perceptron(10,15,5);
+  var optimized = new LSTM(2,1,1)
 
   var unoptimized = optimized.clone();
   unoptimized.setOptimize(false);
@@ -305,23 +441,19 @@ describe("Optimized and Unoptimized Networks Equivalency", function() {
   for (var i = 1; i <= iterations; i++)
   {
       //random input
-      var input = generateRandomArray(10);
+      var input = generateRandomArray(2);
 
       // activate networks
       var output1 = optimized.activate(input);
       var output2 = unoptimized.activate(input);
 
       if (i % 100 == 0)
-        it(' same output for both networks after ' + i + ' iterations', function(){
-          var diff = false;
-          for (var k in output1)
-            if (output1[k] - output2[k] != 0)
-              diff = true;
-          assert(!diff);
+        it('should produce the same output for both networks after ' + i + ' iterations', function(){
+          assert(compare(output1, output2));
         });
 
       // random target
-      var target = generateRandomArray(5);
+      var target = generateRandomArray(1);
 
       // propagate networks
       optimized.propagate(learningRate, target);
@@ -330,7 +462,7 @@ describe("Optimized and Unoptimized Networks Equivalency", function() {
 });
 
 describe("toJSON/fromJSON Networks Equivalency", function() {
-  var original = new Perceptron(10,15,5);
+  var original = new LSTM(10,5,5);
 
   var exported = original.toJSON();
   var imported = Network.fromJSON(exported);
@@ -348,12 +480,8 @@ describe("toJSON/fromJSON Networks Equivalency", function() {
       var output2 = imported.activate(input);
 
       if (i % 100 == 0)
-        it(' same output for both networks after ' + i + ' iterations', function(){
-          var diff = false;
-          for (var k in output1)
-            if (output1[k] - output2[k] != 0)
-              diff = true;
-          assert(!diff);
+        it('should produce the same output for both networks after ' + i + ' iterations', function(){
+          assert(compare(output1, output2));
         });
 
       // random target
@@ -366,7 +494,9 @@ describe("toJSON/fromJSON Networks Equivalency", function() {
 });
 
 describe("Cloned Networks Equivalency", function() {
-  var original = new Perceptron(10,15,5);
+
+  var original = new LSTM(10,5,5);
+
   var cloned = original.clone();
 
   var learningRate = .5;
@@ -382,12 +512,8 @@ describe("Cloned Networks Equivalency", function() {
       var output2 = cloned.activate(input);
 
       if (i % 100 == 0)
-        it(' same output for both networks after ' + i + ' iterations', function(){
-          var diff = false;
-          for (var k in output1)
-            if (output1[k] - output2[k] != 0)
-              diff = true;
-          assert(!diff);
+        it('should produce the same output for both networks after ' + i + ' iterations', function(){
+          assert(compare(output1, output2));
         });
 
       // random target
@@ -399,10 +525,10 @@ describe("Cloned Networks Equivalency", function() {
   }
 });
 
-describe("Manual Override", function() {
+describe("Scheduled Tasks", function() {
   var perceptron = new Perceptron(2, 3, 1);
 
-  it('iterations ended at full 3000', function(){
+  it('should stop training at 3000 iterations', function(){
     var final_stats = perceptron.trainer.XOR({
       iterations: 3000,
       rate: 0.000001,
@@ -419,7 +545,7 @@ describe("Manual Override", function() {
     assert.equal( final_stats.iterations, 3000 )
   });
 
-  it('iterations ended at 2000, not full 3000', function(){
+  it('should abort the training at 2000 iterations', function(){
     var final_stats = perceptron.trainer.XOR({
       iterations: 3000,
       rate: 0.000001,
@@ -436,7 +562,7 @@ describe("Manual Override", function() {
     assert.equal( final_stats.iterations, 2000 )
   });
 
-  it('training works even when schedule() has no return value', function(){
+  it('should work even if shedule.do() returns no value', function(){
     var final_stats = perceptron.trainer.XOR({
       iterations: 3000,
       rate: 0.000001,
@@ -449,18 +575,4 @@ describe("Manual Override", function() {
     assert.equal( final_stats.iterations, 3000 )
   });
 
-  it('using depreciated customLog still works', function(){
-    var counter = 0
-    var final_stats = perceptron.trainer.XOR({
-      iterations: 3000,
-      rate: 0.000001,
-      error: 0.000001,
-      customLog: {
-          every: 1000,
-          do: function(data) { counter++ }
-        }
-    });
-    assert.equal( counter, 3 )
-  });
-
 });

test/test_browser.html

@@ -1,22 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-  <title>test</title>
-  <script src="../dist/synaptic.js"></script>
-</head>
-<body>
-
-</body>
-<script type="text/javascript">
-  var Architect = synaptic.Architect;
-
-  var myPerceptron = new Architect.Perceptron(2,3,1);
-  document.body.innerHTML += JSON.stringify(myPerceptron.trainer.XOR())+'<br>'
-  document.body.innerHTML += JSON.stringify(myPerceptron.trainer.XOR())+'<br>'
-
-  if(document.body.innerHTML.length > 0){
-    document.body.innerHTML += '<br>if you see this, then synaptic works'
-  }
-
-</script>
-</html>