Delete launch.json

Add instructions to accessing examples

.gitignore

@@ -14,3 +14,7 @@ node_modules
 # Demo.
 demo.js
 
+# Degub
+debug.html
+
+.settings

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2014 Juan Cazala (juancazala.com)
+Copyright (c) 2016 Juan Cazala - juancazala.com
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -9,14 +9,38 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 copies of the Software, and to permit persons to whom the Software is
 furnished to do so, subject to the following conditions:
 
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
 
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE
 
+
+
+********************************************************************************************
+                                  SYNAPTIC (v1.0.7)
+********************************************************************************************
+
+Synaptic is a javascript neural network library for node.js and the browser, its generalized
+algorithm is architecture-free, so you can build and train basically any type of first order
+or even second order neural network architectures.
+
+http://en.wikipedia.org/wiki/Recurrent_neural_network#Second_Order_Recurrent_Neural_Network
+
+The library includes a few built-in architectures like multilayer perceptrons, multilayer
+long-short term memory networks (LSTM) or liquid state machines, and a trainer capable of
+training any given network, and includes built-in training tasks/tests like solving an XOR,
+passing a Distracted Sequence Recall test or an Embeded Reber Grammar test.
+
+The algorithm implemented by this library has been taken from Derek D. Monner's paper:
+
+
+A generalized LSTM-like training algorithm for second-order recurrent neural networks
+http://www.overcomplete.net/papers/nn2012.pdf
+
+There are references to the equations in that paper commented through the source code.

README.md

@@ -17,6 +17,11 @@ There are references to the equations in that paper commented through the source
 
 If you have no prior knowledge about Neural Networks, you should start by [reading this guide](https://github.com/cazala/synaptic/wiki/Neural-Networks-101).
 
+
+If you want a practical example on how to feed data to a neural network, then take a look at [this article](https://github.com/cazala/synaptic/wiki/Normalization-101).
+
+You may also want to take a look at [this article](http://blog.webkid.io/neural-networks-in-javascript/).
+
 ####Demos
 
 - [Solve an XOR](http://synaptic.juancazala.com/#/xor)
@@ -36,12 +41,17 @@ The source code of these demos can be found in [this branch](https://github.com/
 - [Trainer](https://github.com/cazala/synaptic/wiki/Trainer/)
 - [Architect](https://github.com/cazala/synaptic/wiki/Architect/)
 
+To try out the examples, checkout the [gh-pages](https://github.com/cazala/synaptic/tree/gh-pages) branch.
+
+`git checkout gh-pages`
+
 
 ##Overview
 
 ###Installation
 
 #####In node
+
 You can install synaptic with [npm](http://npmjs.org):
 
 ```cmd
@@ -49,10 +59,17 @@ npm install synaptic --save
 ```
 
 #####In the browser
-Just include the file synaptic.js from `/dist` directory with a script tag in your HTML:
+
+You can install synaptic with [bower](http://bower.io):
+
+```cmd
+bower install synaptic
+```
+
+Or you can simply use the CDN link, kindly provided by [CDNjs](https://cdnjs.com/)
 
 ```html
-<script src="synaptic.js"></script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/synaptic/1.0.4/synaptic.min.js"></script>
 ```
 
 ###Usage

bower.json

@@ -0,0 +1,33 @@
+{
+  "name": "synaptic",
+  "version": "1.0.7",
+  "homepage": "https://github.com/cazala/synaptic",
+  "authors": [
+    "Juan Cazala <juancazala@gmail.com>"
+  ],
+  "description": "architecture-free neural network library for node.js and the browser",
+  "main": "./dist/synaptic.min.js",
+  "moduleType": [
+    "amd",
+    "globals",
+    "node"
+  ],
+  "keywords": [
+    "neural",
+    "network",
+    "deep",
+    "learning",
+    "machine",
+    "learning",
+    "lstm",
+    "perceptron"
+  ],
+  "license": "MIT",
+  "ignore": [
+    "**/.*",
+    "node_modules",
+    "bower_components",
+    "test",
+    "tests"
+  ]
+}

gulpfile.js

@@ -1,61 +0,0 @@
-'use strict';
-
-var license = '/*\n\nThe MIT License (MIT)\n\nCopyright (c) 2014 Juan Cazala - juancazala.com\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the "Software"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in\nall copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE\n\n\n\n********************************************************************************************\n                                         SYNAPTIC\n********************************************************************************************\n\nSynaptic is a javascript neural network library for node.js and the browser, its generalized\nalgorithm is architecture-free, so you can build and train basically any type of first order\nor even second order neural network architectures.\n\nhttp://en.wikipedia.org/wiki/Recurrent_neural_network#Second_Order_Recurrent_Neural_Network\n\nThe library includes a few built-in architectures like multilayer perceptrons, multilayer\nlong-short term memory networks (LSTM) or liquid state machines, and a trainer capable of\ntraining any given network, and includes built-in training tasks/tests like solving an XOR,\npassing a Distracted Sequence Recall test or an Embeded Reber Grammar test.\n\nThe algorithm implemented by this library has been taken from Derek D. Monner\'s paper:\n\n\nA generalized LSTM-like training algorithm for second-order recurrent neural networks\nhttp://www.overcomplete.net/papers/nn2012.pdf\n\nThere are references to the equations in that paper commented through the source code.\n\n\n********************************************************************************************/\n'
-var globals = 'var Neuron = synaptic.Neuron, Layer = synaptic.Layer, Network = synaptic.Network, Trainer = synaptic.Trainer, Architect = synaptic.Architect;';
-
-// import
-var gulp = require('gulp');
-var browserify = require('browserify');
-var uglify = require('gulp-uglify');
-var mocha = require('gulp-mocha');
-var prepend = require('gulp-insert').prepend;
-var append = require('gulp-insert').append;
-var source = require('vinyl-source-stream');
-var buffer = require('vinyl-buffer');
-
-// default task: runs all the tests, and builds all the files into dist (minified and unminifed)
-gulp.task('default', ['test', 'build', 'min']);
-
-// build source into /dist for the web
-gulp.task('build', function () {
-  return browserify({ entries: ['./src/synaptic.js'] })
-    .bundle()
-    .pipe(source('synaptic.js'))
-    .pipe(buffer())
-    .pipe(append(globals))
-    .pipe(gulp.dest('./dist'));
-});
-
-// build source into /dist for web (minified)
-gulp.task('min', function () {
-  return browserify({ entries: ['./src/synaptic.js'] })
-    .bundle()
-    .pipe(source('synaptic.min.js'))
-    .pipe(buffer())
-    .pipe(uglify())
-    .pipe(prepend(license))
-    .pipe(append(globals))
-    .pipe(gulp.dest('./dist'));
-});
-
-// build source into /dist with sourcemaps for debugging
-gulp.task('debug', function () {
-  return browserify({ entries: ['./src/synaptic.js'], debug: true })
-    .bundle()
-    .pipe(source('synaptic.js'))
-    .pipe(buffer())
-    .pipe(prepend(license))
-    .pipe(append(globals))
-    .pipe(gulp.dest('./dist'));
-});
-
-// run all the tests with mocha
-gulp.task('test', function () {
-    return gulp.src('test/synaptic.js', {read: false})
-        .pipe(mocha());
-});
-
-// watch for changed and re-build (debug)
-gulp.task('dev', function () {
-   gulp.watch('./src/*.js', ['debug']);
-});

index.html

@@ -0,0 +1 @@
+<script src='dist/bundle.js'></script>

license.js

@@ -0,0 +1,10 @@
+// update license year and version
+var fs = require('fs')
+module.exports = function() {
+  var version = require('./package.json').version
+  var license = fs.readFileSync('LICENSE', 'utf-8')
+  .replace(/\(c\) ([0-9]+)/, '(c) ' + (new Date).getFullYear())
+  .replace(/SYNAPTIC \(v(.*)\)/, 'SYNAPTIC (v' + version + ')')
+  fs.writeFileSync('LICENSE', license)
+  return license
+}

package.json

@@ -1,22 +1,19 @@
 {
   "name": "synaptic",
-  "version": "0.1.7",
-  "description": "Architecture-free neural network library",
+  "version": "1.0.7",
+  "description": "architecture-free neural network library",
   "main": "./src/synaptic",
   "scripts": {
-    "test": "mocha test"
+    "test": "mocha test",
+    "build": "webpack --config webpack.config.js"
   },
+  "precommit": [
+    "test",
+    "build"
+  ],
   "devDependencies": {
-    "browserify": "^10.1.3",
-    "gulp": "^3.8.11",
-    "gulp-insert": "^0.4.0",
-    "gulp-mocha": "^2.0.1",
-    "gulp-sourcemaps": "^1.5.2",
-    "gulp-uglify": "^1.2.0",
-    "gulp-util": "^3.0.4",
-    "vinyl-buffer": "^1.0.0",
-    "vinyl-source-stream": "^1.1.0",
-    "mocha": "^2.2.4"
+    "mocha": "^2.2.4",
+    "webpack": "^1.13.1"
   },
   "repository": {
     "type": "git",
@@ -38,5 +35,7 @@
     "url": "https://github.com/cazala/synaptic/issues"
   },
   "homepage": "http://synaptic.juancazala.com",
-  "engines" : { "node" : "^0.10.0" }
+  "engines": {
+    "node": ">=4"
+  }
 }

src/architect.js

@@ -1,13 +1,13 @@
 // import
-var Layer = require('./layer'),
-    Network = require('./network'),
-    Trainer = require('./trainer');
+var Layer   = require('./layer')
+,   Network = require('./network')
+,   Trainer = require('./trainer')
 
 /*******************************************************************************************
                                         ARCHITECT
 *******************************************************************************************/
 
-// Colection of useful built-in architectures
+// Collection of useful built-in architectures
 var Architect = {
 
   // Multilayer Perceptron
@@ -15,7 +15,7 @@ var Architect = {
 
     var args = Array.prototype.slice.call(arguments); // convert arguments to Array
     if (args.length < 3)
-      throw "Error: not enough layers (minimum 3) !!";
+      throw new Error("not enough layers (minimum 3) !!");
 
     var inputs = args.shift(); // first argument
     var outputs = args.pop(); // last argument
@@ -53,28 +53,28 @@ var Architect = {
 
     var args = Array.prototype.slice.call(arguments); // convert arguments to array
     if (args.length < 3)
-      throw "Error: not enough layers (minimum 3) !!";
+      throw new Error("not enough layers (minimum 3) !!");
 
     var last = args.pop();
     var option = {
       peepholes: Layer.connectionType.ALL_TO_ALL,
-      hiddentohidden: false,
-      outtohidden: false,
-      outtogates: false,
-      intoout: true,
+      hiddenToHidden: false,
+      outputToHidden: false,
+      outputToGates: false,
+      inputToOutput: true,
     };
     if (typeof last != 'number') {
       var outputs = args.pop();
       if (last.hasOwnProperty('peepholes'))
         option.peepholes = last.peepholes;
-      if (last.hasOwnProperty('hiddentohidden'))
-        option.hiddentohidden = last.hiddentohidden;
-      if (last.hasOwnProperty('outtohidden'))
-        option.outtohidden = last.outtohidden;
-      if (last.hasOwnProperty('outtogates'))
-        option.outtogates = last.outtogates;
-      if (last.hasOwnProperty('intoout'))
-        option.intoout = last.intoout;
+      if (last.hasOwnProperty('hiddenToHidden'))
+        option.hiddenToHidden = last.hiddenToHidden;
+      if (last.hasOwnProperty('outputToHidden'))
+        option.outputToHidden = last.outputToHidden;
+      if (last.hasOwnProperty('outputToGates'))
+        option.outputToGates = last.outputToGates;
+      if (last.hasOwnProperty('inputToOutput'))
+        option.inputToOutput = last.inputToOutput;
     } else
       var outputs = last;
 
@@ -129,20 +129,20 @@ var Architect = {
       var self = memoryCell.project(memoryCell);
 
       // hidden to hidden recurrent connection
-      if (option.hiddentohidden)
+      if (option.hiddenToHidden)
         memoryCell.project(memoryCell, Layer.connectionType.ALL_TO_ELSE);
 
       // out to hidden recurrent connection
-      if (option.outtohidden)
+      if (option.outputToHidden)
         outputLayer.project(memoryCell);
 
       // out to gates recurrent connection
-      if (option.outtogates) {
+      if (option.outputToGates) {
         outputLayer.project(inputGate);
         outputLayer.project(outputGate);
         outputLayer.project(forgetGate);
       }
-      
+
       // peepholes
       memoryCell.project(inputGate, option.peepholes);
       memoryCell.project(forgetGate, option.peepholes);
@@ -159,7 +159,7 @@ var Architect = {
     }
 
     // input to output direct connection
-    if (option.intoout)
+    if (option.inputToOutput)
       inputLayer.project(outputLayer);
 
     // set the layers of the neural network
@@ -270,5 +270,4 @@ for (var architecture in Architect) {
 }
 
 // export
-if (module) module.exports = Architect; 
-
+if (module) module.exports = Architect;

src/layer.js

@@ -1,5 +1,9 @@
+// export
+if (module) module.exports = Layer;
+
 // import
-var Neuron = require('./neuron');
+var Neuron  = require('./neuron')
+,   Network = require('./network')
 
 /*******************************************************************************************
                                             LAYER
@@ -9,7 +13,7 @@ function Layer(size, label) {
   this.size = size | 0;
   this.list = [];
   this.label = label || null;
-  this.connectedto = [];
+  this.connectedTo = [];
 
   while (size--) {
     var neuron = new Neuron();
@@ -26,7 +30,7 @@ Layer.prototype = {
 
     if (typeof input != 'undefined') {
       if (input.length != this.size)
-        throw "INPUT size and LAYER size must be the same to activate!";
+        throw new Error("INPUT size and LAYER size must be the same to activate!");
 
       for (var id in this.list) {
         var neuron = this.list[id];
@@ -48,7 +52,7 @@ Layer.prototype = {
 
     if (typeof target != 'undefined') {
       if (target.length != this.size)
-        throw "TARGET size and LAYER size must be the same to propagate!";
+        throw new Error("TARGET size and LAYER size must be the same to propagate!");
 
       for (var id = this.list.length - 1; id >= 0; id--) {
         var neuron = this.list[id];
@@ -65,14 +69,14 @@ Layer.prototype = {
   // projects a connection from this layer to another one
   project: function(layer, type, weights) {
 
-    if (layer instanceof require('./network'))
+    if (layer instanceof Network)
       layer = layer.layers.input;
 
     if (layer instanceof Layer) {
       if (!this.connected(layer))
         return new Layer.connection(this, layer, type, weights);
     } else
-      throw "Invalid argument, you can only project connections to LAYERS and NETWORKS!";
+      throw new Error("Invalid argument, you can only project connections to LAYERS and NETWORKS!");
 
 
   },
@@ -82,7 +86,7 @@ Layer.prototype = {
 
     if (type == Layer.gateType.INPUT) {
       if (connection.to.size != this.size)
-        throw "GATER layer and CONNECTION.TO layer must be the same size in order to gate!";
+        throw new Error("GATER layer and CONNECTION.TO layer must be the same size in order to gate!");
 
       for (var id in connection.to.list) {
         var neuron = connection.to.list[id];
@@ -95,7 +99,7 @@ Layer.prototype = {
       }
     } else if (type == Layer.gateType.OUTPUT) {
       if (connection.from.size != this.size)
-        throw "GATER layer and CONNECTION.FROM layer must be the same size in order to gate!";
+        throw new Error("GATER layer and CONNECTION.FROM layer must be the same size in order to gate!");
 
       for (var id in connection.from.list) {
         var neuron = connection.from.list[id];
@@ -108,7 +112,7 @@ Layer.prototype = {
       }
     } else if (type == Layer.gateType.ONE_TO_ONE) {
       if (connection.size != this.size)
-        throw "The number of GATER UNITS must be the same as the number of CONNECTIONS to gate!";
+        throw new Error("The number of GATER UNITS must be the same as the number of CONNECTIONS to gate!");
 
       for (var id in connection.list) {
         var gater = this.list[id];
@@ -248,8 +252,8 @@ Layer.connection = function LayerConnection(fromLayer, toLayer, type, weights) {
       this.size = this.list.push(connection);
     }
   }
-  
-  fromLayer.connectedto.push(this);
+
+  fromLayer.connectedTo.push(this);
 }
 
 // types of connections
@@ -270,7 +274,3 @@ Layer.gateType.ONE_TO_ONE = "ONE TO ONE";
     return connections++;
   }
 })();
-
-// export
-if (module) module.exports = Layer;
-

src/network.js

@@ -1,6 +1,10 @@
+// export
+if (module) module.exports = Network;
+
 // import
-var Neuron = require('./neuron'),
-    Layer = require('./layer');
+var Neuron  = require('./neuron')
+,   Layer   = require('./layer')
+,   Trainer = require('./trainer')
 
 /*******************************************************************************************
                                          NETWORK
@@ -27,8 +31,8 @@ Network.prototype = {
       for (var layer in this.layers.hidden)
         this.layers.hidden[layer].activate();
       return this.layers.output.activate();
-    } 
-    else 
+    }
+    else
     {
       if (this.optimized == null)
         this.optimize();
@@ -48,8 +52,8 @@ Network.prototype = {
       reverse.reverse();
       for (var layer in reverse)
         reverse[layer].propagate(rate);
-    } 
-    else 
+    }
+    else
     {
       if (this.optimized == null)
         this.optimize();
@@ -69,7 +73,7 @@ Network.prototype = {
     if (unit instanceof Layer)
       return this.layers.output.project(unit, type, weights);
 
-    throw "Invalid argument, you can only project connections to LAYERS and NETWORKS!";
+    throw new Error("Invalid argument, you can only project connections to LAYERS and NETWORKS!");
   },
 
   // let this network gate a connection
@@ -142,7 +146,6 @@ Network.prototype = {
       hardcode += "F[" + optimized.variables[i].id + "] = " + (optimized.variables[
         i].value || 0) + "; ";
     hardcode += "var activate = function(input){\n";
-    hardcode += "influences = [];";
     for (var i in optimized.inputs)
       hardcode += "F[" + optimized.inputs[i] + "] = input[" + i + "]; ";
     for (var currentLayer in optimized.activation_sentences) {
@@ -356,21 +359,6 @@ Network.prototype = {
       neurons.push(copy);
     }
 
-    if (!ignoreTraces)
-      for (var i in neurons) {
-        var copy = neurons[i];
-
-        for (var input in neuron.trace.elegibility)
-          copy.trace.elegibility[input] = neuron.trace.elegibility[input];
-
-        for (var gated in neuron.trace.extended) {
-          copy.trace.extended[gated] = {};
-          for (var input in neuron.trace.extended[gated])
-            copy.trace.extended[ids[gated]][input] = neuron.trace.extended[
-              gated][input];
-        }
-      }
-
     // get connections
     for (var i in list) {
       var neuron = list[i].neuron;
@@ -391,8 +379,7 @@ Network.prototype = {
           from: ids[neuron.ID],
           to: ids[neuron.ID],
           weight: neuron.selfconnection.weight,
-          gater: neuron.selfconnection.gater ? ids[neuron.selfconnection.gater
-            .ID] : null,
+          gater: neuron.selfconnection.gater ? ids[neuron.selfconnection.gater.ID] : null,
         });
     }
 
@@ -401,37 +388,37 @@ Network.prototype = {
       connections: connections
     }
   },
-  
+
   // export the topology into dot language which can be visualized as graphs using dot
   /* example: ... console.log(net.toDotLang());
               $ node example.js > example.dot
               $ dot example.dot -Tpng > out.png
   */
-  toDot: function(edgeconnection) {
-    if (! typeof edgeconnection)
-      edgeconnection = false;
+  toDot: function(edgeConnection) {
+    if (! typeof edgeConnection)
+      edgeConnection = false;
     var code = "digraph nn {\n    rankdir = BT\n";
     var layers = [this.layers.input].concat(this.layers.hidden, this.layers.output);
     for (var layer in layers) {
-      for (var to in layers[layer].connectedto) { // projections
-        var connection = layers[layer].connectedto[to];
-        var layerto = connection.to;
+      for (var to in layers[layer].connectedTo) { // projections
+        var connection = layers[layer].connectedTo[to];
+        var layerTo = connection.to;
         var size = connection.size;
         var layerID = layers.indexOf(layers[layer]);
-        var layertoID = layers.indexOf(layerto);
+        var layerToID = layers.indexOf(layerTo);
         /* http://stackoverflow.com/questions/26845540/connect-edges-with-graph-dot
          * DOT does not support edge-to-edge connections
          * This workaround produces somewhat weird graphs ...
         */
-        if ( edgeconnection) {
+        if ( edgeConnection) {
           if (connection.gatedfrom.length) {
-            var fakeNode = "fake" + layerID + "_" + layertoID;
+            var fakeNode = "fake" + layerID + "_" + layerToID;
             code += "    " + fakeNode +
               " [label = \"\", shape = point, width = 0.01, height = 0.01]\n";
             code += "    " + layerID + " -> " + fakeNode + " [label = " + size + ", arrowhead = none]\n";
-            code += "    " + fakeNode + " -> " + layertoID + "\n";
+            code += "    " + fakeNode + " -> " + layerToID + "\n";
           } else
-            code += "    " + layerID + " -> " + layertoID + " [label = " + size + "]\n";
+            code += "    " + layerID + " -> " + layerToID + " [label = " + size + "]\n";
           for (var from in connection.gatedfrom) { // gatings
             var layerfrom = connection.gatedfrom[from].layer;
             var type = connection.gatedfrom[from].type;
@@ -439,12 +426,12 @@ Network.prototype = {
             code += "    " + layerfromID + " -> " + fakeNode + " [color = blue]\n";
           }
         } else {
-          code += "    " + layerID + " -> " + layertoID + " [label = " + size + "]\n";
+          code += "    " + layerID + " -> " + layerToID + " [label = " + size + "]\n";
           for (var from in connection.gatedfrom) { // gatings
             var layerfrom = connection.gatedfrom[from].layer;
             var type = connection.gatedfrom[from].type;
             var layerfromID = layers.indexOf(layerfrom);
-            code += "    " + layerfromID + " -> " + layertoID + " [color = blue]\n";
+            code += "    " + layerfromID + " -> " + layerToID + " [color = blue]\n";
           }
         }
       }
@@ -473,7 +460,7 @@ Network.prototype = {
     // build network activation
     for (var neuron in data.activate) { // shouldn't this be layer?
       for (var sentence in data.activate[neuron])
-        activation += data.activate[neuron][sentence] + "\n";
+        activation += data.activate[neuron][sentence].join('') + "\n";
     }
 
     // build outputs
@@ -506,44 +493,110 @@ Network.prototype = {
     return new Function(hardcode)();
   },
 
-  worker: function() {
+
+  // Return a HTML5 WebWorker specialized on training the network stored in `memory`.
+  // Train based on the given dataSet and options.
+  // The worker returns the updated `memory` when done.
+  worker: function(memory, set, options) {
+
+    // Copy the options and set defaults (options might be different for each worker)
+    var workerOptions = {};
+    if(options) workerOptions = options
+    workerOptions.rate = options.rate || .2;
+    workerOptions.iterations = options.iterations || 100000;
+    workerOptions.error = options.error || .005;
+    workerOptions.cost = options.cost || null;
+    workerOptions.crossValidate = options.crossValidate || null;
+
+    // Cost function might be different for each worker
+    costFunction = "var cost = " + (options && options.cost || this.cost || Trainer.cost.MSE) + ";\n";
+    var workerFunction = Network.getWorkerSharedFunctions();
+    workerFunction = workerFunction.replace(/var cost = options && options\.cost \|\| this\.cost \|\| Trainer\.cost\.MSE;/g, costFunction);
+
+    // Set what we do when training is finished
+    workerFunction = workerFunction.replace('return results;',
+                      'postMessage({action: "done", message: results, memoryBuffer: F}, [F.buffer]);');
+
+    // Replace log with postmessage
+    workerFunction = workerFunction.replace("console.log('iterations', iterations, 'error', error, 'rate', currentRate)",
+              "postMessage({action: 'log', message: {\n" +
+                  "iterations: iterations,\n" +
+                  "error: error,\n" +
+                  "rate: currentRate\n" +
+                "}\n" +
+              "})");
+
+    // Replace schedule with postmessage
+    workerFunction = workerFunction.replace("abort = this.schedule.do({ error: error, iterations: iterations, rate: currentRate })",
+              "postMessage({action: 'schedule', message: {\n" +
+                  "iterations: iterations,\n" +
+                  "error: error,\n" +
+                  "rate: currentRate\n" +
+                "}\n" +
+              "})");
+
     if (!this.optimized)
       this.optimize();
 
-    var hardcode = "var inputs = " + this.optimized.data.inputs.length +
-      ";\n";
-    hardcode += "var outputs = " + this.optimized.data.outputs.length +
-      ";\n";
-    hardcode += "var F = null;\n";
-    hardcode += "var activate = " + this.optimized.activate.toString() +
-      ";\n";
-    hardcode += "var propagate = " + this.optimized.propagate.toString() +
-      ";\n";
-    hardcode += "onmessage = function(e){\n";
-    hardcode += "F = e.data.memoryBuffer;\n";
-    hardcode += "if (e.data.action == 'activate'){\n";
-    hardcode += "if (e.data.input.length == inputs){\n";
+    var hardcode = "var inputs = " + this.optimized.data.inputs.length + ";\n";
+    hardcode += "var outputs = " + this.optimized.data.outputs.length + ";\n";
+    hardcode += "var F =  new Float64Array([" + this.optimized.memory.toString() + "]);\n";
+    hardcode += "var activate = " + this.optimized.activate.toString() + ";\n";
+    hardcode += "var propagate = " + this.optimized.propagate.toString() + ";\n";
     hardcode +=
-      "postMessage( { action: 'activate', output: activate(e.data.input), memoryBuffer: F }, [F.buffer]);\n";
-    hardcode += "}\n}\nelse if (e.data.action == 'propagate'){\n";
-    hardcode += "propagate(e.data.rate, e.data.target);\n";
-    hardcode +=
-      "postMessage({ action: 'propagate', memoryBuffer: F }, [F.buffer]);\n";
-    hardcode += "}\n}\n";
+        "onmessage = function(e) {\n" +
+          "if (e.data.action == 'startTraining') {\n" +
+            "train(" + JSON.stringify(set) + "," + JSON.stringify(workerOptions) + ");\n" +
+          "}\n" +
+        "}";
 
-    var blob = new Blob([hardcode]);
+    var workerSourceCode = workerFunction + '\n' + hardcode;
+    var blob = new Blob([workerSourceCode]);
     var blobURL = window.URL.createObjectURL(blob);
 
     return new Worker(blobURL);
   },
 
   // returns a copy of the network
-  clone: function(ignoreTraces) {
-    return Network.fromJSON(this.toJSON(ignoreTraces));
+  clone: function() {
+    return Network.fromJSON(this.toJSON());
   }
 }
 
-// rebuild a network that has been stored in a json using the method toJson()
+/**
+ * Creates a static String to store the source code of the functions
+ *  that are identical for all the workers (train, _trainSet, test)
+ *
+ * @return {String} Source code that can train a network inside a worker.
+ * @static
+ */
+Network.getWorkerSharedFunctions = function() {
+  // If we already computed the source code for the shared functions
+  if(typeof Network._SHARED_WORKER_FUNCTIONS !== 'undefined')
+    return Network._SHARED_WORKER_FUNCTIONS;
+
+  // Otherwise compute and return the source code
+  // We compute them by simply copying the source code of the train, _trainSet and test functions
+  //  using the .toString() method
+
+  // Load and name the train function
+  var train_f = Trainer.prototype.train.toString();
+  train_f = train_f.replace('function (set', 'function train(set') + '\n';
+
+  // Load and name the _trainSet function
+  var _trainSet_f = Trainer.prototype._trainSet.toString().replace(/this.network./g, '');
+  _trainSet_f = _trainSet_f.replace('function (set', 'function _trainSet(set') + '\n';
+  _trainSet_f = _trainSet_f.replace('this.crossValidate', 'crossValidate');
+  _trainSet_f = _trainSet_f.replace('crossValidate = true', 'crossValidate = { }');
+
+  // Load and name the test function
+  var test_f = Trainer.prototype.test.toString().replace(/this.network./g, '');
+  test_f = test_f.replace('function (set', 'function test(set') + '\n';
+
+  return Network._SHARED_WORKER_FUNCTIONS = train_f + _trainSet_f + test_f;
+};
+
+// rebuild a network that has been stored in a json using the method toJSON()
 Network.fromJSON = function(json) {
 
   var neurons = [];
@@ -558,14 +611,13 @@ Network.fromJSON = function(json) {
     var config = json.neurons[i];
 
     var neuron = new Neuron();
-    neuron.trace.elegibility = config.trace.elegibility;
-    neuron.trace.extended = config.trace.extended;
+    neuron.trace.elegibility = {};
+    neuron.trace.extended = {};
     neuron.state = config.state;
     neuron.old = config.old;
     neuron.activation = config.activation;
     neuron.bias = config.bias;
-    neuron.squash = config.squash in Neuron.squash ? Neuron.squash[config.squash] :
-      Neuron.squash.LOGISTIC;
+    neuron.squash = config.squash in Neuron.squash ? Neuron.squash[config.squash] : Neuron.squash.LOGISTIC;
     neurons.push(neuron);
 
     if (config.layer == 'input')
@@ -593,7 +645,3 @@ Network.fromJSON = function(json) {
 
   return new Network(layers);
 }
-
-// export
-if (module) module.exports = Network;
-

src/neuron.js

@@ -1,3 +1,6 @@
+// export
+if (module) module.exports = Neuron;
+
 /******************************************************************************************
                                          NEURON
 *******************************************************************************************/
@@ -76,7 +79,7 @@ Neuron.prototype = {
       }
       influences[neuron.ID] = influence;
     }
-    
+
     for (var i in this.connections.inputs) {
       var input = this.connections.inputs[i];
 
@@ -117,7 +120,7 @@ Neuron.prototype = {
     // output neurons get their error from the enviroment
     if (isOutput)
       this.error.responsibility = this.error.projected = target - this.activation; // Eq. 10
-    
+
     else // the rest of the neuron compute their error responsibilities by backpropagation
     {
       // error responsibilities from all the connections projected from this neuron
@@ -299,7 +302,7 @@ Neuron.prototype = {
 
   // hardcodes the behaviour of the neuron into an optimized function
   optimize: function(optimized, layer) {
-    
+
     optimized = optimized || {};
     var that = this;
     var store_activation = [];
@@ -460,19 +463,20 @@ Neuron.prototype = {
           buildSentence(derivative, ' = 1', store_activation);
           break;
         case Neuron.squash.HLIM:
-          buildSentence(activation, ' = +(', state, ' > 0)',
-            store_activation);
+          buildSentence(activation, ' = +(', state, ' > 0)', store_activation);
           buildSentence(derivative, ' = 1', store_activation);
+        case Neuron.squash.RELU:
+          buildSentence(activation, ' = ', state, ' > 0 ? ', state, ' : 0', store_activation);
+          buildSentence(derivative, ' = ', state, ' > 0 ? 1 : 0', store_activation);
           break;
       }
 
-      influences = [];
       for (var id in this.trace.extended) {
         // calculate extended elegibility traces in advance
-        
+
         var xtrace = this.trace.extended[id];
         var neuron = this.neighboors[id];
-        var influence = getVar('aux');
+        var influence = getVar('influences[' + neuron.ID + ']');
         var neuron_old = getVar(neuron, 'old');
         var initialized = false;
         if (neuron.selfconnection.gater == this)
@@ -487,19 +491,14 @@ Neuron.prototype = {
             [incoming].from, 'activation');
 
           if (initialized)
-            buildSentence(influence, ' += ', incoming_weight, ' * ',
-              incoming_activation, store_trace);
+            buildSentence(influence, ' += ', incoming_weight, ' * ', incoming_activation, store_trace);
           else {
-            buildSentence(influence, ' = ', incoming_weight, ' * ',
-              incoming_activation, store_trace);
+            buildSentence(influence, ' = ', incoming_weight, ' * ', incoming_activation, store_trace);
             initialized = true;
           }
         }
-        
-        influences.push(neuron.ID);
-        buildSentence("influences[" + (influences.length - 1) + "] = ", influence, store_trace);
       }
-      
+
       for (var i in this.connections.inputs) {
         var input = this.connections.inputs[i];
         if (input.gater)
@@ -535,7 +534,7 @@ Neuron.prototype = {
           // extended elegibility trace
           var xtrace = this.trace.extended[id];
           var neuron = this.neighboors[id];
-          var influence = getVar('aux');
+          var influence = getVar('influences[' + neuron.ID + ']');
           var neuron_old = getVar(neuron, 'old');
 
           var trace = getVar(this, 'trace', 'elegibility', input.ID, this.trace
@@ -550,14 +549,14 @@ Neuron.prototype = {
             if (neuron.selfconnection.gater)
               buildSentence(xtrace, ' = ', neuron_self_gain, ' * ',
                 neuron_self_weight, ' * ', xtrace, ' + ', derivative, ' * ',
-                trace, ' * ', "influences[" + influences.indexOf(neuron.ID) + "]", store_trace);
+                trace, ' * ', influence, store_trace);
             else
               buildSentence(xtrace, ' = ', neuron_self_weight, ' * ',
                 xtrace, ' + ', derivative, ' * ', trace, ' * ',
-                "influences[" + influences.indexOf(neuron.ID) + "]", store_trace);
+                influence, store_trace);
           else
             buildSentence(xtrace, ' = ', derivative, ' * ', trace, ' * ',
-              "influences[" + influences.indexOf(neuron.ID) + "]", store_trace);
+              influence, store_trace);
         }
       }
       for (var connection in this.connections.gated) {
@@ -742,7 +741,7 @@ Neuron.prototype = {
 Neuron.connection = function Connection(from, to, weight) {
 
   if (!from || !to)
-    throw "Connection Error: Invalid neurons";
+    throw new Error("Connection Error: Invalid neurons");
 
   this.ID = Neuron.connection.uid();
   this.from = from;
@@ -775,7 +774,12 @@ Neuron.squash.IDENTITY = function(x, derivate) {
   return derivate ? 1 : x;
 };
 Neuron.squash.HLIM = function(x, derivate) {
-  return derivate ? 1 : +(x > 0);
+  return derivate ? 1 : x > 0 ? 1 : 0;
+};
+Neuron.squash.RELU = function(x, derivate) {
+  if (derivate)
+    return x > 0 ? 1 : 0;
+  return x > 0 ? x : 0;
 };
 
 // unique ID's
@@ -795,7 +799,3 @@ Neuron.squash.HLIM = function(x, derivate) {
     }
   }
 })();
-
-// export
-if (module) module.exports = Neuron;
-

src/synaptic.js

@@ -1,54 +1,3 @@
-/*
-
-The MIT License (MIT)
-
-Copyright (c) 2014 Juan Cazala - juancazala.com
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE
-
-
-
-********************************************************************************************
-                                         SYNAPTIC
-********************************************************************************************
-
-Synaptic is a javascript neural network library for node.js and the browser, its generalized
-algorithm is architecture-free, so you can build and train basically any type of first order
-or even second order neural network architectures.
-
-http://en.wikipedia.org/wiki/Recurrent_neural_network#Second_Order_Recurrent_Neural_Network
-
-The library includes a few built-in architectures like multilayer perceptrons, multilayer
-long-short term memory networks (LSTM) or liquid state machines, and a trainer capable of
-training any given network, and includes built-in training tasks/tests like solving an XOR,
-passing a Distracted Sequence Recall test or an Embeded Reber Grammar test.
-
-The algorithm implemented by this library has been taken from Derek D. Monner's paper:
-
-A generalized LSTM-like training algorithm for second-order recurrent neural networks
-http://www.overcomplete.net/papers/nn2012.pdf
-
-There are references to the equations in that paper commented through the source code.
-
-
-********************************************************************************************/
-
 var Synaptic = {
     Neuron: require('./neuron'),
     Layer: require('./layer'),
@@ -72,12 +21,12 @@ if (typeof module !== 'undefined' && module.exports)
 // Browser
 if (typeof window == 'object')
 {
-  (function(){ 
+  (function(){
     var oldSynaptic = window['synaptic'];
-    Synaptic.ninja = function(){ 
-      window['synaptic'] = oldSynaptic; 
+    Synaptic.ninja = function(){
+      window['synaptic'] = oldSynaptic;
       return Synaptic;
-    };	
+    };
   })();
 
   window['synaptic'] = Synaptic;

src/trainer.js

@@ -1,3 +1,6 @@
+// export
+if (module) module.exports = Trainer;
+
 /*******************************************************************************************
                                         TRAINER
 *******************************************************************************************/
@@ -7,8 +10,9 @@ function Trainer(network, options) {
   this.network = network;
   this.rate = options.rate || .2;
   this.iterations = options.iterations || 100000;
-  this.error = options.error || .005
-  this.cost = options.cost || Trainer.cost.CROSS_ENTROPY;
+  this.error = options.error || .005;
+  this.cost = options.cost || null;
+  this.crossValidate = options.crossValidate || null;
 }
 
 Trainer.prototype = {
@@ -18,8 +22,10 @@ Trainer.prototype = {
 
     var error = 1;
     var iterations = bucketSize = 0;
-    var abort_training = false;
+    var abort = false;
     var input, output, target, currentRate;
+    var cost = options && options.cost || this.cost || Trainer.cost.MSE;
+    var crossValidate = false, testSet, trainSet;
 
     var start = Date.now();
 
@@ -47,6 +53,14 @@ Trainer.prototype = {
         console.log('Deprecated: use schedule instead of customLog')
         this.schedule = options.customLog;
       }
+      if (this.crossValidate || options.crossValidate) {
+        if(!this.crossValidate) this.crossValidate = {};
+        crossValidate = true;
+        if (options.crossValidate.testSize)
+          this.crossValidate.testSize = options.crossValidate.testSize;
+        if (options.crossValidate.testError)
+          this.crossValidate.testError = options.crossValidate.testError;
+      }
     }
 
     currentRate = this.rate;
@@ -54,37 +68,42 @@ Trainer.prototype = {
       bucketSize = Math.floor(this.iterations / this.rate.length);
     }
 
+    if(crossValidate) {
+      var numTrain = Math.ceil((1 - this.crossValidate.testSize) * set.length);
+      trainSet = set.slice(0, numTrain);
+      testSet = set.slice(numTrain);
+    }
 
-    while (!abort_training && iterations < this.iterations && error > this.error) {
+    while ((!abort && iterations < this.iterations && error > this.error)) {
+      if (crossValidate && error <= this.crossValidate.testError) {
+        break;
+      }
+
+      var currentSetSize = set.length;
       error = 0;
 
       if(bucketSize > 0) {
         var currentBucket = Math.floor(iterations / bucketSize);
-        currentRate = this.rate[currentBucket];
+        currentRate = this.rate[currentBucket] || currentRate;
       }
 
-      for (var train in set) {
-        input = set[train].input;
-        target = set[train].output;
-
-        output = this.network.activate(input);
-        this.network.propagate(currentRate, target);
-
-        error += this.cost(target, output);
+      if (crossValidate) {
+        this._trainSet(trainSet, currentRate, cost);
+        error += this.test(testSet).error;
+        currentSetSize = 1;
+      } else {
+        error += this._trainSet(set, currentRate, cost);
+        currentSetSize = set.length;
       }
 
       // check error
       iterations++;
-      error /= set.length;
+      error /= currentSetSize;
 
       if (options) {
         if (this.schedule && this.schedule.every && iterations %
           this.schedule.every == 0)
-          abort_training = this.schedule.do({
-            error: error,
-            iterations: iterations,
-            rate: currentRate
-          });
+          abort = this.schedule.do({ error: error, iterations: iterations, rate: currentRate });
         else if (options.log && iterations % options.log == 0) {
           console.log('iterations', iterations, 'error', error, 'rate', currentRate);
         };
@@ -102,140 +121,116 @@ Trainer.prototype = {
     return results;
   },
 
-  // trains any given set to a network using a WebWorker
-  workerTrain: function(set, callback, options) {
+  // trains any given set to a network, using a WebWorker (only for the browser). Returns a Promise of the results.
+  trainAsync: function(set, options) {
+    var train = this.workerTrain.bind(this)
+    return new Promise(function(resolve, reject) {
+      try {
+        train(set, resolve, options, true)
+      } catch(e) {
+        reject(e)
+      }
+    })
+  },
 
-    var that = this;
-    var error = 1;
-    var iterations = bucketSize = 0;
-    var input, output, target, currentRate;
-    var length = set.length;
-    var abort_training = false;
+  // preforms one training epoch and returns the error (private function used in this.train)
+  _trainSet: function(set, currentRate, costFunction) {
+    var errorSum = 0;
+    for (var train in set) {
+      input = set[train].input;
+      target = set[train].output;
+
+      output = this.network.activate(input);
+      this.network.propagate(currentRate, target);
+
+      errorSum += costFunction(target, output);
+    }
+    return errorSum;
+  },
+
+  // tests a set and returns the error and elapsed time
+  test: function(set, options) {
+
+    var error = 0;
+    var abort = false;
+    var input, output, target;
+    var cost = options && options.cost || this.cost || Trainer.cost.MSE;
 
     var start = Date.now();
 
-    if (options) {
-      if (options.shuffle) {
-        //+ Jonas Raoni Soares Silva
-        //@ http://jsfromhell.com/array/shuffle [v1.0]
-        function shuffle(o) { //v1.0
-          for (var j, x, i = o.length; i; j = Math.floor(Math.random() *
-              i), x = o[--i], o[i] = o[j], o[j] = x);
-          return o;
-        };
-      }
-      if (options.iterations)
-        this.iterations = options.iterations;
-      if (options.error)
-        this.error = options.error;
-      if (options.rate)
-        this.rate = options.rate;
-      if (options.cost)
-        this.cost = options.cost;
-      if (options.schedule)
-        this.schedule = options.schedule;
-      if (options.customLog)
-        // for backward compatibility with code that used customLog
-        console.log('Deprecated: use schedule instead of customLog')
-        this.schedule = options.customLog;
+    for (var test in set) {
+      input = set[test].input;
+      target = set[test].output;
+      output = this.network.activate(input);
+      error += cost(target, output);
     }
 
-    // dynamic learning rate
-    currentRate = this.rate;
-    if(Array.isArray(this.rate)) {
-      bucketSize = Math.floor(this.iterations / this.rate.length);
+    error /= set.length;
+
+    var results = {
+      error: error,
+      time: Date.now() - start
     }
 
-    // create a worker
-    var worker = this.network.worker();
+    return results;
+  },
 
-    // activate the network
-    function activateWorker(input)
-    {
-        worker.postMessage({ 
-            action: "activate",
-            input: input,
-            memoryBuffer: that.network.optimized.memory
-        }, [that.network.optimized.memory.buffer]);
-    }
+  // trains any given set to a network using a WebWorker [deprecated: use trainAsync instead]
+  workerTrain: function(set, callback, options, suppressWarning) {
 
-    // backpropagate the network
-    function propagateWorker(target){
-        if(bucketSize > 0) {
-          var currentBucket = Math.floor(iterations / bucketSize);
-          currentRate = this.rate[currentBucket];
-        }
-        worker.postMessage({ 
-            action: "propagate",
-            target: target,
-            rate: currentRate,
-            memoryBuffer: that.network.optimized.memory
-        }, [that.network.optimized.memory.buffer]);
+    if (!suppressWarning) {
+      console.warn('Deprecated: do not use `workerTrain`, use `trainAsync` instead.')
     }
+    var that = this;
+
+    if (!this.network.optimized)
+      this.network.optimize();
+
+    // Create a new worker
+    var worker = this.network.worker(this.network.optimized.memory, set, options);
 
     // train the worker
-    worker.onmessage = function(e){
-        // give control of the memory back to the network
-        that.network.optimized.ownership(e.data.memoryBuffer);
+    worker.onmessage = function(e) {
+      switch(e.data.action) {
+          case 'done':
+            var iterations = e.data.message.iterations;
+            var error = e.data.message.error;
+            var time = e.data.message.time;
 
-        if (e.data.action == "propagate")
-        {
-            if (index >= length)
-            {
-                index = 0;
-                iterations++;
-                error /= set.length;
+            that.network.optimized.ownership(e.data.memoryBuffer);
 
-                // log
-                if (options) {
-                  if (this.schedule && this.schedule.every && iterations % this.schedule.every == 0)
-                    abort_training = this.schedule.do({
-                      error: error,
-                      iterations: iterations
-                    });
-                  else if (options.log && iterations % options.log == 0) {
-                    console.log('iterations', iterations, 'error', error);
-                  };
-                  if (options.shuffle)
-                    shuffle(set);
-                }
+            // Done callback
+            callback({
+              error: error,
+              iterations: iterations,
+              time: time
+            });
 
-                if (!abort_training && iterations < that.iterations && error > that.error)
-                {
-                    activateWorker(set[index].input);
-                } else {
-                    // callback
-                    callback({
-                      error: error,
-                      iterations: iterations,
-                      time: Date.now() - start
-                    })
-                }
-                error = 0;
-            } else {
-                activateWorker(set[index].input);
+            // Delete the worker and all its associated memory
+            worker.terminate();
+          break;
+
+          case 'log':
+            console.log(e.data.message);
+
+          case 'schedule':
+            if (options && options.schedule && typeof options.schedule.do === 'function') {
+              var scheduled = options.schedule.do
+              scheduled(e.data.message)
             }
-        }
-
-        if (e.data.action == "activate")
-        {
-            error += that.cost(set[index].output, e.data.output);
-            propagateWorker(set[index].output); 
-            index++;
-        }
+          break;
+      }
     }
 
-    // kick it
-    var index = 0;
-    var iterations = 0;
-    activateWorker(set[index].input);
+    // Start the worker
+    worker.postMessage({action: 'startTraining'});
   },
 
   // trains an XOR to the network
   XOR: function(options) {
 
     if (this.network.inputs() != 2 || this.network.outputs() != 1)
-      throw "Error: Incompatible network (2 inputs, 1 output)";
+      throw new Error("Incompatible network (2 inputs, 1 output)");
 
     var defaults = {
       iterations: 100000,
@@ -276,6 +271,7 @@ Trainer.prototype = {
     var rate = options.rate || .1;
     var log = options.log || 0;
     var schedule = options.schedule || {};
+    var cost = options.cost || this.cost || Trainer.cost.CROSS_ENTROPY;
 
     var trial = correct = i = j = success = 0,
       error = 1,
@@ -351,10 +347,7 @@ Trainer.prototype = {
           this.network.propagate(rate, output);
         }
 
-        var delta = 0;
-        for (var j in prediction)
-          delta += Math.pow(output[j] - prediction[j], 2);
-        error += delta / this.network.outputs();
+        error += cost(output, prediction);
 
         if (distractorsCorrect + targetsCorrect == length)
           correct++;
@@ -399,6 +392,7 @@ Trainer.prototype = {
     var criterion = options.error || .05;
     var rate = options.rate || .1;
     var log = options.log || 500;
+    var cost = options.cost || this.cost || Trainer.cost.CROSS_ENTROPY;
 
     // gramar node
     var Node = function() {
@@ -557,12 +551,7 @@ Trainer.prototype = {
         read = sequence.charAt(++i);
         predict = sequence.charAt(i + 1);
 
-        var delta = 0;
-        for (var k in output)
-          delta += Math.pow(target[k] - output[k], 2)
-        delta /= output.length;
-
-        error += delta;
+        error += cost(target, output);
       }
       error /= sequence.length;
       iteration++;
@@ -579,6 +568,78 @@ Trainer.prototype = {
       test: test,
       generate: generate
     }
+  },
+
+  timingTask: function(options){
+
+    if (this.network.inputs() != 2 || this.network.outputs() != 1)
+      throw new Error("Invalid Network: must have 2 inputs and one output");
+
+    if (typeof options == 'undefined')
+      var options = {};
+
+    // helper
+    function getSamples (trainingSize, testSize){
+
+      // sample size
+      var size = trainingSize + testSize;
+
+      // generate samples
+      var t = 0;
+      var set  = [];
+      for (var i = 0; i < size; i++) {
+        set.push({ input: [0,0], output: [0] });
+      }
+      while(t < size - 20) {
+          var n = Math.round(Math.random() * 20);
+          set[t].input[0] = 1;
+          for (var j = t; j <= t + n; j++){
+              set[j].input[1] = n / 20;
+              set[j].output[0] = 0.5;
+          }
+          t += n;
+          n = Math.round(Math.random() * 20);
+          for (var k = t+1; k <= (t + n) &&  k < size; k++)
+              set[k].input[1] = set[t].input[1];
+          t += n;
+      }
+
+      // separate samples between train and test sets
+      var trainingSet = []; var testSet = [];
+      for (var l = 0; l < size; l++)
+          (l < trainingSize ? trainingSet : testSet).push(set[l]);
+
+      // return samples
+      return {
+          train: trainingSet,
+          test: testSet
+      }
+    }
+
+    var iterations = options.iterations || 200;
+    var error = options.error || .005;
+    var rate = options.rate || [.03, .02];
+    var log = options.log === false ? false : options.log || 10;
+    var cost = options.cost || this.cost || Trainer.cost.MSE;
+    var trainingSamples = options.trainSamples || 7000;
+    var testSamples = options.trainSamples || 1000;
+
+    // samples for training and testing
+    var samples = getSamples(trainingSamples, testSamples);
+
+    // train
+    var result = this.train(samples.train, {
+      rate: rate,
+      log: log,
+      iterations: iterations,
+      error: error,
+      cost: cost
+    });
+
+    return {
+      train: result,
+      test: this.test(samples.test)
+    }
   }
 };
 
@@ -598,9 +659,11 @@ Trainer.cost = {
     for (var i in output)
       mse += Math.pow(target[i] - output[i], 2);
     return mse / output.length;
+  },
+  BINARY: function(target, output){
+    var misses = 0;
+    for (var i in output)
+      misses += Math.round(target[i] * 2) != Math.round(output[i] * 2);
+    return misses;
   }
 }
-
-// export
-if (module) module.exports = Trainer;
-

test/synaptic.js

@@ -12,29 +12,43 @@ var Perceptron = synaptic.Architect.Perceptron,
 
 // utils
 
-var noRepeat = function(range, avoid) {
+function noRepeat (range, avoid) {
   var number = Math.random() * range | 0;
-  var used = false;
-  for (var i in avoid)
-    if (number == avoid[i])
-      used = true;
-  return used ? noRepeat(range, avoid) : number;
-};
+  for (var i in avoid){
+    if (number == avoid[i]){
+      return noRepeat(range,avoid);
+    }
+  }
+  return number;
+}
 
-var equal = function(prediction, output) {
+function equal (prediction, output) {
   for (var i in prediction)
     if (Math.round(prediction[i]) != output[i])
       return false;
   return true;
-};
+}
 
-var generateRandomArray = function(size){
+function generateRandomArray (size){
     var array = [];
     for (var j = 0; j < size; j++)
         array.push(Math.random() + .5 | 0);
     return array;
 }
 
+function compare (a, b) {
+  var mse = 0;
+  for (var k in a)
+    mse += Math.pow(a[k] - b[k], 2);
+  mse /= a.length;
+
+  return mse < 1e-10;
+}
+
+function equalWithError (output, expected, error) {
+  return Math.abs(output - expected) <= error;
+}
+
 // specs
 
 describe('Basic Neural Network', function() {
@@ -78,7 +92,7 @@ describe('Basic Neural Network', function() {
     var test01 = Math.round(network.activate([0, 1]));
     assert.equal(test01, 0, "[0,1] did not output 0");
 
-    var test10 = Math.round(network.activate([0, 1]));
+    var test10 = Math.round(network.activate([1, 0]));
     assert.equal(test10, 0, "[1,0] did not output 0");
 
     var test11 = Math.round(network.activate([1, 1]));
@@ -124,7 +138,7 @@ describe('Basic Neural Network', function() {
     var test01 = Math.round(network.activate([0, 1]));
     assert.equal(test01, 1, "[0,1] did not output 1");
 
-    var test10 = Math.round(network.activate([0, 1]));
+    var test10 = Math.round(network.activate([1, 0]));
     assert.equal(test10, 1, "[1,0] did not output 1");
 
     var test11 = Math.round(network.activate([1, 1]));
@@ -196,6 +210,111 @@ describe("Perceptron - XOR", function() {
   });
 });
 
+describe("Perceptron - SIN", function() {
+  var mySin = function(x) {
+    return (Math.sin(x)+1)/2;
+  };
+
+  var sinNetwork = new Perceptron(1, 12, 1);
+
+  var trainingSet = Array.apply(null, Array(800)).map(function () {
+    var inputValue = Math.random() * Math.PI * 2;
+    return {
+      input: [inputValue],
+      output: [mySin(inputValue)]
+    };
+  });
+
+  var results = sinNetwork.trainer.train(trainingSet, {
+    iterations: 2000,
+    log: false,
+    error: 1e-6,
+    cost: Trainer.cost.MSE,
+  });
+
+  var test0 = sinNetwork.activate([0])[0];
+  var expected0 = mySin(0);
+  it("input: [0] output: " + test0 + ", expected: " + expected0, function() {
+    var eq = equalWithError(test0, expected0, .035);
+    assert.equal(eq, true, "[0] did not output " + expected0);
+  });
+
+  var test05PI = sinNetwork.activate([.5*Math.PI])[0];
+  var expected05PI = mySin(.5*Math.PI);
+  it("input: [0.5*Math.PI] output: " + test05PI + ", expected: " + expected05PI, function() {
+    var eq = equalWithError(test05PI, expected05PI, .035);
+    assert.equal(eq, true, "[0.5*Math.PI] did not output " + expected05PI);
+  });
+
+  var test2 = sinNetwork.activate([2])[0];
+  var expected2 = mySin(2);
+  it("input: [2] output: " + test2 + ", expected: " + expected2, function() {
+    var eq = equalWithError(test2, expected2, .035);
+    assert.equal(eq, true, "[2] did not output " + expected2);
+  });
+
+  var errorResult = results.error;
+  it("Sin error: " + errorResult, function() {
+    var lessThanOrEqualError = errorResult <= .001;
+    assert.equal(lessThanOrEqualError, true, "Sin error not less than or equal to desired error.");
+  });
+});
+
+describe("Perceptron - SIN - CrossValidate", function() {
+
+  var mySin = function(x) {
+    return (Math.sin(x)+1)/2;
+  };
+
+  var sinNetwork = new Perceptron(1, 12, 1);
+
+  var trainingSet = Array.apply(null, Array(800)).map(function () {
+    var inputValue = Math.random() * Math.PI * 2;
+    return {
+      input: [inputValue],
+      output: [mySin(inputValue)]
+    };
+  });
+
+  var results = sinNetwork.trainer.train(trainingSet, {
+    iterations: 2000,
+    log: false,
+    error: 1e-6,
+    cost: Trainer.cost.MSE,
+    crossValidate: {
+      testSize: .3,
+      testError: 1e-6
+    }
+  });
+
+  var test0 = sinNetwork.activate([0])[0];
+  var expected0 = mySin(0);
+  it("input: [0] output: " + test0 + ", expected: " + expected0, function() {
+    var eq = equalWithError(test0, expected0, .035);
+    assert.equal(eq, true, "[0] did not output " + expected0);
+  });
+
+  var test05PI = sinNetwork.activate([.5*Math.PI])[0];
+  var expected05PI = mySin(.5*Math.PI);
+  it("input: [0.5*Math.PI] output: " + test05PI + ", expected: " + expected05PI, function() {
+    var eq = equalWithError(test05PI, expected05PI, .035);
+    assert.equal(eq, true, "[0.5*Math.PI] did not output " + expected05PI);
+  });
+
+  var test2 = sinNetwork.activate([2])[0];
+  var expected2 = mySin(2);
+  it("input: [2] output: " + test2 + ", expected: " + expected2, function() {
+    var eq = equalWithError(test2, expected2, .035);
+    assert.equal(eq, true, "[2] did not output " + expected2);
+  });
+
+  var errorResult = results.error;
+  it("CrossValidation error: " + errorResult, function() {
+    var lessThanOrEqualError = errorResult <= .001;
+    assert.equal(lessThanOrEqualError, true, "CrossValidation error not less than or equal to desired error.");
+  });
+});
+
 describe("LSTM - Discrete Sequence Recall", function() {
 
   var targets = [2, 4];
@@ -293,8 +412,25 @@ describe("LSTM - Discrete Sequence Recall", function() {
   }
 });
 
+describe("LSTM - Timing Task", function() {
+  var network = new synaptic.Architect.LSTM(2,7,1);
+  var result = network.trainer.timingTask({
+    log: false,
+    trainSamples: 4000,
+    testSamples: 500
+  });
+
+  it("should complete the training in less than 200 iterations", function() {
+    assert(result.train.iterations <= 200);
+  });
+
+  it("should pass the test with an error smaller than 0.05", function() {
+    assert(result.test.error < .05);
+  });
+});
+
 describe("Optimized and Unoptimized Networks Equivalency", function() {
-  var optimized = new Perceptron(10,15,5);
+  var optimized = new LSTM(2,1,1)
 
   var unoptimized = optimized.clone();
   unoptimized.setOptimize(false);
@@ -305,23 +441,19 @@ describe("Optimized and Unoptimized Networks Equivalency", function() {
   for (var i = 1; i <= iterations; i++)
   {
       //random input
-      var input = generateRandomArray(10);
+      var input = generateRandomArray(2);
 
       // activate networks
       var output1 = optimized.activate(input);
       var output2 = unoptimized.activate(input);
 
       if (i % 100 == 0)
-        it(' same output for both networks after ' + i + ' iterations', function(){
-          var diff = false;
-          for (var k in output1)
-            if (output1[k] - output2[k] != 0)
-              diff = true;
-          assert(!diff);
+        it('should produce the same output for both networks after ' + i + ' iterations', function(){
+          assert(compare(output1, output2));
         });
 
       // random target
-      var target = generateRandomArray(5);
+      var target = generateRandomArray(1);
 
       // propagate networks
       optimized.propagate(learningRate, target);
@@ -330,7 +462,7 @@ describe("Optimized and Unoptimized Networks Equivalency", function() {
 });
 
 describe("toJSON/fromJSON Networks Equivalency", function() {
-  var original = new Perceptron(10,15,5);
+  var original = new LSTM(10,5,5);
 
   var exported = original.toJSON();
   var imported = Network.fromJSON(exported);
@@ -348,12 +480,8 @@ describe("toJSON/fromJSON Networks Equivalency", function() {
       var output2 = imported.activate(input);
 
       if (i % 100 == 0)
-        it(' same output for both networks after ' + i + ' iterations', function(){
-          var diff = false;
-          for (var k in output1)
-            if (output1[k] - output2[k] != 0)
-              diff = true;
-          assert(!diff);
+        it('should produce the same output for both networks after ' + i + ' iterations', function(){
+          assert(compare(output1, output2));
         });
 
       // random target
@@ -366,7 +494,9 @@ describe("toJSON/fromJSON Networks Equivalency", function() {
 });
 
 describe("Cloned Networks Equivalency", function() {
-  var original = new Perceptron(10,15,5);
+
+  var original = new LSTM(10,5,5);
+
   var cloned = original.clone();
 
   var learningRate = .5;
@@ -382,12 +512,8 @@ describe("Cloned Networks Equivalency", function() {
       var output2 = cloned.activate(input);
 
       if (i % 100 == 0)
-        it(' same output for both networks after ' + i + ' iterations', function(){
-          var diff = false;
-          for (var k in output1)
-            if (output1[k] - output2[k] != 0)
-              diff = true;
-          assert(!diff);
+        it('should produce the same output for both networks after ' + i + ' iterations', function(){
+          assert(compare(output1, output2));
         });
 
       // random target
@@ -399,10 +525,10 @@ describe("Cloned Networks Equivalency", function() {
   }
 });
 
-describe("Manual Override", function() {
+describe("Scheduled Tasks", function() {
   var perceptron = new Perceptron(2, 3, 1);
 
-  it('iterations ended at full 3000', function(){
+  it('should stop training at 3000 iterations', function(){
     var final_stats = perceptron.trainer.XOR({
       iterations: 3000,
       rate: 0.000001,
@@ -419,7 +545,7 @@ describe("Manual Override", function() {
     assert.equal( final_stats.iterations, 3000 )
   });
 
-  it('iterations ended at 2000, not full 3000', function(){
+  it('should abort the training at 2000 iterations', function(){
     var final_stats = perceptron.trainer.XOR({
       iterations: 3000,
       rate: 0.000001,
@@ -436,7 +562,7 @@ describe("Manual Override", function() {
     assert.equal( final_stats.iterations, 2000 )
   });
 
-  it('training works even when schedule() has no return value', function(){
+  it('should work even if shedule.do() returns no value', function(){
     var final_stats = perceptron.trainer.XOR({
       iterations: 3000,
       rate: 0.000001,
@@ -449,18 +575,4 @@ describe("Manual Override", function() {
     assert.equal( final_stats.iterations, 3000 )
   });
 
-  it('using depreciated customLog still works', function(){
-    var counter = 0
-    var final_stats = perceptron.trainer.XOR({
-      iterations: 3000,
-      rate: 0.000001,
-      error: 0.000001,
-      customLog: {
-          every: 1000,
-          do: function(data) { counter++ }
-        }
-    });
-    assert.equal( counter, 3 )
-  });
-
 });

test/test_browser.html

@@ -1,22 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-  <title>test</title>
-  <script src="../dist/synaptic.js"></script>
-</head>
-<body>
-
-</body>
-<script type="text/javascript">
-  var Architect = synaptic.Architect;
-
-  var myPerceptron = new Architect.Perceptron(2,3,1);
-  document.body.innerHTML += JSON.stringify(myPerceptron.trainer.XOR())+'<br>'
-  document.body.innerHTML += JSON.stringify(myPerceptron.trainer.XOR())+'<br>'
-
-  if(document.body.innerHTML.length > 0){
-    document.body.innerHTML += '<br>if you see this, then synaptic works'
-  }
-
-</script>
-</html>

webpack.config.js

@@ -0,0 +1,16 @@
+var webpack = require('webpack')
+var license = require('./license.js')
+module.exports = {
+  context: __dirname,
+  entry: [
+    './src/synaptic.js'
+  ],
+  output: {
+    path: 'dist',
+    filename: 'synaptic.js',
+  },
+  plugins: [
+    new webpack.NoErrorsPlugin(),
+    new webpack.BannerPlugin(license())
+  ]
+}