deleted ntm

.settings/launch.json

@@ -0,0 +1,38 @@
+{
+	"version": "0.1.0",
+	// List of configurations. Add new configurations or edit existing ones.  
+	// ONLY "node" and "mono" are supported, change "type" to switch.
+	"configurations": [
+		{
+			// Name of configuration; appears in the launch configuration drop down menu.
+			"name": "Launch app.js",
+			// Type of configuration. Possible values: "node", "mono".
+			"type": "node",
+			// Workspace relative or absolute path to the program.
+			"program": "app.js",
+			// Automatically stop program after launch.
+			"stopOnEntry": true,
+			// Command line arguments passed to the program.
+			"args": [],
+			// Workspace relative or absolute path to the working directory of the program being debugged. Default is the current workspace.
+			"cwd": ".",
+			// Workspace relative or absolute path to the runtime executable to be used. Default is the runtime executable on the PATH.
+			"runtimeExecutable": null,
+			// Optional arguments passed to the runtime executable.
+			"runtimeArguments": [],
+			// Environment variables passed to the program.
+			"env": { },
+			// Use JavaScript source maps (if they exist).
+			"sourceMaps": false
+		}, 
+		{
+			"name": "Attach",
+			"type": "node",
+			// TCP/IP address. Default is "localhost".
+			"address": "localhost",
+			// Port to attach to.
+			"port": 5858,
+			"sourceMaps": false
+		}
+	]
+}

.settings/settings.json

@@ -0,0 +1,4 @@
+// Place your settings in this file to overwrite default and user settings.
+{
+	"editor.fontSize": 15
+}

.settings/tasks.json

@@ -12,6 +12,11 @@
             "isTestCommand": true,
             "problemMatcher": "$tsc"
         },
+        {
+            "taskName": "test-ntm",
+            "isTestCommand": true,
+            "problemMatcher": "$tsc"
+        },
         {
             "isBuildCommand": true,
             "taskName": "build",

dist/src/architect.d.ts

@@ -1,4 +1,4 @@
-import hopfield = require('./architect/hopfield');
+import hopfield = require('./architect/Hopfield');
 import lstm = require('./architect/LSTM');
 import lsm = require('./architect/Liquid');
 import perceptron = require('./architect/Perceptron');

dist/src/architect/NTM.d.ts

@@ -0,0 +1,47 @@
+import network = require('../network');
+import trainer = require('../trainer');
+import Layer = require('../layer');
+import Synaptic = require('../synaptic');
+export declare class NTM extends network.Network {
+    trainer: trainer.Trainer;
+    data: Float64Array[];
+    blockWidth: number;
+    blocks: number;
+    heads: Head[];
+    inputValues: Float64Array;
+    inputLayer: Layer.Layer;
+    hiddenLayer: Layer.Layer;
+    outputLayer: Layer.Layer;
+    dirty: boolean;
+    constructor(inputs: number, outputs: number, memBlocks: number, blockWidth: number, heads: number, hiddenSize: number);
+    clean(): void;
+    activate(input: Synaptic.INumericArray): Synaptic.INumericArray;
+    propagate(rate: number, target: Synaptic.INumericArray): void;
+    addHead(subArray: Float64Array): Head;
+    doTimeStep(): void;
+    doAdd(w: Synaptic.INumericArray, addGate: Synaptic.INumericArray): void;
+    doErase(w: Synaptic.INumericArray, eraseGate: Synaptic.INumericArray): void;
+}
+export declare class Head {
+    static ADDITIONAL_INPUT_VALUES: number;
+    memory: NTM;
+    w_weightings: Float64Array;
+    eraseGate: Float64Array;
+    addGate: Float64Array;
+    k_keys: Float64Array;
+    g_interpolation: number;
+    Y_focus: number;
+    s_shiftingValue: number;
+    s_shiftingVector: Float64Array;
+    wc_focusedWeights: Float64Array;
+    readVector: Float64Array;
+    ß_keyStrength: number;
+    prevFocus: number;
+    shiftLength: number;
+    layer: Layer.Layer;
+    shiftingLayer: Layer.Layer;
+    constructor(memory: NTM, destinationArray?: Float64Array);
+    private readParams(activation);
+    doShiftings(): void;
+    doTimeStep(): void;
+}

dist/src/layer.d.ts

@@ -1,17 +1,20 @@
 import neuron = require('./neuron');
+import network = require('./network');
 import Synaptic = require('./synaptic');
 /*******************************************************************************************
                                             LAYER
 *******************************************************************************************/
 export declare class Layer {
+    optimizable: boolean;
     list: neuron.Neuron[];
     label: string;
     connectedto: any[];
     size: number;
+    currentActivation: Float64Array;
     constructor(size: number, label?: string);
-    activate(input: any): any[];
-    propagate(rate: any, target: any): void;
-    project(layer: any, type?: any, weights?: any): Layer.LayerConnection;
+    activate(input?: Synaptic.INumericArray): Synaptic.INumericArray;
+    propagate(rate: number, target?: Synaptic.INumericArray): void;
+    project(layer: network.Network | Layer, type?: string, weights?: Synaptic.INumericArray): Layer.LayerConnection;
     gate(connection: any, type: any): void;
     selfconnected(): boolean;
     connected(layer: any): string;

dist/src/network.d.ts

@@ -1,3 +1,4 @@
+import Synaptic = require('./synaptic');
 import _neuron = require('./neuron');
 export declare class Network {
     optimized: any;
@@ -6,9 +7,9 @@ export declare class Network {
         hidden: {};
         output: any;
     };
-    constructor(layers: any);
-    activate(input: any): any;
-    propagate(rate: number, target?: any): void;
+    constructor(layers?: any);
+    activate(input: Synaptic.INumericArray): any;
+    propagate(rate: number, target?: Synaptic.INumericArray): void;
     project(unit: any, type: any, weights: any): any;
     gate(connection: any, type: any): void;
     clear(): void;

dist/src/neuron.d.ts

@@ -5,6 +5,7 @@ import Squash = require('./squash');
                                          NEURON
 *******************************************************************************************/
 export declare class Neuron {
+    optimizable: boolean;
     ID: number;
     label: any;
     connections: Neuron.INeuronConnections;
@@ -26,6 +27,9 @@ export declare class Neuron {
     neighboors: {};
     bias: number;
     derivative: number;
+    constructor();
+    readIncommingConnections(input?: number): number;
+    updateTraces(): void;
     activate(input?: number): number;
     propagate(rate: number, target?: number): void;
     project(neuron: any, weight?: number): Neuron.Connection;

dist/src/softmaxLayer.d.ts

@@ -0,0 +1,7 @@
+import Synaptic = require('./synaptic');
+import Layer = require('./layer');
+export declare class SoftMaxLayer extends Layer.Layer {
+    constructor(size: number, label?: string);
+    activate(input?: Synaptic.INumericArray): Synaptic.INumericArray;
+    static NormalizeConnectionWeights(layerConnection: Layer.Layer.LayerConnection): void;
+}

dist/src/squash.d.ts

@@ -1,4 +1,6 @@
-export declare function LOGISTIC(x: number, derivate?: boolean): any;
+export declare function LOGISTIC(x: number, derivate?: boolean): number;
 export declare function TANH(x: number, derivate?: boolean): any;
 export declare function IDENTITY(x: number, derivate?: boolean): number;
 export declare function HLIM(x: number, derivate?: boolean): number;
+export declare function SOFTPLUS(x: number, derivate?: boolean): number;
+export declare function EXP(x: number, derivate?: boolean): number;

dist/src/squash.js

@@ -0,0 +1,37 @@
+// squashing functions
+function LOGISTIC(x, derivate) {
+    if (derivate) {
+        var fx = LOGISTIC(x);
+        return fx * (1 - fx);
+    }
+    return 1 / (1 + Math.exp(-x));
+}
+exports.LOGISTIC = LOGISTIC;
+function TANH(x, derivate) {
+    if (derivate)
+        return 1 - Math.pow(TANH(x), 2);
+    var eP = Math.exp(x);
+    var eN = 1 / eP;
+    return (eP - eN) / (eP + eN);
+}
+exports.TANH = TANH;
+function IDENTITY(x, derivate) {
+    return derivate ? 1 : x;
+}
+exports.IDENTITY = IDENTITY;
+function HLIM(x, derivate) {
+    return derivate ? 1 : +(x > 0);
+}
+exports.HLIM = HLIM;
+function SOFTPLUS(x, derivate) {
+    if (derivate)
+        return 1 - 1 / (1 + Math.exp(x));
+    return Math.log(1 + Math.exp(x));
+}
+exports.SOFTPLUS = SOFTPLUS;
+function EXP(x, derivate) {
+    return Math.exp(x);
+}
+exports.EXP = EXP;
+
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbInNyYy9zcXVhc2gudHMiXSwibmFtZXMiOlsiTE9HSVNUSUMiLCJUQU5IIiwiSURFTlRJVFkiLCJITElNIiwiU09GVFBMVVMiLCJFWFAiXSwibWFwcGluZ3MiOiJBQUVBLEFBRUEsc0JBRnNCO1NBRU4sUUFBUSxDQUFDLENBQVMsRUFBRSxRQUFrQjtJQUNyREEsRUFBRUEsQ0FBQ0EsQ0FBQ0EsUUFBUUEsQ0FBQ0EsQ0FBQ0EsQ0FBQ0E7UUFDZEEsSUFBSUEsRUFBRUEsR0FBR0EsUUFBUUEsQ0FBQ0EsQ0FBQ0EsQ0FBQ0EsQ0FBQ0E7UUFDckJBLE1BQU1BLENBQUNBLEVBQUVBLEdBQUdBLENBQUNBLENBQUNBLEdBQUdBLEVBQUVBLENBQUNBLENBQUNBO0lBQ3RCQSxDQUFDQTtJQUNEQSxNQUFNQSxDQUFDQSxDQUFDQSxHQUFHQSxDQUFDQSxDQUFDQSxHQUFHQSxJQUFJQSxDQUFDQSxHQUFHQSxDQUFDQSxDQUFDQSxDQUFDQSxDQUFDQSxDQUFDQSxDQUFDQTtBQUUvQkEsQ0FBQ0E7QUFQZSxnQkFBUSxHQUFSLFFBT2YsQ0FBQTtBQUVELFNBQWdCLElBQUksQ0FBQyxDQUFTLEVBQUUsUUFBa0I7SUFDakRDLEVBQUVBLENBQUNBLENBQUNBLFFBQVFBLENBQUNBO1FBQ1pBLE1BQU1BLENBQUNBLENBQUNBLEdBQUdBLElBQUlBLENBQUNBLEdBQUdBLENBQUNBLElBQUlBLENBQUNBLENBQUNBLENBQUNBLEVBQUVBLENBQUNBLENBQUNBLENBQUNBO0lBQ2pDQSxJQUFJQSxFQUFFQSxHQUFHQSxJQUFJQSxDQUFDQSxHQUFHQSxDQUFDQSxDQUFDQSxDQUFDQSxDQUFDQTtJQUNyQkEsSUFBSUEsRUFBRUEsR0FBR0EsQ0FBQ0EsR0FBR0EsRUFBRUEsQ0FBQ0E7SUFDaEJBLE1BQU1BLENBQUNBLENBQUNBLEVBQUVBLEdBQUdBLEVBQUVBLENBQUNBLEdBQUdBLENBQUNBLEVBQUVBLEdBQUdBLEVBQUVBLENBQUNBLENBQUNBO0FBQzlCQSxDQUFDQTtBQU5lLFlBQUksR0FBSixJQU1mLENBQUE7QUFFRCxTQUFnQixRQUFRLENBQUMsQ0FBUyxFQUFFLFFBQWtCO0lBQ3JEQyxNQUFNQSxDQUFDQSxRQUFRQSxHQUFHQSxDQUFDQSxHQUFHQSxDQUFDQSxDQUFDQTtBQUN6QkEsQ0FBQ0E7QUFGZSxnQkFBUSxHQUFSLFFBRWYsQ0FBQTtBQUVELFNBQWdCLElBQUksQ0FBQyxDQUFTLEVBQUUsUUFBa0I7SUFDakRDLE1BQU1BLENBQUNBLFFBQVFBLEdBQUdBLENBQUNBLEdBQUdBLENBQUNBLENBQUNBLENBQUNBLEdBQUdBLENBQUNBLENBQUNBLENBQUNBO0FBQ2hDQSxDQUFDQTtBQUZlLFlBQUksR0FBSixJQUVmLENBQUE7QUFFRCxTQUFnQixRQUFRLENBQUMsQ0FBUyxFQUFFLFFBQWtCO0lBQ3JEQyxFQUFFQSxDQUFDQSxDQUFDQSxRQUFRQSxDQUFDQTtRQUNaQSxNQUFNQSxDQUFDQSxDQUFDQSxHQUFHQSxDQUFDQSxHQUFHQSxDQUFDQSxDQUFDQSxHQUFHQSxJQUFJQSxDQUFDQSxHQUFHQSxDQUFDQSxDQUFDQSxDQUFDQSxDQUFDQSxDQUFDQTtJQUNsQ0EsTUFBTUEsQ0FBQ0EsSUFBSUEsQ0FBQ0EsR0FBR0EsQ0FBQ0EsQ0FBQ0EsR0FBR0EsSUFBSUEsQ0FBQ0EsR0FBR0EsQ0FBQ0EsQ0FBQ0EsQ0FBQ0EsQ0FBQ0EsQ0FBQ0E7QUFDbENBLENBQUNBO0FBSmUsZ0JBQVEsR0FBUixRQUlmLENBQUE7QUFFRCxTQUFnQixHQUFHLENBQUMsQ0FBUyxFQUFFLFFBQWtCO0lBQ2hEQyxNQUFNQSxDQUFDQSxJQUFJQSxDQUFDQSxHQUFHQSxDQUFDQSxDQUFDQSxDQUFDQSxDQUFDQTtBQUNwQkEsQ0FBQ0E7QUFGZSxXQUFHLEdBQUgsR0FFZixDQUFBIiwiZmlsZSI6InNyYy9zcXVhc2guanMiLCJzb3VyY2VzQ29udGVudCI6WyJpbXBvcnQgU3luYXB0aWMgPSByZXF1aXJlKCcuL3N5bmFwdGljJyk7XG5cbi8vIHNxdWFzaGluZyBmdW5jdGlvbnNcblxuZXhwb3J0IGZ1bmN0aW9uIExPR0lTVElDKHg6IG51bWJlciwgZGVyaXZhdGU/OiBib29sZWFuKSB7XG5cdGlmIChkZXJpdmF0ZSkge1xuXHRcdHZhciBmeCA9IExPR0lTVElDKHgpO1xuXHRcdHJldHVybiBmeCAqICgxIC0gZngpO1xuXHR9XG5cdHJldHVybiAxIC8gKDEgKyBNYXRoLmV4cCgteCkpO1xuXG59XG5cbmV4cG9ydCBmdW5jdGlvbiBUQU5IKHg6IG51bWJlciwgZGVyaXZhdGU/OiBib29sZWFuKSB7XG5cdGlmIChkZXJpdmF0ZSlcblx0XHRyZXR1cm4gMSAtIE1hdGgucG93KFRBTkgoeCksIDIpO1xuXHR2YXIgZVAgPSBNYXRoLmV4cCh4KTtcblx0dmFyIGVOID0gMSAvIGVQO1xuXHRyZXR1cm4gKGVQIC0gZU4pIC8gKGVQICsgZU4pO1xufVxuXG5leHBvcnQgZnVuY3Rpb24gSURFTlRJVFkoeDogbnVtYmVyLCBkZXJpdmF0ZT86IGJvb2xlYW4pIHtcblx0cmV0dXJuIGRlcml2YXRlID8gMSA6IHg7XG59XG5cbmV4cG9ydCBmdW5jdGlvbiBITElNKHg6IG51bWJlciwgZGVyaXZhdGU/OiBib29sZWFuKSB7XG5cdHJldHVybiBkZXJpdmF0ZSA/IDEgOiArKHggPiAwKTtcbn1cblxuZXhwb3J0IGZ1bmN0aW9uIFNPRlRQTFVTKHg6IG51bWJlciwgZGVyaXZhdGU/OiBib29sZWFuKSB7XG5cdGlmIChkZXJpdmF0ZSlcblx0XHRyZXR1cm4gMSAtIDEgLyAoMSArIE1hdGguZXhwKHgpKTtcblx0cmV0dXJuIE1hdGgubG9nKDEgKyBNYXRoLmV4cCh4KSk7XG59XG5cbmV4cG9ydCBmdW5jdGlvbiBFWFAoeDogbnVtYmVyLCBkZXJpdmF0ZT86IGJvb2xlYW4pIHtcblx0cmV0dXJuIE1hdGguZXhwKHgpO1xufSAiXSwic291cmNlUm9vdCI6Ii9zb3VyY2UvIn0=

dist/src/synaptic.d.ts

@@ -4,6 +4,7 @@ import neuron = require('./neuron');
 import trainer = require('./trainer');
 import architect = require('./architect');
 import squash = require('./squash');
+import utils = require('./utils');
 declare module Synaptic {
     interface Dictionary<T> {
         [id: string]: T;
@@ -21,11 +22,16 @@ declare module Synaptic {
         propagation_sentences?: any[];
         layers?: any;
     }
+    interface INumericArray {
+        [index: number]: number;
+        length: number;
+    }
     var Neuron: typeof neuron.Neuron;
     var Layer: typeof layer.Layer;
     var Network: typeof network.Network;
     var Trainer: typeof trainer.Trainer;
     var Squash: typeof squash;
     var Architect: typeof architect;
+    var Utils: typeof utils.Utils;
 }
 export = Synaptic;

dist/src/trainer.d.ts

@@ -41,6 +41,7 @@ export declare module Trainer {
     }
     var cost: {
         CROSS_ENTROPY: (target: any, output: any) => number;
+        CROSS_ENTROPY_SOFTMAX: (target: any, output: any) => number;
         MSE: (target: any, output: any) => number;
     };
 }

dist/src/utils.d.ts

@@ -0,0 +1,17 @@
+import Synaptic = require('./synaptic');
+export declare class Utils {
+    static transformationMatrixCache: {
+        [index: number]: Float64Array[];
+    };
+    static softMax<T extends Synaptic.INumericArray>(outputArray: T): T;
+    static softMaxDerivative<T extends Synaptic.INumericArray>(outputArray: T): T;
+    static softMaxReinforcement<T extends Synaptic.INumericArray>(array: T, temperature?: number): T;
+    static getCosineSimilarity(arrayA: Synaptic.INumericArray, arrayB: Synaptic.INumericArray): number;
+    static interpolateArray(output_inputA: Synaptic.INumericArray, inputB: Synaptic.INumericArray, g: any): Synaptic.INumericArray;
+    static sharpArray(output: Synaptic.INumericArray, wn: Synaptic.INumericArray, Y: number): void;
+    static scalarShifting(wg: Synaptic.INumericArray, shiftScalar: number): Float64Array;
+    static normalizeShift(shift: Float64Array): void;
+    static vectorInvertedShifting(wg: Float64Array, shiftings: Synaptic.INumericArray): void;
+    static initRandomSoftmaxArray(array: Float64Array): void;
+    static buildCirculantMatrix(length: number, offset?: number): Float64Array[];
+}

dist/synaptic.js

@@ -1,5 +1,5 @@
 (function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);var f=new Error("Cannot find module '"+o+"'");throw f.code="MODULE_NOT_FOUND",f}var l=n[o]={exports:{}};t[o][0].call(l.exports,function(e){var n=t[o][1][e];return s(n?n:e)},l,l.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o<r.length;o++)s(r[o]);return s})({1:[function(require,module,exports){
-var hopfield = require('./architect/hopfield');
+var hopfield = require('./architect/Hopfield');
 var lstm = require('./architect/LSTM');
 var lsm = require('./architect/Liquid');
 var perceptron = require('./architect/Perceptron');
@@ -8,7 +8,54 @@ exports.Liquid = lsm.Liquid;
 exports.Hopfield = hopfield.Hopfield;
 exports.Perceptron = perceptron.Perceptron;
 
-},{"./architect/LSTM":2,"./architect/Liquid":3,"./architect/Perceptron":4,"./architect/hopfield":5}],2:[function(require,module,exports){
+},{"./architect/Hopfield":2,"./architect/LSTM":3,"./architect/Liquid":4,"./architect/Perceptron":5}],2:[function(require,module,exports){
+var __extends = this.__extends || function (d, b) {
+    for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p];
+    function __() { this.constructor = d; }
+    __.prototype = b.prototype;
+    d.prototype = new __();
+};
+var network = require('../network');
+var trainer = require('../trainer');
+var layer = require('../layer');
+var Hopfield = (function (_super) {
+    __extends(Hopfield, _super);
+    function Hopfield(size) {
+        var inputLayer = new layer.Layer(size);
+        var outputLayer = new layer.Layer(size);
+        inputLayer.project(outputLayer, layer.Layer.connectionType.ALL_TO_ALL);
+        _super.call(this, {
+            input: inputLayer,
+            hidden: [],
+            output: outputLayer
+        });
+        this.trainer = new trainer.Trainer(this);
+    }
+    Hopfield.prototype.learn = function (patterns) {
+        var set = [];
+        for (var p in patterns)
+            set.push({
+                input: patterns[p],
+                output: patterns[p]
+            });
+        return this.trainer.train(set, {
+            iterations: 500000,
+            error: .00005,
+            rate: 1
+        });
+    };
+    Hopfield.prototype.feed = function (pattern) {
+        var output = this.activate(pattern);
+        var patterns = [];
+        for (var i in output)
+            patterns[i] = output[i] > .5 ? 1 : 0;
+        return patterns;
+    };
+    return Hopfield;
+})(network.Network);
+exports.Hopfield = Hopfield;
+
+},{"../layer":6,"../network":7,"../trainer":11}],3:[function(require,module,exports){
 var __extends = this.__extends || function (d, b) {
     for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p];
     function __() { this.constructor = d; }
@@ -131,7 +178,7 @@ var LSTM = (function (_super) {
 exports.LSTM = LSTM;
 ;
 
-},{"../layer":6,"../network":7,"../trainer":11}],3:[function(require,module,exports){
+},{"../layer":6,"../network":7,"../trainer":11}],4:[function(require,module,exports){
 var __extends = this.__extends || function (d, b) {
     for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p];
     function __() { this.constructor = d; }
@@ -182,7 +229,7 @@ var Liquid = (function (_super) {
 })(network.Network);
 exports.Liquid = Liquid;
 
-},{"../layer":6,"../network":7,"../trainer":11}],4:[function(require,module,exports){
+},{"../layer":6,"../network":7,"../trainer":11}],5:[function(require,module,exports){
 var __extends = this.__extends || function (d, b) {
     for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p];
     function __() { this.constructor = d; }
@@ -232,53 +279,6 @@ var Perceptron = (function (_super) {
 exports.Perceptron = Perceptron;
 ;
 
-},{"../layer":6,"../network":7,"../trainer":11}],5:[function(require,module,exports){
-var __extends = this.__extends || function (d, b) {
-    for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p];
-    function __() { this.constructor = d; }
-    __.prototype = b.prototype;
-    d.prototype = new __();
-};
-var network = require('../network');
-var trainer = require('../trainer');
-var layer = require('../layer');
-var Hopfield = (function (_super) {
-    __extends(Hopfield, _super);
-    function Hopfield(size) {
-        var inputLayer = new layer.Layer(size);
-        var outputLayer = new layer.Layer(size);
-        inputLayer.project(outputLayer, layer.Layer.connectionType.ALL_TO_ALL);
-        _super.call(this, {
-            input: inputLayer,
-            hidden: [],
-            output: outputLayer
-        });
-        this.trainer = new trainer.Trainer(this);
-    }
-    Hopfield.prototype.learn = function (patterns) {
-        var set = [];
-        for (var p in patterns)
-            set.push({
-                input: patterns[p],
-                output: patterns[p]
-            });
-        return this.trainer.train(set, {
-            iterations: 500000,
-            error: .00005,
-            rate: 1
-        });
-    };
-    Hopfield.prototype.feed = function (pattern) {
-        var output = this.activate(pattern);
-        var patterns = [];
-        for (var i in output)
-            patterns[i] = output[i] > .5 ? 1 : 0;
-        return patterns;
-    };
-    return Hopfield;
-})(network.Network);
-exports.Hopfield = Hopfield;
-
 },{"../layer":6,"../network":7,"../trainer":11}],6:[function(require,module,exports){
 var neuron = require('./neuron');
 var network = require('./network');
@@ -287,6 +287,7 @@ var network = require('./network');
 *******************************************************************************************/
 var Layer = (function () {
     function Layer(size, label) {
+        this.optimizable = true;
         this.list = [];
         this.label = null;
         this.connectedto = [];
@@ -295,6 +296,7 @@ var Layer = (function () {
         this.list = [];
         this.label = label || null;
         this.connectedto = [];
+        this.currentActivation = new Float64Array(size);
         while (size--) {
             var theNeuron = new neuron.Neuron();
             this.list.push(theNeuron);
@@ -302,24 +304,22 @@ var Layer = (function () {
     }
     // activates all the neurons in the layer
     Layer.prototype.activate = function (input) {
-        var activations = [];
+        if (this.currentActivation.length != this.list.length)
+            this.currentActivation = new Float64Array(this.list.length);
+        var activationIndex = 0;
         if (typeof input != 'undefined') {
             if (input.length != this.size)
                 throw "INPUT size and LAYER size must be the same to activate!";
             for (var id in this.list) {
-                var neuron = this.list[id];
-                var activation = neuron.activate(input[id]);
-                activations.push(activation);
+                this.currentActivation[activationIndex++] = this.list[id].activate(input[id]);
             }
         }
         else {
             for (var id in this.list) {
-                var neuron = this.list[id];
-                var activation = neuron.activate();
-                activations.push(activation);
+                this.currentActivation[activationIndex++] = this.list[id].activate();
             }
         }
-        return activations;
+        return this.currentActivation;
     };
     // propagates the error on all the neurons of the layer
     Layer.prototype.propagate = function (rate, target) {
@@ -550,10 +550,9 @@ var Network = (function () {
         if (typeof layers != 'undefined') {
             this.layers = layers || {
                 input: null,
-                hidden: {},
+                hidden: [],
                 output: null
             };
-            this.optimized = null;
         }
     }
     // feed-forward activation of all the layers to produce an ouput
@@ -1072,6 +1071,7 @@ var Squash = require('./squash');
 */
 var Neuron = (function () {
     function Neuron() {
+        this.optimizable = true;
         this.ID = Neuron.uid();
         this.label = null;
         this.connections = {
@@ -1098,8 +1098,7 @@ var Neuron = (function () {
         this.bias = Math.random() * .2 - .1;
         this.derivative = 0;
     }
-    // activate the neuron
-    Neuron.prototype.activate = function (input) {
+    Neuron.prototype.readIncommingConnections = function (input) {
         // activation from enviroment (for input neurons)
         if (typeof input != 'undefined') {
             this.activation = input;
@@ -1120,6 +1119,8 @@ var Neuron = (function () {
         this.activation = this.squash(this.state);
         // f'(s)
         this.derivative = this.squash(this.state, true);
+    };
+    Neuron.prototype.updateTraces = function () {
         // update traces
         var influences = [];
         for (var id in this.trace.extended) {
@@ -1155,6 +1156,11 @@ var Neuron = (function () {
         for (var connection in this.connections.gated) {
             this.connections.gated[connection].gain = this.activation;
         }
+    };
+    // activate the neuron
+    Neuron.prototype.activate = function (input) {
+        this.readIncommingConnections(input);
+        this.updateTraces();
         return this.activation;
     };
     // back-propagate the error
@@ -1740,10 +1746,11 @@ var Neuron;
 },{"./squash":9}],9:[function(require,module,exports){
 // squashing functions
 function LOGISTIC(x, derivate) {
-    if (!derivate)
-        return 1 / (1 + Math.exp(-x));
-    var fx = LOGISTIC(x);
-    return fx * (1 - fx);
+    if (derivate) {
+        var fx = LOGISTIC(x);
+        return fx * (1 - fx);
+    }
+    return 1 / (1 + Math.exp(-x));
 }
 exports.LOGISTIC = LOGISTIC;
 function TANH(x, derivate) {
@@ -1762,6 +1769,16 @@ function HLIM(x, derivate) {
     return derivate ? 1 : +(x > 0);
 }
 exports.HLIM = HLIM;
+function SOFTPLUS(x, derivate) {
+    if (derivate)
+        return 1 - 1 / (1 + Math.exp(x));
+    return Math.log(1 + Math.exp(x));
+}
+exports.SOFTPLUS = SOFTPLUS;
+function EXP(x, derivate) {
+    return Math.exp(x);
+}
+exports.EXP = EXP;
 
 },{}],10:[function(require,module,exports){
 /*
@@ -1795,9 +1812,10 @@ var neuron = require('./neuron');
 var trainer = require('./trainer');
 var architect = require('./architect');
 var squash = require('./squash');
+var utils = require('./utils');
 var Synaptic;
 (function (Synaptic) {
-    var oldSynaptic = window && window['Synaptic'];
+    var oldSynaptic = typeof window != "undefined" && window && window['Synaptic'];
     function ninja() {
         window['synaptic'] = oldSynaptic;
         return Synaptic;
@@ -1809,12 +1827,13 @@ var Synaptic;
     Synaptic.Trainer = trainer.Trainer;
     Synaptic.Squash = squash;
     Synaptic.Architect = architect;
+    Synaptic.Utils = utils.Utils;
 })(Synaptic || (Synaptic = {}));
 if (typeof window != "undefined")
     window['synaptic'] = Synaptic;
 module.exports = Synaptic;
 
-},{"./architect":1,"./layer":6,"./network":7,"./neuron":8,"./squash":9,"./trainer":11}],11:[function(require,module,exports){
+},{"./architect":1,"./layer":6,"./network":7,"./neuron":8,"./squash":9,"./trainer":11,"./utils":12}],11:[function(require,module,exports){
 /*******************************************************************************************
                                         TRAINER
 *******************************************************************************************/
@@ -1885,15 +1904,15 @@ var Trainer = (function () {
             iterations++;
             error /= set.length;
             if (options) {
-                if (this.schedule && this.schedule.every && iterations %
-                    this.schedule.every == 0)
+                if (this.schedule && this.schedule.every && iterations % this.schedule.every == 0) {
                     abort_training = this.schedule.do({
                         error: error,
                         iterations: iterations,
                         rate: currentRate
                     });
+                }
                 else if (options.log && iterations % options.log == 0) {
-                    console.log('iterations', iterations, 'error', error, 'rate', currentRate);
+                    console.log('iterations', iterations, 'error', error, 'rate', currentRate, 'T:', target, 'O:', output);
                 }
                 ;
                 if (options.shuffle)
@@ -2143,7 +2162,7 @@ var Trainer = (function () {
             // log
             if (log && trial % log == 0)
                 console.log("iterations:", trial, " success:", success, " correct:", correct, " time:", Date.now() - start, " error:", error);
-            if (schedule.do && schedule.every && trial % schedule.every == 0)
+            if (schedule.do && schedule.every && trial % schedule.every == 0) {
                 schedule.do({
                     iterations: trial,
                     success: success,
@@ -2151,6 +2170,7 @@ var Trainer = (function () {
                     time: Date.now() - start,
                     correct: correct
                 });
+            }
         }
         return {
             iterations: trial,
@@ -2335,6 +2355,12 @@ var Trainer;
                 crossentropy -= (target[i] * Math.log(output[i] + 1e-15)) + ((1 - target[i]) * Math.log((1 + 1e-15) - output[i])); // +1e-15 is a tiny push away to avoid Math.log(0)
             return crossentropy;
         },
+        CROSS_ENTROPY_SOFTMAX: function (target, output) {
+            var crossentropy = 0;
+            for (var i in output)
+                crossentropy -= target[i] * Math.log(output[i] + 1e-15);
+            return crossentropy;
+        },
         MSE: function (target, output) {
             var mse = 0;
             for (var i in output)
@@ -2344,5 +2370,182 @@ var Trainer;
     };
 })(Trainer = exports.Trainer || (exports.Trainer = {}));
 
+},{}],12:[function(require,module,exports){
+var Utils = (function () {
+    function Utils() {
+    }
+    Utils.softMax = function (outputArray) {
+        // for all i ∈ array
+        // sum = ∑ array[n]^e
+        // i = î^e / sum
+        // where the result ∑ array[0..n] = 1
+        if (!outputArray.length)
+            return outputArray;
+        var sum = 0;
+        var Amax = outputArray[0];
+        for (var i = 0; i < outputArray.length; i++) {
+            if (outputArray[i] < Amax)
+                Amax = outputArray[i];
+        }
+        // sum = ∑ array[n]^e
+        for (var i = 0; i < outputArray.length; i++) {
+            outputArray[i] = Math.exp(outputArray[i] - Amax);
+            sum += outputArray[i];
+        }
+        for (var i = 0; i < outputArray.length; i++)
+            outputArray[i] /= sum;
+        return outputArray;
+    };
+    Utils.softMaxDerivative = function (outputArray) {
+        // http://sysmagazine.com/posts/155235/
+        if (!outputArray.length)
+            return outputArray;
+        var sum = 0;
+        // sum = ∑ array[n]^e
+        for (var i = 0; i < outputArray.length; i++) {
+            outputArray[i] = Math.exp(outputArray[i]);
+            sum += outputArray[i];
+        }
+        for (var i = 0; i < outputArray.length; i++) {
+            var t = outputArray[i] /= sum;
+            outputArray[i] = t * (1 - t);
+        }
+        return outputArray;
+    };
+    Utils.softMaxReinforcement = function (array, temperature) {
+        // Reinforcement learning
+        if (temperature === void 0) { temperature = 1; }
+        if (!array.length)
+            return array;
+        temperature = temperature || 1;
+        var sum = 0;
+        // sum = ∑ array[n]^e
+        for (var i = 0; i < array.length; i++) {
+            array[i] = Math.exp(array[i] / temperature);
+            sum += array[i];
+        }
+        if (sum != 0) {
+            for (var i = 0; i < array.length; i++)
+                array[i] /= sum;
+        }
+        else {
+            var div = 1 / array.length;
+            for (var i = 0; i < array.length; i++)
+                array[i] = div;
+        }
+        return array;
+    };
+    Utils.getCosineSimilarity = function (arrayA, arrayB) {
+        // http://en.wikipedia.org/wiki/Cosine_similarity
+        var dotPr = 0;
+        var acumA = 0, acumB = 0;
+        for (var i = 0; i < arrayA.length; i++) {
+            dotPr += arrayA[i] * arrayB[i];
+            acumA += arrayA[i] * arrayA[i];
+            acumB += arrayB[i] * arrayB[i];
+        }
+        return dotPr / (Math.sqrt(acumA) * Math.sqrt(acumB) + .00005);
+    };
+    Utils.interpolateArray = function (output_inputA, inputB, g) {
+        // 3.3.2 focus by location (7)
+        var gInverted = 1 - g;
+        for (var i = 0; i < output_inputA.length; i++)
+            output_inputA[i] = output_inputA[i] * g + gInverted * inputB[i];
+        return output_inputA;
+    };
+    // w_sharpWn
+    Utils.sharpArray = function (output, wn, Y) {
+        // 3.3.2 (9)
+        var sum = 0;
+        // ∀ a ∈ wn → a = a^Y
+        // sum = ∑ a^Y 
+        for (var i = 0; i < wn.length; i++) {
+            wn[i] = Math.pow(wn[i], Y);
+            sum += wn[i];
+        }
+        // ∀ a ∈ wn → a = a^Y / sum
+        if (sum != 0) {
+            for (var i = 0; i < wn.length; i++)
+                output[i] = wn[i] / sum;
+        }
+        else {
+            var div = 1 / wn.length;
+            for (var i = 0; i < wn.length; i++)
+                output[i] = div;
+        }
+    };
+    //wn_shift
+    Utils.scalarShifting = function (wg, shiftScalar) {
+        // w~ 3.3.2 (8)
+        var shiftings = new Float64Array(wg.length);
+        var wn = new Float64Array(wg.length);
+        var intPart = shiftScalar | 0;
+        var decimalPart = shiftScalar - intPart;
+        shiftings[intPart % shiftings.length] = 1 - decimalPart;
+        shiftings[(intPart + 1) % shiftings.length] = decimalPart;
+        for (var i = 0; i < wn.length; i++) {
+            var acum = 0;
+            for (var j = 0; j < wn.length; j++) {
+                if ((i - j) < 0)
+                    acum += wg[j] * shiftings[shiftings.length - Math.abs(i - j)];
+                else
+                    acum += wg[j] * shiftings[(i - j) % shiftings.length];
+            }
+            wn[i] = acum;
+        }
+        return wn;
+    };
+    Utils.normalizeShift = function (shift) {
+        var sum = 0;
+        for (var i = 0; i < shift.length; i++) {
+            sum += shift[i];
+        }
+        for (var j = 0; j < shift.length; j++) {
+            shift[j] /= sum;
+        }
+    };
+    Utils.vectorInvertedShifting = function (wg, shiftings) {
+        // w~ 3.3.2 (8)
+        var ret = new Float64Array(wg.length);
+        var corrimientoIndex = -((shiftings.length - 1) / 2) | 0;
+        var circulantMatrix = Utils.transformationMatrixCache[wg.length] || (Utils.transformationMatrixCache[wg.length] = Utils.buildCirculantMatrix(wg.length));
+        for (var i = 0; i < wg.length; i++) {
+            for (var x = 0; x < wg.length; x++) {
+                var tmp = 0;
+                for (var shift = 0; shift < shiftings.length; shift++) {
+                    var matRow = i - x + corrimientoIndex + shift;
+                    while (matRow < 0)
+                        matRow += wg.length;
+                    matRow %= wg.length;
+                    tmp += wg[circulantMatrix[x][matRow]] * shiftings[shift];
+                }
+                ret[i] = tmp;
+            }
+        }
+        wg.set(ret);
+    };
+    Utils.initRandomSoftmaxArray = function (array) {
+        for (var i = 0; i < array.length; i++) {
+            array[i] = Math.random();
+        }
+        Utils.softMax(array);
+    };
+    Utils.buildCirculantMatrix = function (length, offset) {
+        if (offset === void 0) { offset = 0; }
+        var ret = [];
+        for (var i = 0; i < length; i++) {
+            var arr = new Float64Array(length);
+            ret.push(arr);
+            for (var n = 0; n < length; n++) {
+                arr[n] = ((i + n) % length);
+            }
+        }
+        return ret;
+    };
+    Utils.transformationMatrixCache = {};
+    return Utils;
+})();
+exports.Utils = Utils;
+
 },{}]},{},[10]);
 var synaptic = synaptic || Synaptic;var Neuron = synaptic.Neuron, Layer = synaptic.Layer, Network = synaptic.Network, Trainer = synaptic.Trainer, Architect = synaptic.Architect;

gulpfile.js

@@ -30,8 +30,8 @@ gulp.task('node', function () {
         .pipe(ts(tsProject));
     
     return merge2([
-        tsResult.js.pipe(sm.write()).pipe(gulp.dest('./node-dist')),
-        tsResult.dts.pipe(gulp.dest('./node-dist'))
+        tsResult.js.pipe(sm.write()).pipe(gulp.dest('./dist')),
+        tsResult.dts.pipe(gulp.dest('./dist'))
     ]);
 });
 
@@ -62,7 +62,7 @@ gulp.task('min', function () {
 });
 
 // build source into /dist with sourcemaps for debugging
-gulp.task('debug', ['tsc'], function () {
+gulp.task('debug', function () {
   return browserify({ debug: true })
     .add('./src/synaptic.ts')
     .plugin('tsify')
@@ -79,6 +79,13 @@ gulp.task('test', ['node'], function () {
         .pipe(mocha());
 });
 
+// run all the tests with mocha
+gulp.task('test-ntm', ['node'], function () {
+    return gulp.src('test/ntm.js', {read: false})
+        .pipe(mocha());
+});
+
+
 // watch for changed and re-build (debug)
 gulp.task('dev', function () {
    gulp.watch('./src/*.ts', ['debug']);

node-dist/src/architect.d.ts

@@ -1,8 +0,0 @@
-import hopfield = require('./architect/Hopfield');
-import lstm = require('./architect/LSTM');
-import lsm = require('./architect/Liquid');
-import perceptron = require('./architect/Perceptron');
-export declare var LSTM: typeof lstm.LSTM;
-export declare var Liquid: typeof lsm.Liquid;
-export declare var Hopfield: typeof hopfield.Hopfield;
-export declare var Perceptron: typeof perceptron.Perceptron;

node-dist/src/architect/Hopfield.d.ts

@@ -1,12 +0,0 @@
-import network = require('../network');
-import trainer = require('../trainer');
-export declare class Hopfield extends network.Network {
-    trainer: trainer.Trainer;
-    constructor(size: number);
-    learn(patterns: any): {
-        error: number;
-        iterations: number;
-        time: number;
-    };
-    feed(pattern: any): any[];
-}

node-dist/src/architect/LSTM.d.ts

@@ -1,6 +0,0 @@
-import network = require('../network');
-import trainer = require('../trainer');
-export declare class LSTM extends network.Network {
-    trainer: trainer.Trainer;
-    constructor(...args: any[]);
-}

node-dist/src/architect/Liquid.d.ts

@@ -1,6 +0,0 @@
-import network = require('../network');
-import trainer = require('../trainer');
-export declare class Liquid extends network.Network {
-    trainer: trainer.Trainer;
-    constructor(inputs: any, hidden: any, outputs: any, connections: any, gates: any);
-}

node-dist/src/architect/Perceptron.d.ts

@@ -1,6 +0,0 @@
-import network = require('../network');
-import trainer = require('../trainer');
-export declare class Perceptron extends network.Network {
-    trainer: trainer.Trainer;
-    constructor(...args: number[]);
-}

node-dist/src/layer.d.ts

@@ -1,49 +0,0 @@
-import neuron = require('./neuron');
-import Synaptic = require('./synaptic');
-/*******************************************************************************************
-                                            LAYER
-*******************************************************************************************/
-export declare class Layer {
-    list: neuron.Neuron[];
-    label: string;
-    connectedto: any[];
-    size: number;
-    constructor(size: number, label?: string);
-    activate(input: any): any[];
-    propagate(rate: any, target: any): void;
-    project(layer: any, type?: any, weights?: any): Layer.LayerConnection;
-    gate(connection: any, type: any): void;
-    selfconnected(): boolean;
-    connected(layer: any): string;
-    clear(): void;
-    reset(): void;
-    neurons(): neuron.Neuron[];
-    add(neuron: any): void;
-    set(options: any): Layer;
-}
-export declare module Layer {
-    var layerQty: number;
-    function uid(): number;
-    var connectionType: {
-        ALL_TO_ALL: string;
-        ONE_TO_ONE: string;
-        ALL_TO_ELSE: string;
-    };
-    var gateType: {
-        INPUT: string;
-        OUTPUT: string;
-        ONE_TO_ONE: string;
-    };
-    class LayerConnection {
-        ID: number;
-        from: Layer;
-        to: Layer;
-        selfconnection: boolean;
-        type: string;
-        connections: Synaptic.Dictionary<neuron.Neuron.Connection>;
-        list: neuron.Neuron.Connection[];
-        size: number;
-        gatedfrom: any[];
-        constructor(fromLayer: any, toLayer: any, type: any, weights: any);
-    }
-}

node-dist/src/network.d.ts

@@ -1,41 +0,0 @@
-import _neuron = require('./neuron');
-export declare class Network {
-    optimized: any;
-    layers: {
-        input: any;
-        hidden: {};
-        output: any;
-    };
-    constructor(layers: any);
-    activate(input: any): any;
-    propagate(rate: number, target?: any): void;
-    project(unit: any, type: any, weights: any): any;
-    gate(connection: any, type: any): void;
-    clear(): void;
-    reset(): void;
-    optimize(): void;
-    restore(): void;
-    neurons(): Network.INetworkNeuron[];
-    inputs(): number;
-    outputs(): number;
-    set(layers: any): void;
-    setOptimize(bool: any): void;
-    toJSON(ignoreTraces: any): {
-        neurons: any[];
-        connections: any[];
-    };
-    toDot(edgeconnection: any): {
-        code: string;
-        link: string;
-    };
-    standalone(): any;
-    worker(): Worker;
-    clone(ignoreTraces: any): Network;
-    static fromJSON(json: any): Network;
-}
-export declare module Network {
-    interface INetworkNeuron {
-        neuron: _neuron.Neuron;
-        layer: string;
-    }
-}

node-dist/src/neuron.d.ts

@@ -1,67 +0,0 @@
-/// <reference path="synaptic.d.ts" />
-import Synaptic = require('./synaptic');
-import Squash = require('./squash');
-/******************************************************************************************
-                                         NEURON
-*******************************************************************************************/
-export declare class Neuron {
-    ID: number;
-    label: any;
-    connections: Neuron.INeuronConnections;
-    error: {
-        responsibility: number;
-        projected: number;
-        gated: number;
-    };
-    trace: {
-        elegibility: {};
-        extended: {};
-        influences: {};
-    };
-    state: number;
-    old: number;
-    activation: number;
-    selfconnection: Neuron.Connection;
-    squash: typeof Squash.LOGISTIC;
-    neighboors: {};
-    bias: number;
-    derivative: number;
-    activate(input?: number): number;
-    propagate(rate: number, target?: number): void;
-    project(neuron: any, weight?: number): Neuron.Connection;
-    gate(connection: any): void;
-    selfconnected(): boolean;
-    connected(neuron: any): {
-        type: string;
-        connection: Neuron.Connection;
-    };
-    clear(): void;
-    reset(): void;
-    optimize(optimized: any, layer: any): Synaptic.ICompiledParameters;
-}
-export declare module Neuron {
-    interface INeuronConnections {
-        inputs: Synaptic.Dictionary<Neuron.Connection>;
-        projected: {};
-        gated: {};
-    }
-    class Connection {
-        ID: number;
-        from: any;
-        to: any;
-        gain: number;
-        weight: number;
-        gater: any;
-        constructor(from: any, to: any, weight?: number);
-    }
-    var neuronQty: number;
-    function uid(): number;
-    function quantity(): {
-        neurons: number;
-        connections: number;
-    };
-}
-export declare module Neuron.Connection {
-    var connectionQty: number;
-    function uid(): number;
-}

node-dist/src/squash.d.ts

@@ -1,4 +0,0 @@
-export declare function LOGISTIC(x: number, derivate?: boolean): any;
-export declare function TANH(x: number, derivate?: boolean): any;
-export declare function IDENTITY(x: number, derivate?: boolean): number;
-export declare function HLIM(x: number, derivate?: boolean): number;

node-dist/src/squash.js

@@ -1,26 +0,0 @@
-// squashing functions
-function LOGISTIC(x, derivate) {
-    if (!derivate)
-        return 1 / (1 + Math.exp(-x));
-    var fx = LOGISTIC(x);
-    return fx * (1 - fx);
-}
-exports.LOGISTIC = LOGISTIC;
-function TANH(x, derivate) {
-    if (derivate)
-        return 1 - Math.pow(TANH(x), 2);
-    var eP = Math.exp(x);
-    var eN = 1 / eP;
-    return (eP - eN) / (eP + eN);
-}
-exports.TANH = TANH;
-function IDENTITY(x, derivate) {
-    return derivate ? 1 : x;
-}
-exports.IDENTITY = IDENTITY;
-function HLIM(x, derivate) {
-    return derivate ? 1 : +(x > 0);
-}
-exports.HLIM = HLIM;
-
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbInNyYy9zcXVhc2gudHMiXSwibmFtZXMiOlsiTE9HSVNUSUMiLCJUQU5IIiwiSURFTlRJVFkiLCJITElNIl0sIm1hcHBpbmdzIjoiQUFFQSxBQUVBLHNCQUZzQjtTQUVOLFFBQVEsQ0FBQyxDQUFTLEVBQUUsUUFBa0I7SUFDckRBLEVBQUVBLENBQUNBLENBQUNBLENBQUNBLFFBQVFBLENBQUNBO1FBQ2JBLE1BQU1BLENBQUNBLENBQUNBLEdBQUdBLENBQUNBLENBQUNBLEdBQUdBLElBQUlBLENBQUNBLEdBQUdBLENBQUNBLENBQUNBLENBQUNBLENBQUNBLENBQUNBLENBQUNBO0lBQy9CQSxJQUFJQSxFQUFFQSxHQUFHQSxRQUFRQSxDQUFDQSxDQUFDQSxDQUFDQSxDQUFDQTtJQUNyQkEsTUFBTUEsQ0FBQ0EsRUFBRUEsR0FBR0EsQ0FBQ0EsQ0FBQ0EsR0FBR0EsRUFBRUEsQ0FBQ0EsQ0FBQ0E7QUFDdEJBLENBQUNBO0FBTGUsZ0JBQVEsR0FBUixRQUtmLENBQUE7QUFFRCxTQUFnQixJQUFJLENBQUMsQ0FBUyxFQUFFLFFBQWtCO0lBQ2pEQyxFQUFFQSxDQUFDQSxDQUFDQSxRQUFRQSxDQUFDQTtRQUNaQSxNQUFNQSxDQUFDQSxDQUFDQSxHQUFHQSxJQUFJQSxDQUFDQSxHQUFHQSxDQUFDQSxJQUFJQSxDQUFDQSxDQUFDQSxDQUFDQSxFQUFFQSxDQUFDQSxDQUFDQSxDQUFDQTtJQUNqQ0EsSUFBSUEsRUFBRUEsR0FBR0EsSUFBSUEsQ0FBQ0EsR0FBR0EsQ0FBQ0EsQ0FBQ0EsQ0FBQ0EsQ0FBQ0E7SUFDckJBLElBQUlBLEVBQUVBLEdBQUdBLENBQUNBLEdBQUdBLEVBQUVBLENBQUNBO0lBQ2hCQSxNQUFNQSxDQUFDQSxDQUFDQSxFQUFFQSxHQUFHQSxFQUFFQSxDQUFDQSxHQUFHQSxDQUFDQSxFQUFFQSxHQUFHQSxFQUFFQSxDQUFDQSxDQUFDQTtBQUM5QkEsQ0FBQ0E7QUFOZSxZQUFJLEdBQUosSUFNZixDQUFBO0FBRUQsU0FBZ0IsUUFBUSxDQUFDLENBQVMsRUFBRSxRQUFrQjtJQUNyREMsTUFBTUEsQ0FBQ0EsUUFBUUEsR0FBR0EsQ0FBQ0EsR0FBR0EsQ0FBQ0EsQ0FBQ0E7QUFDekJBLENBQUNBO0FBRmUsZ0JBQVEsR0FBUixRQUVmLENBQUE7QUFFRCxTQUFnQixJQUFJLENBQUMsQ0FBUyxFQUFFLFFBQWtCO0lBQ2pEQyxNQUFNQSxDQUFDQSxRQUFRQSxHQUFHQSxDQUFDQSxHQUFHQSxDQUFDQSxDQUFDQSxDQUFDQSxHQUFHQSxDQUFDQSxDQUFDQSxDQUFDQTtBQUNoQ0EsQ0FBQ0E7QUFGZSxZQUFJLEdBQUosSUFFZixDQUFBIiwiZmlsZSI6InNyYy9zcXVhc2guanMiLCJzb3VyY2VzQ29udGVudCI6WyJpbXBvcnQgU3luYXB0aWMgPSByZXF1aXJlKCcuL3N5bmFwdGljJyk7XG5cbi8vIHNxdWFzaGluZyBmdW5jdGlvbnNcblxuZXhwb3J0IGZ1bmN0aW9uIExPR0lTVElDKHg6IG51bWJlciwgZGVyaXZhdGU/OiBib29sZWFuKSB7XG5cdGlmICghZGVyaXZhdGUpXG5cdFx0cmV0dXJuIDEgLyAoMSArIE1hdGguZXhwKC14KSk7XG5cdHZhciBmeCA9IExPR0lTVElDKHgpO1xuXHRyZXR1cm4gZnggKiAoMSAtIGZ4KTtcbn1cblxuZXhwb3J0IGZ1bmN0aW9uIFRBTkgoeDogbnVtYmVyLCBkZXJpdmF0ZT86IGJvb2xlYW4pIHtcblx0aWYgKGRlcml2YXRlKVxuXHRcdHJldHVybiAxIC0gTWF0aC5wb3coVEFOSCh4KSwgMik7XG5cdHZhciBlUCA9IE1hdGguZXhwKHgpO1xuXHR2YXIgZU4gPSAxIC8gZVA7XG5cdHJldHVybiAoZVAgLSBlTikgLyAoZVAgKyBlTik7XG59XG5cbmV4cG9ydCBmdW5jdGlvbiBJREVOVElUWSh4OiBudW1iZXIsIGRlcml2YXRlPzogYm9vbGVhbikge1xuXHRyZXR1cm4gZGVyaXZhdGUgPyAxIDogeDtcbn1cblxuZXhwb3J0IGZ1bmN0aW9uIEhMSU0oeDogbnVtYmVyLCBkZXJpdmF0ZT86IGJvb2xlYW4pIHtcblx0cmV0dXJuIGRlcml2YXRlID8gMSA6ICsoeCA+IDApO1xufVxuIl0sInNvdXJjZVJvb3QiOiIvc291cmNlLyJ9

node-dist/src/synaptic.d.ts

@@ -1,31 +0,0 @@
-import network = require('./network');
-import layer = require('./layer');
-import neuron = require('./neuron');
-import trainer = require('./trainer');
-import architect = require('./architect');
-import squash = require('./squash');
-declare module Synaptic {
-    interface Dictionary<T> {
-        [id: string]: T;
-    }
-    function ninja(): typeof Synaptic;
-    interface ICompiledParameters {
-        memory?: any;
-        neurons?: number;
-        inputs?: any[];
-        outputs?: any[];
-        targets?: any[];
-        variables?: any;
-        activation_sentences?: any[];
-        trace_sentences?: any[];
-        propagation_sentences?: any[];
-        layers?: any;
-    }
-    var Neuron: typeof neuron.Neuron;
-    var Layer: typeof layer.Layer;
-    var Network: typeof network.Network;
-    var Trainer: typeof trainer.Trainer;
-    var Squash: typeof squash;
-    var Architect: typeof architect;
-}
-export = Synaptic;

node-dist/src/synaptic.js

@@ -1,51 +0,0 @@
-/*
-********************************************************************************************
-                                         SYNAPTIC
-********************************************************************************************
-
-Synaptic is a javascript neural network library for node.js and the browser, its generalized
-algorithm is architecture-free, so you can build and train basically any type of first order
-or even second order neural network architectures.
-
-http://en.wikipedia.org/wiki/Recurrent_neural_network#Second_Order_Recurrent_Neural_Network
-
-The library includes a few built-in architectures like multilayer perceptrons, multilayer
-long-short term memory networks (LSTM) or liquid state machines, and a trainer capable of
-training any given network, and includes built-in training tasks/tests like solving an XOR,
-passing a Distracted Sequence Recall test or an Embeded Reber Grammar test.
-
-The algorithm implemented by this library has been taken from Derek D. Monner's paper:
-
-A generalized LSTM-like training algorithm for second-order recurrent neural networks
-http://www.overcomplete.net/papers/nn2012.pdf
-
-There are references to the equations in that paper commented through the source code.
-
-
-********************************************************************************************/
-var network = require('./network');
-var layer = require('./layer');
-var neuron = require('./neuron');
-var trainer = require('./trainer');
-var architect = require('./architect');
-var squash = require('./squash');
-var Synaptic;
-(function (Synaptic) {
-    var oldSynaptic = typeof window != "undefined" && window && window['Synaptic'];
-    function ninja() {
-        window['synaptic'] = oldSynaptic;
-        return Synaptic;
-    }
-    Synaptic.ninja = ninja;
-    Synaptic.Neuron = neuron.Neuron;
-    Synaptic.Layer = layer.Layer;
-    Synaptic.Network = network.Network;
-    Synaptic.Trainer = trainer.Trainer;
-    Synaptic.Squash = squash;
-    Synaptic.Architect = architect;
-})(Synaptic || (Synaptic = {}));
-if (typeof window != "undefined")
-    window['synaptic'] = Synaptic;
-module.exports = Synaptic;
-
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbInNyYy9zeW5hcHRpYy50cyJdLCJuYW1lcyI6WyJTeW5hcHRpYyIsIlN5bmFwdGljLm5pbmphIl0sIm1hcHBpbmdzIjoiQUFBQTs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7OzZGQXdCNkY7QUFJN0YsSUFBTyxPQUFPLFdBQVcsV0FBVyxDQUFDLENBQUM7QUFDdEMsSUFBTyxLQUFLLFdBQVcsU0FBUyxDQUFDLENBQUM7QUFDbEMsSUFBTyxNQUFNLFdBQVcsVUFBVSxDQUFDLENBQUM7QUFDcEMsSUFBTyxPQUFPLFdBQVcsV0FBVyxDQUFDLENBQUM7QUFDdEMsSUFBTyxTQUFTLFdBQVcsYUFBYSxDQUFDLENBQUM7QUFDMUMsSUFBTyxNQUFNLFdBQVcsVUFBVSxDQUFDLENBQUM7QUFJcEMsSUFBTyxRQUFRLENBK0JkO0FBL0JELFdBQU8sUUFBUSxFQUFDLENBQUM7SUFLaEJBLElBQUlBLFdBQVdBLEdBQUdBLE9BQU9BLE1BQU1BLElBQUlBLFdBQVdBLElBQUlBLE1BQU1BLElBQUlBLE1BQU1BLENBQUNBLFVBQVVBLENBQUNBLENBQUNBO0lBRS9FQSxTQUFnQkEsS0FBS0E7UUFDaEJDLE1BQU1BLENBQUNBLFVBQVVBLENBQUNBLEdBQUdBLFdBQVdBLENBQUNBO1FBQ2pDQSxNQUFNQSxDQUFDQSxRQUFRQSxDQUFDQTtJQUNyQkEsQ0FBQ0E7SUFIZUQsY0FBS0EsR0FBTEEsS0FHZkEsQ0FBQUE7SUFlVUEsZUFBTUEsR0FBR0EsTUFBTUEsQ0FBQ0EsTUFBTUEsQ0FBQ0E7SUFDdkJBLGNBQUtBLEdBQUdBLEtBQUtBLENBQUNBLEtBQUtBLENBQUNBO0lBQ3BCQSxnQkFBT0EsR0FBR0EsT0FBT0EsQ0FBQ0EsT0FBT0EsQ0FBQ0E7SUFDMUJBLGdCQUFPQSxHQUFHQSxPQUFPQSxDQUFDQSxPQUFPQSxDQUFDQTtJQUMxQkEsZUFBTUEsR0FBR0EsTUFBTUEsQ0FBQ0E7SUFDaEJBLGtCQUFTQSxHQUFHQSxTQUFTQSxDQUFDQTtBQUNsQ0EsQ0FBQ0EsRUEvQk0sUUFBUSxLQUFSLFFBQVEsUUErQmQ7QUFJRCxFQUFFLENBQUEsQ0FBQyxPQUFPLE1BQU0sSUFBSSxXQUFXLENBQUM7SUFDL0IsTUFBTSxDQUFDLFVBQVUsQ0FBQyxHQUFHLFFBQVEsQ0FBQztBQUgvQixpQkFBUyxRQUFRLENBQUMiLCJmaWxlIjoic3JjL3N5bmFwdGljLmpzIiwic291cmNlc0NvbnRlbnQiOlsiLypcbioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqXG4gICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFNZTkFQVElDXG4qKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKlxuXG5TeW5hcHRpYyBpcyBhIGphdmFzY3JpcHQgbmV1cmFsIG5ldHdvcmsgbGlicmFyeSBmb3Igbm9kZS5qcyBhbmQgdGhlIGJyb3dzZXIsIGl0cyBnZW5lcmFsaXplZFxuYWxnb3JpdGhtIGlzIGFyY2hpdGVjdHVyZS1mcmVlLCBzbyB5b3UgY2FuIGJ1aWxkIGFuZCB0cmFpbiBiYXNpY2FsbHkgYW55IHR5cGUgb2YgZmlyc3Qgb3JkZXJcbm9yIGV2ZW4gc2Vjb25kIG9yZGVyIG5ldXJhbCBuZXR3b3JrIGFyY2hpdGVjdHVyZXMuXG5cbmh0dHA6Ly9lbi53aWtpcGVkaWEub3JnL3dpa2kvUmVjdXJyZW50X25ldXJhbF9uZXR3b3JrI1NlY29uZF9PcmRlcl9SZWN1cnJlbnRfTmV1cmFsX05ldHdvcmtcblxuVGhlIGxpYnJhcnkgaW5jbHVkZXMgYSBmZXcgYnVpbHQtaW4gYXJjaGl0ZWN0dXJlcyBsaWtlIG11bHRpbGF5ZXIgcGVyY2VwdHJvbnMsIG11bHRpbGF5ZXJcbmxvbmctc2hvcnQgdGVybSBtZW1vcnkgbmV0d29ya3MgKExTVE0pIG9yIGxpcXVpZCBzdGF0ZSBtYWNoaW5lcywgYW5kIGEgdHJhaW5lciBjYXBhYmxlIG9mXG50cmFpbmluZyBhbnkgZ2l2ZW4gbmV0d29yaywgYW5kIGluY2x1ZGVzIGJ1aWx0LWluIHRyYWluaW5nIHRhc2tzL3Rlc3RzIGxpa2Ugc29sdmluZyBhbiBYT1IsXG5wYXNzaW5nIGEgRGlzdHJhY3RlZCBTZXF1ZW5jZSBSZWNhbGwgdGVzdCBvciBhbiBFbWJlZGVkIFJlYmVyIEdyYW1tYXIgdGVzdC5cblxuVGhlIGFsZ29yaXRobSBpbXBsZW1lbnRlZCBieSB0aGlzIGxpYnJhcnkgaGFzIGJlZW4gdGFrZW4gZnJvbSBEZXJlayBELiBNb25uZXIncyBwYXBlcjpcblxuQSBnZW5lcmFsaXplZCBMU1RNLWxpa2UgdHJhaW5pbmcgYWxnb3JpdGhtIGZvciBzZWNvbmQtb3JkZXIgcmVjdXJyZW50IG5ldXJhbCBuZXR3b3Jrc1xuaHR0cDovL3d3dy5vdmVyY29tcGxldGUubmV0L3BhcGVycy9ubjIwMTIucGRmXG5cblRoZXJlIGFyZSByZWZlcmVuY2VzIHRvIHRoZSBlcXVhdGlvbnMgaW4gdGhhdCBwYXBlciBjb21tZW50ZWQgdGhyb3VnaCB0aGUgc291cmNlIGNvZGUuXG5cblxuKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKiovXG5cblxuXG5pbXBvcnQgbmV0d29yayA9IHJlcXVpcmUoJy4vbmV0d29yaycpO1xuaW1wb3J0IGxheWVyID0gcmVxdWlyZSgnLi9sYXllcicpO1xuaW1wb3J0IG5ldXJvbiA9IHJlcXVpcmUoJy4vbmV1cm9uJyk7XG5pbXBvcnQgdHJhaW5lciA9IHJlcXVpcmUoJy4vdHJhaW5lcicpO1xuaW1wb3J0IGFyY2hpdGVjdCA9IHJlcXVpcmUoJy4vYXJjaGl0ZWN0Jyk7XG5pbXBvcnQgc3F1YXNoID0gcmVxdWlyZSgnLi9zcXVhc2gnKTtcblxuZGVjbGFyZSB2YXIgd2luZG93O1xuXG5tb2R1bGUgU3luYXB0aWMge1xuXHRleHBvcnQgaW50ZXJmYWNlIERpY3Rpb25hcnk8VD4ge1xuXHRcdFtpZDogc3RyaW5nXSA6IFQ7XG5cdH1cblx0XG5cdHZhciBvbGRTeW5hcHRpYyA9IHR5cGVvZiB3aW5kb3cgIT0gXCJ1bmRlZmluZWRcIiAmJiB3aW5kb3cgJiYgd2luZG93WydTeW5hcHRpYyddO1xuXHRcblx0ZXhwb3J0IGZ1bmN0aW9uIG5pbmphKCkge1xuICAgICAgd2luZG93WydzeW5hcHRpYyddID0gb2xkU3luYXB0aWM7IFxuICAgICAgcmV0dXJuIFN5bmFwdGljO1xuXHR9XG5cdFxuXHRleHBvcnQgaW50ZXJmYWNlIElDb21waWxlZFBhcmFtZXRlcnMge1x0XG5cdFx0bWVtb3J5PzogYW55O1xuXHRcdG5ldXJvbnM/OiBudW1iZXI7XG5cdFx0aW5wdXRzPzogYW55W107XG5cdFx0b3V0cHV0cz86IGFueVtdO1xuXHRcdHRhcmdldHM/OiBhbnlbXTtcblx0XHR2YXJpYWJsZXM/OiBhbnk7XG5cdFx0YWN0aXZhdGlvbl9zZW50ZW5jZXM/OiBhbnlbXTtcblx0XHR0cmFjZV9zZW50ZW5jZXM/OiBhbnlbXTtcblx0XHRwcm9wYWdhdGlvbl9zZW50ZW5jZXM/OiBhbnlbXTtcblx0XHRsYXllcnM/OiBhbnk7XG5cdH1cblx0XG5cdGV4cG9ydCB2YXIgTmV1cm9uID0gbmV1cm9uLk5ldXJvbjtcblx0ZXhwb3J0IHZhciBMYXllciA9IGxheWVyLkxheWVyO1xuXHRleHBvcnQgdmFyIE5ldHdvcmsgPSBuZXR3b3JrLk5ldHdvcms7XG5cdGV4cG9ydCB2YXIgVHJhaW5lciA9IHRyYWluZXIuVHJhaW5lcjtcblx0ZXhwb3J0IHZhciBTcXVhc2ggPSBzcXVhc2g7XG5cdGV4cG9ydCB2YXIgQXJjaGl0ZWN0ID0gYXJjaGl0ZWN0O1xufVxuXG5leHBvcnQgPSBTeW5hcHRpYztcblxuaWYodHlwZW9mIHdpbmRvdyAhPSBcInVuZGVmaW5lZFwiKSBcblx0d2luZG93WydzeW5hcHRpYyddID0gU3luYXB0aWM7XG4iXSwic291cmNlUm9vdCI6Ii9zb3VyY2UvIn0=

node-dist/src/trainer.d.ts

@@ -1,46 +0,0 @@
-import net = require('./network');
-/*******************************************************************************************
-                                        TRAINER
-*******************************************************************************************/
-export declare class Trainer {
-    network: net.Network;
-    rate: any;
-    iterations: number;
-    error: number;
-    cost: Trainer.ITrainerCostFn;
-    schedule: any;
-    constructor(network: net.Network, options?: any);
-    train(set: any, options: any): {
-        error: number;
-        iterations: number;
-        time: number;
-    };
-    workerTrain(set: any, callback: any, options: any): void;
-    XOR(options: any): {
-        error: number;
-        iterations: number;
-        time: number;
-    };
-    DSR(options: any): {
-        iterations: number;
-        success: number;
-        error: number;
-        time: number;
-    };
-    ERG(options: any): {
-        iterations: number;
-        error: number;
-        time: number;
-        test: (str: any) => boolean;
-        generate: () => string;
-    };
-}
-export declare module Trainer {
-    interface ITrainerCostFn {
-        (target: any, output: any): number;
-    }
-    var cost: {
-        CROSS_ENTROPY: (target: any, output: any) => number;
-        MSE: (target: any, output: any) => number;
-    };
-}

src/architect/LSTM.ts

@@ -2,6 +2,7 @@ import network  = require('../network');
 import trainer  = require('../trainer');
 import Layer  = require('../layer');
 import neuron = require('../neuron');
+import softMaxLayer = require('../softmaxLayer');
 
 export class LSTM extends network.Network {
   trainer: trainer.Trainer;

src/layer.ts

@@ -6,209 +6,213 @@ import Synaptic = require('./synaptic');
                                             LAYER
 *******************************************************************************************/
 export class Layer {
-		list: neuron.Neuron[] = [];
-		label: string = null;
-		connectedto = [];
-		size = 0;
+	optimizable = true;
+	list: neuron.Neuron[] = [];
+	label: string = null;
+	connectedto = [];
+	size = 0;
 
-		constructor(size: number, label?: string) {
-			this.size = size | 0;
-			this.list = [];
-			this.label = label || null;
-			this.connectedto = [];
+	currentActivation: Float64Array;
 
-			while (size--) {
-				var theNeuron = new neuron.Neuron();
-				this.list.push(theNeuron);
-			}
+	constructor(size: number, label?: string) {
+		this.size = size | 0;
+		this.list = [];
+		this.label = label || null;
+		this.connectedto = [];
+
+		this.currentActivation = new Float64Array(size);
+
+		while (size--) {
+			var theNeuron = new neuron.Neuron();
+			this.list.push(theNeuron);
 		}
+	}
 
 	
-		// activates all the neurons in the layer
-		activate(input) {
+	// activates all the neurons in the layer
+	activate(input?: Synaptic.INumericArray): Synaptic.INumericArray {
 
-			var activations = [];
+		if (this.currentActivation.length != this.list.length)
+			this.currentActivation = new Float64Array(this.list.length);
 
-			if (typeof input != 'undefined') {
-				if (input.length != this.size)
-					throw "INPUT size and LAYER size must be the same to activate!";
+		var activationIndex = 0;
 
-				for (var id in this.list) {
-					var neuron = this.list[id];
-					var activation = neuron.activate(input[id]);
-					activations.push(activation);
-				}
-			} else {
-				for (var id in this.list) {
-					var neuron = this.list[id];
-					var activation = neuron.activate();
-					activations.push(activation);
-				}
-			}
-			return activations;
-		}
-
-		// propagates the error on all the neurons of the layer
-		propagate(rate, target) {
-
-			if (typeof target != 'undefined') {
-				if (target.length != this.size)
-					throw "TARGET size and LAYER size must be the same to propagate!";
-
-				for (var id = this.list.length - 1; id >= 0; id--) {
-					var neuron = this.list[id];
-					neuron.propagate(rate, target[id]);
-				}
-			} else {
-				for (var id = this.list.length - 1; id >= 0; id--) {
-					var neuron = this.list[id];
-					neuron.propagate(rate);
-				}
-			}
-		}
-
-		// projects a connection from this layer to another one
-		project(layer, type?, weights?) {
-
-			if (layer instanceof network.Network)
-				layer = layer.layers.input;
-
-			if (layer instanceof Layer) {
-				if (!this.connected(layer))
-					return new Layer.LayerConnection(this, layer, type, weights);
-			} else
-				throw "Invalid argument, you can only project connections to LAYERS and NETWORKS!";
-
-
-		}
-
-		// gates a connection betwenn two layers
-		gate(connection, type) {
-
-			if (type == Layer.gateType.INPUT) {
-				if (connection.to.size != this.size)
-					throw "GATER layer and CONNECTION.TO layer must be the same size in order to gate!";
-
-				for (var id in connection.to.list) {
-					var neuron = connection.to.list[id];
-					var gater = this.list[id];
-					for (var input in neuron.connections.inputs) {
-						var gated = neuron.connections.inputs[input];
-						if (gated.ID in connection.connections)
-							gater.gate(gated);
-					}
-				}
-			} else if (type == Layer.gateType.OUTPUT) {
-				if (connection.from.size != this.size)
-					throw "GATER layer and CONNECTION.FROM layer must be the same size in order to gate!";
-
-				for (var id in connection.from.list) {
-					var neuron = connection.from.list[id];
-					var gater = this.list[id];
-					for (var projected in neuron.connections.projected) {
-						var gated = neuron.connections.projected[projected];
-						if (gated.ID in connection.connections)
-							gater.gate(gated);
-					}
-				}
-			} else if (type == Layer.gateType.ONE_TO_ONE) {
-				if (connection.size != this.size)
-					throw "The number of GATER UNITS must be the same as the number of CONNECTIONS to gate!";
-
-				for (var id in connection.list) {
-					var gater = this.list[id];
-					var gated = connection.list[id];
-					gater.gate(gated);
-				}
-			}
-			connection.gatedfrom.push({ layer: this, type: type });
-		}
-
-		// true or false whether the whole layer is self-connected or not
-		selfconnected(): boolean {
+		if (typeof input != 'undefined') {
+			if (input.length != this.size)
+				throw "INPUT size and LAYER size must be the same to activate!";
 
 			for (var id in this.list) {
-				var neuron = this.list[id];
-				if (!neuron.selfconnected())
-					return false;
+				this.currentActivation[activationIndex++] = this.list[id].activate(input[id]);
+			}
+		} else {
+			for (var id in this.list) {
+				this.currentActivation[activationIndex++] = this.list[id].activate();
 			}
-			return true;
 		}
 
-		// true of false whether the layer is connected to another layer (parameter) or not
-		connected(layer) {
-			// Check if ALL to ALL connection
-			var connections = 0;
-			for (var here in this.list) {
-				for (var there in layer.list) {
-					var from = this.list[here];
-					var to = layer.list[there];
-					var connected = from.connected(to);
-					if (connected && connected.type == 'projected')
-						connections++;
+		return this.currentActivation;
+	}
+
+	// propagates the error on all the neurons of the layer
+	propagate(rate: number, target?: Synaptic.INumericArray) {
+		if (typeof target != 'undefined') {
+			if (target.length != this.size)
+				throw "TARGET size and LAYER size must be the same to propagate!";
+
+			for (var id = this.list.length - 1; id >= 0; id--) {
+				var neuron = this.list[id];
+				neuron.propagate(rate, target[id]);
+			}
+		} else {
+			for (var id = this.list.length - 1; id >= 0; id--) {
+				var neuron = this.list[id];
+				neuron.propagate(rate);
+			}
+		}
+	}
+
+	// projects a connection from this layer to another one
+	project(layer: network.Network | Layer, type?: string, weights?: Synaptic.INumericArray): Layer.LayerConnection {
+
+		if (layer instanceof network.Network)
+			layer = (<network.Network>layer).layers.input;
+
+		if (layer instanceof Layer) {
+			if (!this.connected(layer))
+				return new Layer.LayerConnection(this, layer, type, weights);
+		} else
+			throw "Invalid argument, you can only project connections to LAYERS and NETWORKS!";
+
+
+	}
+
+	// gates a connection betwenn two layers
+	gate(connection, type) {
+
+		if (type == Layer.gateType.INPUT) {
+			if (connection.to.size != this.size)
+				throw "GATER layer and CONNECTION.TO layer must be the same size in order to gate!";
+
+			for (var id in connection.to.list) {
+				var neuron = connection.to.list[id];
+				var gater = this.list[id];
+				for (var input in neuron.connections.inputs) {
+					var gated = neuron.connections.inputs[input];
+					if (gated.ID in connection.connections)
+						gater.gate(gated);
 				}
 			}
-			if (connections == this.size * layer.size)
-				return Layer.connectionType.ALL_TO_ALL;
+		} else if (type == Layer.gateType.OUTPUT) {
+			if (connection.from.size != this.size)
+				throw "GATER layer and CONNECTION.FROM layer must be the same size in order to gate!";
 
-			// Check if ONE to ONE connection
-			connections = 0;
-			for (var neuron in this.list) {
-				var from = this.list[neuron];
-				var to = layer.list[neuron];
+			for (var id in connection.from.list) {
+				var neuron = connection.from.list[id];
+				var gater = this.list[id];
+				for (var projected in neuron.connections.projected) {
+					var gated = neuron.connections.projected[projected];
+					if (gated.ID in connection.connections)
+						gater.gate(gated);
+				}
+			}
+		} else if (type == Layer.gateType.ONE_TO_ONE) {
+			if (connection.size != this.size)
+				throw "The number of GATER UNITS must be the same as the number of CONNECTIONS to gate!";
+
+			for (var id in connection.list) {
+				var gater = this.list[id];
+				var gated = connection.list[id];
+				gater.gate(gated);
+			}
+		}
+		connection.gatedfrom.push({ layer: this, type: type });
+	}
+
+	// true or false whether the whole layer is self-connected or not
+	selfconnected(): boolean {
+
+		for (var id in this.list) {
+			var neuron = this.list[id];
+			if (!neuron.selfconnected())
+				return false;
+		}
+		return true;
+	}
+
+	// true of false whether the layer is connected to another layer (parameter) or not
+	connected(layer) {
+		// Check if ALL to ALL connection
+		var connections = 0;
+		for (var here in this.list) {
+			for (var there in layer.list) {
+				var from = this.list[here];
+				var to = layer.list[there];
 				var connected = from.connected(to);
 				if (connected && connected.type == 'projected')
 					connections++;
 			}
-			if (connections == this.size)
-				return Layer.connectionType.ONE_TO_ONE;
 		}
+		if (connections == this.size * layer.size)
+			return Layer.connectionType.ALL_TO_ALL;
 
-		// clears all the neuorns in the layer
-		clear() {
-			for (var id in this.list) {
-				var neuron = this.list[id];
-				neuron.clear();
-			}
+		// Check if ONE to ONE connection
+		connections = 0;
+		for (var neuron in this.list) {
+			var from = this.list[neuron];
+			var to = layer.list[neuron];
+			var connected = from.connected(to);
+			if (connected && connected.type == 'projected')
+				connections++;
 		}
+		if (connections == this.size)
+			return Layer.connectionType.ONE_TO_ONE;
+	}
 
-		// resets all the neurons in the layer
-		reset() {
-			for (var id in this.list) {
-				var neuron = this.list[id];
-				neuron.reset();
-			}
-		}
-
-		// returns all the neurons in the layer (array)
-		neurons() : neuron.Neuron[] {
-			return this.list;
-		}
-
-		// adds a neuron to the layer
-		add(neuron) {
-			neuron = neuron || new neuron.Neuron();
-			this.neurons[neuron.ID] = neuron;
-			this.list.push(neuron);
-			this.size++;
-		}
-
-		set(options) {
-			options = options || {};
-
-			for (var i in this.list) {
-				var neuron = this.list[i];
-				if (options.label)
-					neuron.label = options.label + '_' + neuron.ID;
-				if (options.squash)
-					neuron.squash = options.squash;
-				if (options.bias)
-					neuron.bias = options.bias;
-			}
-			return this;
+	// clears all the neuorns in the layer
+	clear() {
+		for (var id in this.list) {
+			var neuron = this.list[id];
+			neuron.clear();
 		}
 	}
 
+	// resets all the neurons in the layer
+	reset() {
+		for (var id in this.list) {
+			var neuron = this.list[id];
+			neuron.reset();
+		}
+	}
+
+	// returns all the neurons in the layer (array)
+	neurons(): neuron.Neuron[] {
+		return this.list;
+	}
+
+	// adds a neuron to the layer
+	add(neuron) {
+		neuron = neuron || new neuron.Neuron();
+		this.neurons[neuron.ID] = neuron;
+		this.list.push(neuron);
+		this.size++;
+	}
+
+	set(options) {
+		options = options || {};
+
+		for (var i in this.list) {
+			var neuron = this.list[i];
+			if (options.label)
+				neuron.label = options.label + '_' + neuron.ID;
+			if (options.squash)
+				neuron.squash = options.squash;
+			if (options.bias)
+				neuron.bias = options.bias;
+		}
+		return this;
+	}
+}
+
 
 export module Layer {
 	export var layerQty = 0;
@@ -235,7 +239,7 @@ export module Layer {
 		ID = uid();
 		from: Layer;
 		to: Layer;
-		selfconnection : boolean = false;
+		selfconnection: boolean = false;
 		type: string;
 		connections: Synaptic.Dictionary<neuron.Neuron.Connection>;
 		list: neuron.Neuron.Connection[];

src/network.ts

@@ -17,19 +17,18 @@ export class Network {
 		hidden: {},
 		output: null
 	};
-	constructor(layers) {
+	constructor(layers?) {
 		if (typeof layers != 'undefined') {
 			this.layers = layers || {
 				input: null,
-				hidden: {},
+				hidden: [],
 				output: null
 			};
-			this.optimized = null;
 		}
 	}
 
 	// feed-forward activation of all the layers to produce an ouput
-	activate(input) {
+	activate(input : Synaptic.INumericArray) {
 
 		if (this.optimized === false) {
 			this.layers.input.activate(input);
@@ -45,7 +44,7 @@ export class Network {
 	}
 
 	// back-propagate the error thru the network
-	propagate(rate: number, target?) {
+	propagate(rate: number, target?: Synaptic.INumericArray) {
 
 		if (this.optimized === false) {
 			this.layers.output.propagate(rate, target);

src/neuron.ts

@@ -14,6 +14,8 @@ import Squash = require('./squash');
 */
 
 export class Neuron {
+	optimizable = true;
+
 	ID = Neuron.uid();
 	label = null;
 	connections: Neuron.INeuronConnections = {
@@ -39,9 +41,12 @@ export class Neuron {
 	neighboors = {};
 	bias = Math.random() * .2 - .1;
 	derivative = 0;
-  
-	// activate the neuron
-	activate(input?: number) {
+
+	constructor() {
+
+	}
+
+	readIncommingConnections(input?: number) {
 		// activation from enviroment (for input neurons)
 		if (typeof input != 'undefined') {
 			this.activation = input;
@@ -49,7 +54,7 @@ export class Neuron {
 			this.bias = 0;
 			return this.activation;
 		}
-
+		
 		// old state
 		this.old = this.state;
 
@@ -67,7 +72,9 @@ export class Neuron {
 
 		// f'(s)
 		this.derivative = this.squash(this.state, true);
+	}
 
+	updateTraces() {
 		// update traces
 		var influences = [];
 		for (var id in this.trace.extended) {
@@ -106,11 +113,19 @@ export class Neuron {
 				theInput.ID] * influence;
 			}
 		}
-
+		
 		//  update gated connection's gains
 		for (var connection in this.connections.gated) {
 			this.connections.gated[connection].gain = this.activation;
 		}
+	}
+  
+  
+	// activate the neuron
+	activate(input?: number) : number {
+		this.readIncommingConnections(input);
+
+		this.updateTraces();
 
 		return this.activation;
 	}

src/softmaxLayer.ts

@@ -0,0 +1,77 @@
+import Synaptic = require('./synaptic');
+import Layer = require('./layer');
+import Squash = require('./squash');
+import Neuron = require('./neuron');
+import _Utils = require('./utils');
+var Utils = _Utils.Utils;
+
+export class SoftMaxLayer extends Layer.Layer {
+	constructor(size: number, label?: string) {
+		super(size, label);
+
+		this.optimizable = false;
+
+		for (var n = 0; n < this.list.length; n++) {
+			this.list[n].squash = Squash.IDENTITY;
+		}
+	}
+
+	activate(input?: Synaptic.INumericArray): Synaptic.INumericArray {
+		if (this.currentActivation.length != this.list.length)
+			this.currentActivation = new Float64Array(this.list.length);
+
+		var activationIndex = 0;
+
+		var sum = 0;
+		var Amax = null;
+		
+		if (typeof input != 'undefined') {
+			if (input.length != this.size)
+				throw "INPUT size and LAYER size must be the same to activate!";
+
+			Utils.softMax(input);
+
+			for (var id in this.list) {
+				this.list[id].readIncommingConnections(input[id]);
+				if (Amax === null || this.list[id].activation > Amax)
+					Amax = this.list[id].activation;
+			}
+		} else {
+			for (var id in this.list) {
+				this.list[id].readIncommingConnections();
+				if (Amax === null || this.list[id].activation > Amax)
+					Amax = this.list[id].activation;
+			}
+		}
+
+		for (var n = 0; n < this.currentActivation.length; n++) {
+			sum += (this.list[n].activation = Math.exp(this.list[n].activation - Amax));
+		}
+		
+		for (var n = 0; n < this.currentActivation.length; n++) {
+			// set the activations
+			var x = this.list[n].activation / sum;
+			this.list[n].activation = this.currentActivation[n] = x;
+			
+			// set the derivatives
+			
+			//x = this.list[n].activation / (sum - this.list[n].activation);
+			
+			this.list[n].derivative = x*(1-x);
+			
+			this.list[n].updateTraces();
+		}
+		
+		return this.currentActivation;
+	}
+
+	static NormalizeConnectionWeights(layerConnection: Layer.Layer.LayerConnection) {
+		var sum = 0;
+		for (var c = 0; c < layerConnection.list.length; c++) {
+			sum += (layerConnection.list[c].weight = Math.exp(layerConnection.list[c].weight));
+		}
+		for (var c = 0; c < layerConnection.list.length; c++) {
+			layerConnection.list[c].weight /= sum;
+		}
+	}
+}

src/squash.ts

@@ -3,10 +3,12 @@ import Synaptic = require('./synaptic');
 // squashing functions
 
 export function LOGISTIC(x: number, derivate?: boolean) {
-	if (!derivate)
-		return 1 / (1 + Math.exp(-x));
-	var fx = LOGISTIC(x);
-	return fx * (1 - fx);
+	if (derivate) {
+		var fx = LOGISTIC(x);
+		return fx * (1 - fx);
+	}
+	return 1 / (1 + Math.exp(-x));
+
 }
 
 export function TANH(x: number, derivate?: boolean) {
@@ -24,3 +26,13 @@ export function IDENTITY(x: number, derivate?: boolean) {
 export function HLIM(x: number, derivate?: boolean) {
 	return derivate ? 1 : +(x > 0);
 }
+
+export function SOFTPLUS(x: number, derivate?: boolean) {
+	if (derivate)
+		return 1 - 1 / (1 + Math.exp(x));
+	return Math.log(1 + Math.exp(x));
+}
+
+export function EXP(x: number, derivate?: boolean) {
+	return Math.exp(x);
+} 

src/synaptic.ts

@@ -32,6 +32,7 @@ import neuron = require('./neuron');
 import trainer = require('./trainer');
 import architect = require('./architect');
 import squash = require('./squash');
+import utils = require('./utils');
 
 declare var window;
 
@@ -60,12 +61,18 @@ module Synaptic {
 		layers?: any;
 	}
 	
+	export interface INumericArray {
+	  [index: number] : number;
+	  length : number;
+	}
+	
 	export var Neuron = neuron.Neuron;
 	export var Layer = layer.Layer;
 	export var Network = network.Network;
 	export var Trainer = trainer.Trainer;
 	export var Squash = squash;
 	export var Architect = architect;
+	export var Utils = utils.Utils;
 }
 
 export = Synaptic;

src/trainer.ts

@@ -1,4 +1,4 @@
-import net =require('./network');
+import net = require('./network');
 
 /*******************************************************************************************
                                         TRAINER
@@ -86,15 +86,16 @@ export class Trainer {
       error /= set.length;
 
       if (options) {
-        if (this.schedule && this.schedule.every && iterations %
-          this.schedule.every == 0)
+        if (this.schedule && this.schedule.every && iterations % this.schedule.every == 0) {
+
           abort_training = this.schedule.do({
             error: error,
             iterations: iterations,
             rate: currentRate
           });
-        else if (options.log && iterations % options.log == 0) {
-          console.log('iterations', iterations, 'error', error, 'rate', currentRate);
+     
+        } else if (options.log && iterations % options.log == 0) {
+          console.log('iterations', iterations, 'error', error, 'rate', currentRate, 'T:', target, 'O:', output);
         };
         if (options.shuffle)
           shuffle(set);
@@ -379,7 +380,7 @@ export class Trainer {
       if (log && trial % log == 0)
         console.log("iterations:", trial, " success:", success, " correct:",
           correct, " time:", Date.now() - start, " error:", error);
-      if (schedule.do && schedule.every && trial % schedule.every == 0)
+      if (schedule.do && schedule.every && trial % schedule.every == 0) {
         schedule.do({
           iterations: trial,
           success: success,
@@ -387,6 +388,8 @@ export class Trainer {
           time: Date.now() - start,
           correct: correct
         });
+
+      }
     }
 
     return {
@@ -604,6 +607,12 @@ export module Trainer {
         crossentropy -= (target[i] * Math.log(output[i] + 1e-15)) + ((1 - target[i]) * Math.log((1 + 1e-15) - output[i])); // +1e-15 is a tiny push away to avoid Math.log(0)
       return crossentropy;
     },
+    CROSS_ENTROPY_SOFTMAX: function(target, output) {
+      var crossentropy = 0;
+      for (var i in output)
+        crossentropy -= target[i] * Math.log(output[i] + 1e-15);
+      return crossentropy;
+    },
     MSE: function(target, output) {
       var mse = 0;
       for (var i in output)

src/utils.ts

@@ -0,0 +1,218 @@
+import Synaptic = require('./synaptic');
+
+export class Utils {
+  
+  static transformationMatrixCache: { [index: number] : Float64Array[] } = {};
+  
+  static softMax<T extends Synaptic.INumericArray>(outputArray: T): T {
+    // for all i ∈ array
+    // sum = ∑ array[n]^e
+    // i = î^e / sum
+    // where the result ∑ array[0..n] = 1
+
+    if (!outputArray.length) return outputArray;
+
+    var sum = 0;
+
+    var Amax = outputArray[0];
+    for (var i = 0; i < outputArray.length; i++) {
+      if(outputArray[i] < Amax) Amax = outputArray[i];
+    }
+    
+
+    // sum = ∑ array[n]^e
+    for (var i = 0; i < outputArray.length; i++) {
+      outputArray[i] = Math.exp(outputArray[i] - Amax);
+      sum += outputArray[i];
+    }
+
+    for (var i = 0; i < outputArray.length; i++) outputArray[i] /= sum;
+
+    return outputArray;
+  }
+
+  static softMaxDerivative<T extends Synaptic.INumericArray>(outputArray: T): T {
+    // http://sysmagazine.com/posts/155235/
+    if (!outputArray.length) return outputArray;
+
+    var sum = 0;
+
+    // sum = ∑ array[n]^e
+    for (var i = 0; i < outputArray.length; i++) {
+      outputArray[i] = Math.exp(outputArray[i]);
+      sum += outputArray[i];
+    }
+
+    for (var i = 0; i < outputArray.length; i++) {
+      var t = outputArray[i] /= sum;
+      
+      outputArray[i] = t * (1 - t);
+    }
+
+    return outputArray;
+  }
+
+  static softMaxReinforcement<T extends Synaptic.INumericArray>(array: T, temperature = 1): T {
+    // Reinforcement learning
+
+    if (!array.length) return array;
+
+    temperature = temperature || 1;
+
+    var sum = 0;
+
+    // sum = ∑ array[n]^e
+    for (var i = 0; i < array.length; i++) {
+      array[i] = Math.exp(array[i] / temperature);
+      sum += array[i];
+    }
+
+    if (sum != 0) {
+      for (var i = 0; i < array.length; i++) array[i] /= sum;
+    } else {
+      var div = 1 / array.length;
+      for (var i = 0; i < array.length; i++) array[i] = div;
+    }
+
+    return array;
+  }
+
+
+  static getCosineSimilarity(arrayA: Synaptic.INumericArray, arrayB: Synaptic.INumericArray): number {
+    // http://en.wikipedia.org/wiki/Cosine_similarity
+    var dotPr = 0;
+
+    var acumA = 0, acumB = 0;
+
+    for (var i = 0; i < arrayA.length; i++) {
+      dotPr += arrayA[i] * arrayB[i];
+      acumA += arrayA[i] * arrayA[i];
+      acumB += arrayB[i] * arrayB[i];
+    }
+
+    return dotPr / (Math.sqrt(acumA) * Math.sqrt(acumB) + .00005);
+  }
+
+  static interpolateArray(output_inputA: Synaptic.INumericArray, inputB: Synaptic.INumericArray, g) {
+    // 3.3.2 focus by location (7)
+    var gInverted = 1 - g;
+    for (var i = 0; i < output_inputA.length; i++)
+      output_inputA[i] = output_inputA[i] * g + gInverted * inputB[i];
+    return output_inputA;
+  }
+  
+  // w_sharpWn
+  static sharpArray(output: Synaptic.INumericArray, wn: Synaptic.INumericArray, Y: number) {
+    // 3.3.2 (9)
+    var sum = 0;
+
+    // ∀ a ∈ wn → a = a^Y
+    // sum = ∑ a^Y 
+
+    for (var i = 0; i < wn.length; i++) {
+      wn[i] = Math.pow(wn[i], Y);
+      sum += wn[i];
+    }
+
+    // ∀ a ∈ wn → a = a^Y / sum
+    if (sum != 0) {
+      for (var i = 0; i < wn.length; i++) output[i] = wn[i] / sum;
+    } else {
+      var div = 1 / wn.length;
+      for (var i = 0; i < wn.length; i++) output[i] = div;
+    }
+  }
+  
+  //wn_shift
+  static scalarShifting(wg: Synaptic.INumericArray, shiftScalar: number) {
+    // w~ 3.3.2 (8)
+    var shiftings = new Float64Array(wg.length);
+    var wn = new Float64Array(wg.length);
+
+    var intPart = shiftScalar | 0;
+    var decimalPart = shiftScalar - intPart;
+
+    shiftings[intPart % shiftings.length] = 1 - decimalPart;
+    shiftings[(intPart + 1) % shiftings.length] = decimalPart;
+
+
+    for (var i = 0; i < wn.length; i++) {
+      var acum = 0;
+      for (var j = 0; j < wn.length; j++) {
+        if ((i - j) < 0)
+          acum += wg[j] * shiftings[shiftings.length - Math.abs(i - j)];
+        else
+          acum += wg[j] * shiftings[(i - j) % shiftings.length];
+      }
+      wn[i] = acum;
+    }
+
+    return wn;
+  }
+
+  static normalizeShift(shift: Float64Array) {
+    var sum = 0;
+    for (var i = 0; i < shift.length; i++) {
+      sum += shift[i];
+    }
+    for (var j = 0; j < shift.length; j++) {
+      shift[j] /= sum;
+    }
+  }
+
+  static vectorInvertedShifting(wg: Float64Array, shiftings: Synaptic.INumericArray) {
+    // w~ 3.3.2 (8)
+
+    
+    var ret = new Float64Array(wg.length);
+
+    var corrimientoIndex = -((shiftings.length - 1) / 2) | 0;
+
+    var circulantMatrix = Utils.transformationMatrixCache[wg.length] || (Utils.transformationMatrixCache[wg.length] = Utils.buildCirculantMatrix(wg.length));
+
+    for (var i = 0; i < wg.length; i++) {
+      for (var x = 0; x < wg.length; x++) {
+        var tmp = 0;
+
+        for (var shift = 0; shift < shiftings.length; shift++) {
+
+          var matRow = i - x + corrimientoIndex + shift;
+
+          while (matRow < 0)
+            matRow += wg.length;
+
+          matRow %= wg.length;
+
+          tmp += wg[circulantMatrix[x][matRow]] * shiftings[shift];
+        }
+
+        ret[i] = tmp;
+      }
+
+    }
+
+    wg.set(ret);
+  }
+
+  static initRandomSoftmaxArray(array: Float64Array): void {
+    for (var i = 0; i < array.length; i++) {
+      array[i] = Math.random();
+    }
+
+    Utils.softMax(array);
+  }
+
+  static buildCirculantMatrix(length: number, offset: number = 0): Float64Array[] {
+    var ret = [];
+
+    for (var i = 0; i < length; i++) {
+      var arr = new Float64Array(length);
+      ret.push(arr);
+      for (var n = 0; n < length; n++) {
+        arr[n] = ((i + n) % length);
+      }
+    }
+
+    return ret;
+  }
+}

test/ntm.js

@@ -0,0 +1,119 @@
+// import
+
+var assert = require('assert'),
+  Utils = require('../dist/src/utils').Utils;
+
+
+// utils
+
+var noRepeat = function(range, avoid) {
+  var number = Math.random() * range | 0;
+  var used = false;
+  for (var i in avoid)
+    if (number == avoid[i])
+      used = true;
+  return used ? noRepeat(range, avoid) : number;
+};
+
+var equal = function(prediction, output) {
+  for (var i in prediction)
+    if (Math.round(prediction[i]) != output[i])
+      return false;
+  return true;
+};
+
+var generateRandomArray = function(size){
+    var array = [];
+    for (var j = 0; j < size; j++)
+        array.push(Math.random() + .5 | 0);
+    return array;
+}
+
+var sumAll = function(arr){
+  var tmp = 0;
+  for (var index = 0; index < arr.length; index++) {
+    tmp += arr[index];
+  }
+  return parseFloat(tmp.toFixed(3));
+}
+
+// specs
+
+
+describe("NTM Utils", function() {
+  
+  it('Test equality arrays', function(){
+    var fixedArray = new Float64Array(5);
+    
+    fixedArray.set([0,0,0,0,1]);
+    
+    assert.deepEqual(fixedArray, [0,0,0,0,1], 'Float64Array vs array');
+  });
+  
+  it('Softmax array', function(){
+    var fixedArray = new Float64Array(5);
+    
+    fixedArray.set([0,0,0,0,1]);
+    
+    assert.equal(sumAll(Utils.softMax(fixedArray)), 1, 'Fixed array equals 1');
+    
+    
+    fixedArray.set([0,0,0,0,0]);
+    
+    assert.equal(sumAll(Utils.softMax(fixedArray)), 1, 'Fixed zero array equals 1');
+    
+    fixedArray.set([Math.random()* 3,Math.random() * 6,Math.random() * 4,Math.random() * 10,Math.random()]);
+    
+    assert.equal(sumAll(Utils.softMax(fixedArray)), 1, 'Random positive array equals 1');
+    
+    fixedArray.set([Math.random()* -9,Math.random() * 6,Math.random() * -4,Math.random() * 10,Math.random()]);
+    
+    assert.equal(sumAll(Utils.softMax(fixedArray)), 1, 'Random signed array equals 1');
+  });
+
+  it('Shifting', function(){
+    var fixedArray = new Float64Array(5);
+    
+    fixedArray.set([0,0,0,0,1]);
+    Utils.vectorInvertedShifting(fixedArray, [0, 1, 0]);
+    assert.deepEqual(fixedArray, [0,0,0,0,1], 'Unchanged shiftings');
+    
+    fixedArray.set([0,0,0,0,1]);
+    Utils.vectorInvertedShifting(fixedArray, [0, 0, 0]);
+    assert.deepEqual(fixedArray, [0,0,0,0,0], 'All zeros in shift');
+    
+    fixedArray.set([0,0,0,0,1]);
+    Utils.vectorInvertedShifting(fixedArray, [1, 0, 0]);
+    assert.deepEqual(fixedArray, [1,0,0,0,0], 'Plus one shift');
+    
+    fixedArray.set([0,0,0,0,1]);
+    Utils.vectorInvertedShifting(fixedArray, [0, 0, 1]);
+    assert.deepEqual(fixedArray, [0,0,0,1,0], 'Minus one shift');
+
+    fixedArray.set([1,0,0,0,2]);
+    Utils.vectorInvertedShifting(fixedArray, [1, 0, 0]);
+    assert.deepEqual(fixedArray, [2,1,0,0,0], 'Plus one shift, two values');
+    
+    fixedArray.set([1,1,1,1,1]);
+    Utils.vectorInvertedShifting(fixedArray, [1, 1, 1]);
+    assert.deepEqual(fixedArray, [3,3,3,3,3], 'Mixing three values');
+
+    fixedArray.set([1,1,1,1,1]);
+    Utils.vectorInvertedShifting(fixedArray, [1, 1, 0]);
+    assert.deepEqual(fixedArray, [2,2,2,2,2], 'Mixing two values');
+    
+    fixedArray.set([1,1,1,1,1]);
+    Utils.vectorInvertedShifting(fixedArray, [.5, 1, 0.5]);
+    assert.deepEqual(fixedArray, [2,2,2,2,2], 'Mixing threeº values');
+    
+    
+    fixedArray.set([1,1,1,1,1]);
+    Utils.vectorInvertedShifting(fixedArray, [.5, .5, 1, .5, .5]);
+    assert.deepEqual(fixedArray, [3,3,3,3,3], 'Mixing five values');
+
+    fixedArray.set([1,2,1,3,1]);
+    Utils.vectorInvertedShifting(fixedArray, [.5, 0, .5]);
+    assert.deepEqual(fixedArray, [1.5,1,2.5,1,2], 'Mixing five values');
+
+  });
+});

test/synaptic.js

@@ -1,7 +1,8 @@
 // import
 
 var assert = require('assert'),
-  synaptic = require('../node-dist/src/synaptic');
+  synaptic = require('../dist/src/synaptic'),
+  softMaxLayer = require('../dist/src/softmaxLayer');
 
 var Perceptron = synaptic.Architect.Perceptron,
   LSTM = synaptic.Architect.LSTM,
@@ -12,7 +13,7 @@ var Perceptron = synaptic.Architect.Perceptron,
 
 // utils
 
-var noRepeat = function(range, avoid) {
+var noRepeat = function (range, avoid) {
   var number = Math.random() * range | 0;
   var used = false;
   for (var i in avoid)
@@ -21,28 +22,28 @@ var noRepeat = function(range, avoid) {
   return used ? noRepeat(range, avoid) : number;
 };
 
-var equal = function(prediction, output) {
+var equal = function (prediction, output) {
   for (var i in prediction)
     if (Math.round(prediction[i]) != output[i])
       return false;
   return true;
 };
 
-var generateRandomArray = function(size){
-    var array = [];
-    for (var j = 0; j < size; j++)
-        array.push(Math.random() + .5 | 0);
-    return array;
+var generateRandomArray = function (size) {
+  var array = [];
+  for (var j = 0; j < size; j++)
+    array.push(Math.random() + .5 | 0);
+  return array;
 }
 
 // specs
 
-describe('Basic Neural Network', function() {
+describe('Basic Neural Network', function () {
 
-  it("trains an AND gate", function() {
+  it("trains an AND gate", function () {
 
     var inputLayer = new Layer(2),
-        outputLayer = new Layer(1);
+      outputLayer = new Layer(1);
 
     inputLayer.project(outputLayer);
 
@@ -57,15 +58,15 @@ describe('Basic Neural Network', function() {
       input: [0, 0],
       output: [0]
     }, {
-      input: [0, 1],
-      output: [0]
-    }, {
-      input: [1, 0],
-      output: [0]
-    }, {
-      input: [1, 1],
-      output: [1]
-    }];
+        input: [0, 1],
+        output: [0]
+      }, {
+        input: [1, 0],
+        output: [0]
+      }, {
+        input: [1, 1],
+        output: [1]
+      }];
 
     trainer.train(trainingSet, {
       iterations: 1000,
@@ -85,7 +86,7 @@ describe('Basic Neural Network', function() {
     assert.equal(test11, 1, "[1,1] did not output 1");
   });
 
-  it("trains an OR gate", function() {
+  it("trains an OR gate", function () {
 
     var inputLayer = new Layer(2),
       outputLayer = new Layer(1);
@@ -103,15 +104,15 @@ describe('Basic Neural Network', function() {
       input: [0, 0],
       output: [0]
     }, {
-      input: [0, 1],
-      output: [1]
-    }, {
-      input: [1, 0],
-      output: [1]
-    }, {
-      input: [1, 1],
-      output: [1]
-    }];
+        input: [0, 1],
+        output: [1]
+      }, {
+        input: [1, 0],
+        output: [1]
+      }, {
+        input: [1, 1],
+        output: [1]
+      }];
 
     trainer.train(trainingSet, {
       iterations: 1000,
@@ -131,7 +132,7 @@ describe('Basic Neural Network', function() {
     assert.equal(test11, 1, "[1,1] did not output 1");
   });
 
-  it("trains a NOT gate", function() {
+  it("trains a NOT gate", function () {
 
     var inputLayer = new Layer(1),
       outputLayer = new Layer(1),
@@ -149,9 +150,9 @@ describe('Basic Neural Network', function() {
       input: [0],
       output: [1]
     }, {
-      input: [1],
-      output: [0]
-    }];
+        input: [1],
+        output: [0]
+      }];
 
     trainer.train(trainingSet, {
       iterations: 1000,
@@ -166,37 +167,109 @@ describe('Basic Neural Network', function() {
   });
 });
 
-describe("Perceptron - XOR", function() {
+describe("Perceptron - XOR", function () {
 
   var perceptron = new Perceptron(2, 3, 1);
   perceptron.trainer.XOR();
 
   var test00 = Math.round(perceptron.activate([0, 0]));
-  it("input: [0,0] output: " + test00, function() {
+  it("input: [0,0] output: " + test00, function () {
 
     assert.equal(test00, 0, "[0,0] did not output 0");
   });
 
   var test01 = Math.round(perceptron.activate([0, 1]));
-  it("input: [0,1] output: " + test01, function() {
+  it("input: [0,1] output: " + test01, function () {
 
     assert.equal(test01, 1, "[0,1] did not output 1");
   });
 
   var test10 = Math.round(perceptron.activate([1, 0]));
-  it("input: [1,0] output: " + test10, function() {
+  it("input: [1,0] output: " + test10, function () {
 
     assert.equal(test10, 1, "[1,0] did not output 1");
   });
 
   var test11 = Math.round(perceptron.activate([1, 1]));
-  it("input: [1,1] output: " + test11, function() {
+  it("input: [1,1] output: " + test11, function () {
 
     assert.equal(test11, 0, "[1,1] did not output 0");
   });
 });
 
-describe("LSTM - Discrete Sequence Recall", function() {
+describe("Perceptron - XOR Softmax", function () {
+
+
+
+  var input = new synaptic.Layer(2);
+  var hidden = new softMaxLayer.SoftMaxLayer(5);
+  var output = new softMaxLayer.SoftMaxLayer(2);
+
+  // generate hidden layers
+  softMaxLayer.SoftMaxLayer.NormalizeConnectionWeights(input.project(hidden));
+  softMaxLayer.SoftMaxLayer.NormalizeConnectionWeights(hidden.project(output));
+  
+  // set layers of the neural network
+      
+  var perceptron = new synaptic.Network({
+    input: input,
+    hidden: [hidden],
+    output: output
+  });
+  
+  // trainer for the network
+  perceptron.trainer = new synaptic.Trainer(perceptron);
+  perceptron.optimized = false;
+
+
+  var trainingSet = [
+    {
+      input: [0, 0],
+      output: [0, 1]
+    }, {
+      input: [1, 1],
+      output: [0, 1]
+    },
+    {
+      input: [0, 1],
+      output: [1, 0]
+    }, {
+      input: [1, 0],
+      output: [1, 0]
+    }
+  ];
+
+  perceptron.trainer.train(trainingSet, {
+    log: 5000,
+    cost: synaptic.Trainer.cost.CROSS_ENTROPY_SOFTMAX,
+    error: 0.001,
+    iterations: 100000,
+    rate: 0.1
+  });
+
+  var test00 = Math.round(perceptron.activate([0, 0])[0]);
+
+  it("input: [0,0] output: " + test00, function () {
+    assert.equal(test00, 0, "[0,0] did not output 0");
+  });
+
+  var test01 = Math.round(perceptron.activate([0, 1])[0]);
+  it("input: [0,1] output: " + test01, function () {
+    assert.equal(test01, 1, "[0,1] did not output 1");
+  });
+
+  var test10 = Math.round(perceptron.activate([1, 0])[0]);
+  it("input: [1,0] output: " + test10, function () {
+    assert.equal(test10, 1, "[1,0] did not output 1");
+  });
+
+  var test11 = Math.round(perceptron.activate([1, 1])[0]);
+  it("input: [1,1] output: " + test11, function () {
+    assert.equal(test11, 0, "[1,1] did not output 0");
+  });
+});
+
+describe("LSTM - Discrete Sequence Recall", function () {
 
   var targets = [2, 4];
   var distractors = [3, 5];
@@ -234,7 +307,7 @@ describe("LSTM - Discrete Sequence Recall", function() {
     sequence.push(prompts[i]);
   }
 
-  var check = function(which) {
+  var check = function (which) {
     // generate input from sequence
     var input = [];
     for (j = 0; j < symbols; j++)
@@ -259,7 +332,7 @@ describe("LSTM - Discrete Sequence Recall", function() {
     };
   };
 
-  var value = function(array) {
+  var value = function (array) {
     var max = .5;
     var res = -1;
     for (var i in array)
@@ -270,31 +343,31 @@ describe("LSTM - Discrete Sequence Recall", function() {
     return res == -1 ? '-' : targets[res];
   };
 
-  it("targets: " + targets, function() {
+  it("targets: " + targets, function () {
     assert(true);
   });
-  it("distractors: " + distractors, function() {
+  it("distractors: " + distractors, function () {
     assert(true);
   });
-  it("prompts: " + prompts, function() {
+  it("prompts: " + prompts, function () {
     assert(true);
   });
-  it("length: " + length + "\n", function() {
+  it("length: " + length + "\n", function () {
     assert(true);
   });
 
   for (var i = 0; i < length; i++) {
     var test = check(i);
     it((i + 1) + ") input: " + sequence[i] + " output: " + value(test.prediction),
-      function() {
+      function () {
         var ok = equal(test.prediction, test.output);
         assert(ok);
       });
   }
 });
 
-describe("Optimized and Unoptimized Networks Equivalency", function() {
-  var optimized = new Perceptron(10,15,5);
+describe("Optimized and Unoptimized Networks Equivalency", function () {
+  var optimized = new Perceptron(10, 15, 5);
 
   var unoptimized = optimized.clone();
   unoptimized.setOptimize(false);
@@ -302,35 +375,34 @@ describe("Optimized and Unoptimized Networks Equivalency", function() {
   var learningRate = .5;
   var iterations = 1000;
 
-  for (var i = 1; i <= iterations; i++)
-  {
-      //random input
-      var input = generateRandomArray(10);
+  for (var i = 1; i <= iterations; i++) {
+    //random input
+    var input = generateRandomArray(10);
 
-      // activate networks
-      var output1 = optimized.activate(input);
-      var output2 = unoptimized.activate(input);
+    // activate networks
+    var output1 = optimized.activate(input);
+    var output2 = unoptimized.activate(input);
 
-      if (i % 100 == 0)
-        it(' same output for both networks after ' + i + ' iterations', function(){
-          var diff = false;
-          for (var k in output1)
-            if (output1[k] - output2[k] != 0)
-              diff = true;
-          assert(!diff);
-        });
+    if (i % 100 == 0)
+      it(' same output for both networks after ' + i + ' iterations', function () {
+        var diff = false;
+        for (var k in output1)
+          if (output1[k] - output2[k] != 0)
+            diff = true;
+        assert(!diff);
+      });
 
-      // random target
-      var target = generateRandomArray(5);
+    // random target
+    var target = generateRandomArray(5);
 
-      // propagate networks
-      optimized.propagate(learningRate, target);
-      unoptimized.propagate(learningRate, target);
+    // propagate networks
+    optimized.propagate(learningRate, target);
+    unoptimized.propagate(learningRate, target);
   }
 });
 
-describe("toJSON/fromJSON Networks Equivalency", function() {
-  var original = new Perceptron(10,15,5);
+describe("toJSON/fromJSON Networks Equivalency", function () {
+  var original = new Perceptron(10, 15, 5);
 
   var exported = original.toJSON();
   var imported = Network.fromJSON(exported);
@@ -338,129 +410,127 @@ describe("toJSON/fromJSON Networks Equivalency", function() {
   var learningRate = .5;
   var iterations = 1000;
 
-  for (var i = 1; i <= iterations; i++)
-  {
-      //random input
-      var input = generateRandomArray(10);
+  for (var i = 1; i <= iterations; i++) {
+    //random input
+    var input = generateRandomArray(10);
 
-      // activate networks
-      var output1 = original.activate(input);
-      var output2 = imported.activate(input);
+    // activate networks
+    var output1 = original.activate(input);
+    var output2 = imported.activate(input);
 
-      if (i % 100 == 0)
-        it(' same output for both networks after ' + i + ' iterations', function(){
-          var diff = false;
-          for (var k in output1)
-            if (output1[k] - output2[k] != 0)
-              diff = true;
-          assert(!diff);
-        });
+    if (i % 100 == 0)
+      it(' same output for both networks after ' + i + ' iterations', function () {
+        var diff = false;
+        for (var k in output1)
+          if (output1[k] - output2[k] != 0)
+            diff = true;
+        assert(!diff);
+      });
 
-      // random target
-      var target = generateRandomArray(5);
+    // random target
+    var target = generateRandomArray(5);
 
-      // propagate networks
-      original.propagate(learningRate, target);
-      imported.propagate(learningRate, target);
+    // propagate networks
+    original.propagate(learningRate, target);
+    imported.propagate(learningRate, target);
   }
 });
 
-describe("Cloned Networks Equivalency", function() {
-  var original = new Perceptron(10,15,5);
+describe("Cloned Networks Equivalency", function () {
+  var original = new Perceptron(10, 15, 5);
   var cloned = original.clone();
 
   var learningRate = .5;
   var iterations = 1000;
 
-  for (var i = 1; i <= iterations; i++)
-  {
-      //random input
-      var input = generateRandomArray(10);
+  for (var i = 1; i <= iterations; i++) {
+    //random input
+    var input = generateRandomArray(10);
 
-      // activate networks
-      var output1 = original.activate(input);
-      var output2 = cloned.activate(input);
+    // activate networks
+    var output1 = original.activate(input);
+    var output2 = cloned.activate(input);
 
-      if (i % 100 == 0)
-        it(' same output for both networks after ' + i + ' iterations', function(){
-          var diff = false;
-          for (var k in output1)
-            if (output1[k] - output2[k] != 0)
-              diff = true;
-          assert(!diff);
-        });
+    if (i % 100 == 0)
+      it(' same output for both networks after ' + i + ' iterations', function () {
+        var diff = false;
+        for (var k in output1)
+          if (output1[k] - output2[k] != 0)
+            diff = true;
+        assert(!diff);
+      });
 
-      // random target
-      var target = generateRandomArray(5);
+    // random target
+    var target = generateRandomArray(5);
 
-      // propagate networks
-      original.propagate(learningRate, target);
-      cloned.propagate(learningRate, target);
+    // propagate networks
+    original.propagate(learningRate, target);
+    cloned.propagate(learningRate, target);
   }
 });
 
-describe("Manual Override", function() {
+describe("Manual Override", function () {
   var perceptron = new Perceptron(2, 3, 1);
 
-  it('iterations ended at full 3000', function(){
+  it('iterations ended at full 3000', function () {
     var final_stats = perceptron.trainer.XOR({
       iterations: 3000,
       rate: 0.000001,
       error: 0.000001,
       schedule: {
-          every: 1000,
-          do: function(data) {
-            if( data.iterations == 20000){
-              return true
-            }
+        every: 1000,
+        do: function (data) {
+          if (data.iterations == 20000) {
+            return true
           }
         }
+      }
     });
-    assert.equal( final_stats.iterations, 3000 )
+    assert.equal(final_stats.iterations, 3000)
   });
 
-  it('iterations ended at 2000, not full 3000', function(){
+  it('iterations ended at 2000, not full 3000', function () {
     var final_stats = perceptron.trainer.XOR({
       iterations: 3000,
       rate: 0.000001,
       error: 0.000001,
       schedule: {
-          every: 1000,
-          do: function(data) {
-            if( data.iterations == 2000){
-              return true
-            }
+        every: 1000,
+        do: function (data) {
+          if (data.iterations == 2000) {
+            return true
           }
         }
+      }
     });
-    assert.equal( final_stats.iterations, 2000 )
+    assert.equal(final_stats.iterations, 2000)
   });
 
-  it('training works even when schedule() has no return value', function(){
+  it('training works even when schedule() has no return value', function () {
     var final_stats = perceptron.trainer.XOR({
       iterations: 3000,
       rate: 0.000001,
       error: 0.000001,
       schedule: {
-          every: 1000,
-          do: function(data) {}
-        }
+        every: 1000,
+        do: function (data) { }
+      }
     });
-    assert.equal( final_stats.iterations, 3000 )
+    assert.equal(final_stats.iterations, 3000)
   });
 
-  it('using depreciated customLog still works', function(){
-    var counter = 0
+  it('using depreciated customLog still works', function () {
+    var counter = 0;
     var final_stats = perceptron.trainer.XOR({
       iterations: 3000,
       rate: 0.000001,
       error: 0.000001,
       customLog: {
-          every: 1000,
-          do: function(data) { counter++ }
-        }
+        every: 1000,
+        do: function (data) { counter++; }
+      }
     });
-    assert.equal( counter, 3 )
+    assert.equal(counter, 3)
   });
 
 });

tsconfig.json

@@ -12,8 +12,10 @@
         "src/squash.ts",
         "src/neuron.ts",
         "src/layer.ts",
+        "src/softmaxLayer.ts",
         "src/network.ts",
         "src/trainer.ts",
+        "src/utils.ts",
         "src/architect.ts",
         "src/architect/Perceptron.ts",
         "src/architect/LSTM.ts",