wellton/synaptic-do-wellton
1
0
Fork 0
Código Issues Pull Requests Ações Pacotes Projetos Versões Wiki Atividade

deleted ntm

Ver Código-Fonte
Esse commit está contido em:
Agustin Mendez (Menduz)
2015-06-28 18:50:34 -03:00
pai bc284c4369
commit f1a33e5ccc
10 arquivos alterados com 17 adições e 1112 exclusões
Baixar Arquivo de Patch Baixar Arquivo de Diff Expandir todos os arquivos Colapsar todos os arquivos
dist/src/architect.d.ts
externo
-2
Ver Arquivo
@@ -2,9 +2,7 @@ import hopfield = require('./architect/Hopfield');
import lstm = require('./architect/LSTM');
import lsm = require('./architect/Liquid');
import perceptron = require('./architect/Perceptron');
import mb = require('./architect/NTM');
export declare var LSTM: typeof lstm.LSTM;
export declare var Liquid: typeof lsm.Liquid;
export declare var Hopfield: typeof hopfield.Hopfield;
export declare var Perceptron: typeof perceptron.Perceptron;
export declare var NTM: typeof mb.NTM;
dist/src/architect.js
externo
+1 -3
Ver Arquivo
@@ -2,11 +2,9 @@ var hopfield = require('./architect/Hopfield');
var lstm = require('./architect/LSTM');
var lsm = require('./architect/Liquid');
var perceptron = require('./architect/Perceptron');
var mb = require('./architect/NTM');
exports.LSTM = lstm.LSTM;
exports.Liquid = lsm.Liquid;
exports.Hopfield = hopfield.Hopfield;
exports.Perceptron = perceptron.Perceptron;
exports.NTM = mb.NTM;
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbInNyYy9hcmNoaXRlY3QudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsSUFBTyxRQUFRLFdBQVcsc0JBQXNCLENBQUMsQ0FBQztBQUNsRCxJQUFPLElBQUksV0FBVyxrQkFBa0IsQ0FBQyxDQUFDO0FBQzFDLElBQU8sR0FBRyxXQUFXLG9CQUFvQixDQUFDLENBQUM7QUFDM0MsSUFBTyxVQUFVLFdBQVcsd0JBQXdCLENBQUMsQ0FBQztBQUN0RCxJQUFPLEVBQUUsV0FBVyxpQkFBaUIsQ0FBQyxDQUFDO0FBRTVCLFlBQUksR0FBRyxJQUFJLENBQUMsSUFBSSxDQUFDO0FBQ2pCLGNBQU0sR0FBRyxHQUFHLENBQUMsTUFBTSxDQUFDO0FBQ3BCLGdCQUFRLEdBQUcsUUFBUSxDQUFDLFFBQVEsQ0FBQztBQUM3QixrQkFBVSxHQUFHLFVBQVUsQ0FBQyxVQUFVLENBQUM7QUFDbkMsV0FBRyxHQUFHLEVBQUUsQ0FBQyxHQUFHLENBQUMiLCJmaWxlIjoic3JjL2FyY2hpdGVjdC5qcyIsInNvdXJjZXNDb250ZW50IjpbImltcG9ydCBob3BmaWVsZCA9IHJlcXVpcmUoJy4vYXJjaGl0ZWN0L0hvcGZpZWxkJyk7XG5pbXBvcnQgbHN0bSA9IHJlcXVpcmUoJy4vYXJjaGl0ZWN0L0xTVE0nKTtcbmltcG9ydCBsc20gPSByZXF1aXJlKCcuL2FyY2hpdGVjdC9MaXF1aWQnKTtcbmltcG9ydCBwZXJjZXB0cm9uID0gcmVxdWlyZSgnLi9hcmNoaXRlY3QvUGVyY2VwdHJvbicpO1xuaW1wb3J0IG1iID0gcmVxdWlyZSgnLi9hcmNoaXRlY3QvTlRNJyk7XG5cbmV4cG9ydCB2YXIgTFNUTSA9IGxzdG0uTFNUTTtcbmV4cG9ydCB2YXIgTGlxdWlkID0gbHNtLkxpcXVpZDtcbmV4cG9ydCB2YXIgSG9wZmllbGQgPSBob3BmaWVsZC5Ib3BmaWVsZDtcbmV4cG9ydCB2YXIgUGVyY2VwdHJvbiA9IHBlcmNlcHRyb24uUGVyY2VwdHJvbjtcbmV4cG9ydCB2YXIgTlRNID0gbWIuTlRNOyJdLCJzb3VyY2VSb290IjoiL3NvdXJjZS8ifQ==
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbInNyYy9hcmNoaXRlY3QudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsSUFBTyxRQUFRLFdBQVcsc0JBQXNCLENBQUMsQ0FBQztBQUNsRCxJQUFPLElBQUksV0FBVyxrQkFBa0IsQ0FBQyxDQUFDO0FBQzFDLElBQU8sR0FBRyxXQUFXLG9CQUFvQixDQUFDLENBQUM7QUFDM0MsSUFBTyxVQUFVLFdBQVcsd0JBQXdCLENBQUMsQ0FBQztBQUUzQyxZQUFJLEdBQUcsSUFBSSxDQUFDLElBQUksQ0FBQztBQUNqQixjQUFNLEdBQUcsR0FBRyxDQUFDLE1BQU0sQ0FBQztBQUNwQixnQkFBUSxHQUFHLFFBQVEsQ0FBQyxRQUFRLENBQUM7QUFDN0Isa0JBQVUsR0FBRyxVQUFVLENBQUMsVUFBVSxDQUFDIiwiZmlsZSI6InNyYy9hcmNoaXRlY3QuanMiLCJzb3VyY2VzQ29udGVudCI6WyJpbXBvcnQgaG9wZmllbGQgPSByZXF1aXJlKCcuL2FyY2hpdGVjdC9Ib3BmaWVsZCcpO1xuaW1wb3J0IGxzdG0gPSByZXF1aXJlKCcuL2FyY2hpdGVjdC9MU1RNJyk7XG5pbXBvcnQgbHNtID0gcmVxdWlyZSgnLi9hcmNoaXRlY3QvTGlxdWlkJyk7XG5pbXBvcnQgcGVyY2VwdHJvbiA9IHJlcXVpcmUoJy4vYXJjaGl0ZWN0L1BlcmNlcHRyb24nKTtcblxuZXhwb3J0IHZhciBMU1RNID0gbHN0bS5MU1RNO1xuZXhwb3J0IHZhciBMaXF1aWQgPSBsc20uTGlxdWlkO1xuZXhwb3J0IHZhciBIb3BmaWVsZCA9IGhvcGZpZWxkLkhvcGZpZWxkO1xuZXhwb3J0IHZhciBQZXJjZXB0cm9uID0gcGVyY2VwdHJvbi5QZXJjZXB0cm9uO1xuIl0sInNvdXJjZVJvb3QiOiIvc291cmNlLyJ9
dist/synaptic.js
externo
+12 -267
Ver Arquivo
@@ -3,14 +3,12 @@ var hopfield = require('./architect/Hopfield');
var lstm = require('./architect/LSTM');
var lsm = require('./architect/Liquid');
var perceptron = require('./architect/Perceptron');
var mb = require('./architect/NTM');
exports.LSTM = lstm.LSTM;
exports.Liquid = lsm.Liquid;
exports.Hopfield = hopfield.Hopfield;
exports.Perceptron = perceptron.Perceptron;
exports.NTM = mb.NTM;
},{"./architect/Hopfield":2,"./architect/LSTM":3,"./architect/Liquid":4,"./architect/NTM":5,"./architect/Perceptron":6}],2:[function(require,module,exports){
},{"./architect/Hopfield":2,"./architect/LSTM":3,"./architect/Liquid":4,"./architect/Perceptron":5}],2:[function(require,module,exports){
var __extends = this.__extends || function (d, b) {
for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p];
function __() { this.constructor = d; }
@@ -57,7 +55,7 @@ var Hopfield = (function (_super) {
})(network.Network);
exports.Hopfield = Hopfield;
},{"../layer":7,"../network":8,"../trainer":13}],3:[function(require,module,exports){
},{"../layer":6,"../network":7,"../trainer":11}],3:[function(require,module,exports){
var __extends = this.__extends || function (d, b) {
for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p];
function __() { this.constructor = d; }
@@ -180,7 +178,7 @@ var LSTM = (function (_super) {
exports.LSTM = LSTM;
;
},{"../layer":7,"../network":8,"../trainer":13}],4:[function(require,module,exports){
},{"../layer":6,"../network":7,"../trainer":11}],4:[function(require,module,exports){
var __extends = this.__extends || function (d, b) {
for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p];
function __() { this.constructor = d; }
@@ -231,190 +229,7 @@ var Liquid = (function (_super) {
})(network.Network);
exports.Liquid = Liquid;
},{"../layer":7,"../network":8,"../trainer":13}],5:[function(require,module,exports){
var __extends = this.__extends || function (d, b) {
for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p];
function __() { this.constructor = d; }
__.prototype = b.prototype;
d.prototype = new __();
};
var network = require('../network');
var trainer = require('../trainer');
var Layer = require('../layer');
var Squash = require('../squash');
var _utils = require('../utils');
var softmaxLayer = require('../softmaxLayer');
var Utils = _utils.Utils;
var NTM = (function (_super) {
__extends(NTM, _super);
function NTM(inputs, outputs, memBlocks, blockWidth, heads, hiddenSize) {
// build the memory
_super.call(this);
this.heads = new Array();
this.dirty = false;
this.trainer = new trainer.Trainer(this);
this.blocks = memBlocks;
this.blockWidth = blockWidth;
this.data = new Array(this.blocks);
for (var index = 0; index < this.data.length; index++) {
this.data[index] = new Float64Array(blockWidth);
}
this.clean();
// build the network
var inputLength = inputs + heads * memBlocks;
this.inputValues = new Float64Array(inputLength);
this.layers.input = this.inputLayer = new Layer.Layer(inputLength);
this.hiddenLayer = new Layer.Layer(hiddenSize);
this.layers.output = this.outputLayer = new Layer.Layer(outputs);
this.inputLayer.project(this.hiddenLayer, Layer.Layer.connectionType.ALL_TO_ALL);
this.hiddenLayer.project(this.outputLayer, Layer.Layer.connectionType.ALL_TO_ALL);
var inputCounter = inputs - 1;
for (var headIndex = 0; headIndex < heads; headIndex++) {
this.addHead(this.inputValues.subarray(inputCounter, inputCounter + memBlocks));
inputCounter += memBlocks;
}
this.optimized = false;
}
NTM.prototype.clean = function () {
for (var location = 0; location < this.blocks; location++) {
Utils.initRandomSoftmaxArray(this.data[location]);
}
this.dirty = false;
};
NTM.prototype.activate = function (input) {
this.inputValues.set(input);
this.inputLayer.activate(this.inputValues);
this.hiddenLayer.activate();
this.doTimeStep();
return this.outputLayer.activate();
};
NTM.prototype.propagate = function (rate, target) {
this.outputLayer.propagate(rate, target);
for (var i = this.heads.length - 1; i >= 0; i--) {
this.heads[i].shiftingLayer && this.heads[i].shiftingLayer.propagate(rate);
this.heads[i].layer.propagate(rate);
}
this.hiddenLayer.propagate(rate);
this.dirty = true;
};
NTM.prototype.addHead = function (subArray) {
var head = new Head(this, subArray);
this.heads.push(head);
return head;
};
NTM.prototype.doTimeStep = function () {
var _this = this;
this.heads.forEach(function (head, headIndex) {
head.doTimeStep();
});
// parallelizable
this.heads.forEach(function (head, headIndex) {
_this.doErase(head.w_weightings, head.eraseGate);
});
// parallelizable
this.heads.forEach(function (head, headIndex) {
_this.doAdd(head.w_weightings, head.addGate);
});
//this.data.forEach((e) => e = Utils.softMax(e))
};
NTM.prototype.doAdd = function (w, addGate) {
for (var n = 0; n < this.blocks; n++) {
var M = this.data[n];
for (var i = 0; i < this.blockWidth; i++) {
M[i] += addGate[n] * w[i];
}
}
};
NTM.prototype.doErase = function (w, eraseGate) {
for (var n = 0; n < this.blocks; n++) {
var M = this.data[n];
for (var i = 0; i < this.blockWidth; i++) {
M[i] *= 1 - eraseGate[n] * w[i];
}
}
};
return NTM;
})(network.Network);
exports.NTM = NTM;
var Head = (function () {
function Head(memory, destinationArray) {
this.s_shiftingValue = null;
this.prevFocus = 1;
this.memory = memory;
this.wc_focusedWeights = new Float64Array(this.memory.blocks);
this.w_weightings = new Float64Array(this.memory.blocks);
Utils.initRandomSoftmaxArray(this.w_weightings);
this.shiftLength = 3; //this.memory.blocks;
this.k_keys = new Float64Array(this.memory.blockWidth);
this.ß_keyStrength = 0;
this.eraseGate = new Float64Array(this.memory.blocks);
this.addGate = new Float64Array(this.memory.blocks);
this.readVector = destinationArray || new Float64Array(this.memory.blocks);
// Head layer
this.layer = new Layer.Layer(this.memory.blockWidth + this.memory.blocks * 3 + Head.ADDITIONAL_INPUT_VALUES, "NTM: Head layer");
this.memory.hiddenLayer.project(this.layer, Layer.Layer.connectionType.ALL_TO_ALL);
this.layer.project(this.memory.outputLayer, Layer.Layer.connectionType.ALL_TO_ALL);
// shifting layer
this.shiftingLayer = new softmaxLayer.SoftMaxLayer(this.shiftLength, "NTM: Shifting layer");
this.memory.hiddenLayer.project(this.shiftingLayer, Layer.Layer.connectionType.ALL_TO_ALL);
this.shiftingLayer.project(this.memory.hiddenLayer, Layer.Layer.connectionType.ALL_TO_ALL);
this.s_shiftingVector = this.shiftingLayer.currentActivation;
}
Head.prototype.readParams = function (activation) {
this.ß_keyStrength = activation[0];
this.g_interpolation = activation[1];
this.Y_focus = activation[2] + 1; //Squash.SOFTPLUS(activation[2]) + 1;
var startAt = 3;
for (var k = 0; k < this.k_keys.length; k++) {
this.k_keys[k] = this.layer.list[k + startAt].activation;
}
startAt += this.k_keys.length;
for (var k = 0; k < this.addGate.length; k++) {
this.addGate[k] = this.layer.list[k + startAt].activation;
}
startAt += this.addGate.length;
for (var k = 0; k < this.eraseGate.length; k++) {
this.eraseGate[k] = Squash.LOGISTIC(this.layer.list[k + startAt].activation);
}
var M = this.memory.data;
// focus by content, obtains an array of similarity indexes for each memoryBlock
for (var i = 0; i < M.length; i++)
this.wc_focusedWeights[i] = Utils.getCosineSimilarity(M[i], this.k_keys) * this.ß_keyStrength;
Utils.softMax(this.wc_focusedWeights);
// focus by location (interpolation)
Utils.interpolateArray(this.wc_focusedWeights, this.w_weightings, this.g_interpolation);
// convolutional shift
//this.doShiftings();
Utils.vectorInvertedShifting(this.wc_focusedWeights, this.s_shiftingVector);
// sharpening
Utils.sharpArray(this.w_weightings, this.wc_focusedWeights, this.Y_focus);
// since ∑ w = 1, we have to softmax the array
Utils.softMax(this.w_weightings);
/// we got wt!
};
Head.prototype.doShiftings = function () {
// call this fn in case of not using a softmaxLayer for shifting
Utils.softMax(this.s_shiftingVector);
Utils.vectorInvertedShifting(this.wc_focusedWeights, this.s_shiftingVector);
};
Head.prototype.doTimeStep = function () {
var activation = this.layer.activate();
this.shiftingLayer && this.shiftingLayer.activate();
this.readParams(activation);
// reading
for (var index = 0; index < this.memory.blocks; index++) {
this.readVector[index] = 0;
for (var cell = 0; cell < this.memory.blockWidth; cell++) {
this.readVector[index] += this.memory.data[index][cell] * this.w_weightings[index];
}
}
};
Head.ADDITIONAL_INPUT_VALUES = 3;
return Head;
})();
exports.Head = Head;
},{"../layer":7,"../network":8,"../softmaxLayer":10,"../squash":11,"../trainer":13,"../utils":14}],6:[function(require,module,exports){
},{"../layer":6,"../network":7,"../trainer":11}],5:[function(require,module,exports){
var __extends = this.__extends || function (d, b) {
for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p];
function __() { this.constructor = d; }
@@ -464,7 +279,7 @@ var Perceptron = (function (_super) {
exports.Perceptron = Perceptron;
;
},{"../layer":7,"../network":8,"../trainer":13}],7:[function(require,module,exports){
},{"../layer":6,"../network":7,"../trainer":11}],6:[function(require,module,exports){
var neuron = require('./neuron');
var network = require('./network');
/*******************************************************************************************
@@ -720,7 +535,7 @@ var Layer;
Layer.LayerConnection = LayerConnection;
})(Layer = exports.Layer || (exports.Layer = {}));
},{"./network":8,"./neuron":9}],8:[function(require,module,exports){
},{"./network":7,"./neuron":8}],7:[function(require,module,exports){
var layer = require('./layer');
var Squash = require('./squash');
var _neuron = require('./neuron');
@@ -1243,7 +1058,7 @@ var Network = (function () {
})();
exports.Network = Network;
},{"./layer":7,"./neuron":9,"./squash":11}],9:[function(require,module,exports){
},{"./layer":6,"./neuron":8,"./squash":9}],8:[function(require,module,exports){
/// <reference path="synaptic.ts" />
var Squash = require('./squash');
/******************************************************************************************
@@ -1928,77 +1743,7 @@ var Neuron;
})(Connection = Neuron.Connection || (Neuron.Connection = {}));
})(Neuron = exports.Neuron || (exports.Neuron = {}));
},{"./squash":11}],10:[function(require,module,exports){
var __extends = this.__extends || function (d, b) {
for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p];
function __() { this.constructor = d; }
__.prototype = b.prototype;
d.prototype = new __();
};
var Layer = require('./layer');
var Squash = require('./squash');
var _Utils = require('./utils');
var Utils = _Utils.Utils;
var SoftMaxLayer = (function (_super) {
__extends(SoftMaxLayer, _super);
function SoftMaxLayer(size, label) {
_super.call(this, size, label);
this.optimizable = false;
for (var n = 0; n < this.list.length; n++) {
this.list[n].squash = Squash.IDENTITY;
}
}
SoftMaxLayer.prototype.activate = function (input) {
if (this.currentActivation.length != this.list.length)
this.currentActivation = new Float64Array(this.list.length);
var activationIndex = 0;
var sum = 0;
var Amax = null;
if (typeof input != 'undefined') {
if (input.length != this.size)
throw "INPUT size and LAYER size must be the same to activate!";
Utils.softMax(input);
for (var id in this.list) {
this.list[id].readIncommingConnections(input[id]);
if (Amax === null || this.list[id].activation > Amax)
Amax = this.list[id].activation;
}
}
else {
for (var id in this.list) {
this.list[id].readIncommingConnections();
if (Amax === null || this.list[id].activation > Amax)
Amax = this.list[id].activation;
}
}
for (var n = 0; n < this.currentActivation.length; n++) {
sum += (this.list[n].activation = Math.exp(this.list[n].activation - Amax));
}
for (var n = 0; n < this.currentActivation.length; n++) {
// set the activations
var x = this.list[n].activation / sum;
this.list[n].activation = this.currentActivation[n] = x;
// set the derivatives
//x = this.list[n].activation / (sum - this.list[n].activation);
this.list[n].derivative = x * (1 - x);
this.list[n].updateTraces();
}
return this.currentActivation;
};
SoftMaxLayer.NormalizeConnectionWeights = function (layerConnection) {
var sum = 0;
for (var c = 0; c < layerConnection.list.length; c++) {
sum += (layerConnection.list[c].weight = Math.exp(layerConnection.list[c].weight));
}
for (var c = 0; c < layerConnection.list.length; c++) {
layerConnection.list[c].weight /= sum;
}
};
return SoftMaxLayer;
})(Layer.Layer);
exports.SoftMaxLayer = SoftMaxLayer;
},{"./layer":7,"./squash":11,"./utils":14}],11:[function(require,module,exports){
},{"./squash":9}],9:[function(require,module,exports){
// squashing functions
function LOGISTIC(x, derivate) {
if (derivate) {
@@ -2035,7 +1780,7 @@ function EXP(x, derivate) {
}
exports.EXP = EXP;
},{}],12:[function(require,module,exports){
},{}],10:[function(require,module,exports){
/*
********************************************************************************************
SYNAPTIC
@@ -2088,7 +1833,7 @@ if (typeof window != "undefined")
window['synaptic'] = Synaptic;
module.exports = Synaptic;
},{"./architect":1,"./layer":7,"./network":8,"./neuron":9,"./squash":11,"./trainer":13,"./utils":14}],13:[function(require,module,exports){
},{"./architect":1,"./layer":6,"./network":7,"./neuron":8,"./squash":9,"./trainer":11,"./utils":12}],11:[function(require,module,exports){
/*******************************************************************************************
TRAINER
*******************************************************************************************/
@@ -2625,7 +2370,7 @@ var Trainer;
};
})(Trainer = exports.Trainer || (exports.Trainer = {}));
},{}],14:[function(require,module,exports){
},{}],12:[function(require,module,exports){
var Utils = (function () {
function Utils() {
}
@@ -2802,5 +2547,5 @@ var Utils = (function () {
})();
exports.Utils = Utils;
},{}]},{},[12]);
},{}]},{},[10]);
var synaptic = synaptic || Synaptic;var Neuron = synaptic.Neuron, Layer = synaptic.Layer, Network = synaptic.Network, Trainer = synaptic.Trainer, Architect = synaptic.Architect;
dist/synaptic.min.js
externo
+2 -3
Ver Arquivo
Diff do arquivo suprimido porque uma ou mais linhas são muito longas
src/architect.ts
-2
Ver Arquivo
@@ -2,10 +2,8 @@ import hopfield = require('./architect/Hopfield');
import lstm = require('./architect/LSTM');
import lsm = require('./architect/Liquid');
import perceptron = require('./architect/Perceptron');
import mb = require('./architect/NTM');
export var LSTM = lstm.LSTM;
export var Liquid = lsm.Liquid;
export var Hopfield = hopfield.Hopfield;
export var Perceptron = perceptron.Perceptron;
export var NTM = mb.NTM;
src/architect/NTM.ts
-264
Ver Arquivo
@@ -1,264 +0,0 @@
import network = require('../network');
import trainer = require('../trainer');
import Layer = require('../layer');
import neuron = require('../neuron');
import Synaptic = require('../synaptic');
import Squash = require('../squash');
import _utils = require('../utils');
import softmaxLayer = require('../softmaxLayer');
var Utils = _utils.Utils;
export class NTM extends network.Network {
trainer: trainer.Trainer;
data: Float64Array[];
blockWidth: number;
blocks: number;
heads: Head[] = new Array();
inputValues: Float64Array;
inputLayer: Layer.Layer;
hiddenLayer: Layer.Layer;
outputLayer: Layer.Layer;
dirty = false;
constructor(inputs: number, outputs: number, memBlocks: number, blockWidth: number, heads: number, hiddenSize: number) {
// build the memory
super();
this.trainer = new trainer.Trainer(this);
this.blocks = memBlocks;
this.blockWidth = blockWidth;
this.data = new Array(this.blocks);
for (var index = 0; index < this.data.length; index++) {
this.data[index] = new Float64Array(blockWidth);
}
this.clean();
// build the network
var inputLength = inputs + heads * memBlocks;
this.inputValues = new Float64Array(inputLength);
this.layers.input = this.inputLayer = new Layer.Layer(inputLength);
this.hiddenLayer = new Layer.Layer(hiddenSize);
this.layers.output = this.outputLayer = new Layer.Layer(outputs);
this.inputLayer.project(this.hiddenLayer, Layer.Layer.connectionType.ALL_TO_ALL);
this.hiddenLayer.project(this.outputLayer, Layer.Layer.connectionType.ALL_TO_ALL);
var inputCounter = inputs - 1;
for (var headIndex = 0; headIndex < heads; headIndex++) {
this.addHead(this.inputValues.subarray(inputCounter, inputCounter + memBlocks));
inputCounter += memBlocks;
}
this.optimized = false;
}
clean() {
for (var location = 0; location < this.blocks; location++) {
Utils.initRandomSoftmaxArray(this.data[location]);
}
this.dirty = false;
}
activate(input: Synaptic.INumericArray) {
this.inputValues.set(<any>input);
this.inputLayer.activate(this.inputValues);
this.hiddenLayer.activate();
this.doTimeStep();
return this.outputLayer.activate();
}
propagate(rate: number, target: Synaptic.INumericArray) {
this.outputLayer.propagate(rate, target);
for (var i = this.heads.length - 1; i >= 0; i--) {
this.heads[i].shiftingLayer && this.heads[i].shiftingLayer.propagate(rate);
this.heads[i].layer.propagate(rate);
}
this.hiddenLayer.propagate(rate);
this.dirty = true;
}
addHead(subArray: Float64Array): Head {
var head = new Head(this, subArray);
this.heads.push(head);
return head;
}
doTimeStep() {
this.heads.forEach((head, headIndex) => {
head.doTimeStep();
});
// parallelizable
this.heads.forEach((head, headIndex) => {
this.doErase(head.w_weightings, head.eraseGate);
});
// parallelizable
this.heads.forEach((head, headIndex) => {
this.doAdd(head.w_weightings, head.addGate);
});
//this.data.forEach((e) => e = Utils.softMax(e))
}
doAdd(w: Synaptic.INumericArray, addGate: Synaptic.INumericArray) {
for (var n = 0; n < this.blocks; n++) {
var M = this.data[n];
for (var i = 0; i < this.blockWidth; i++) {
M[i] += addGate[n] * w[i];
}
}
}
doErase(w: Synaptic.INumericArray, eraseGate: Synaptic.INumericArray) {
for (var n = 0; n < this.blocks; n++) {
var M = this.data[n];
for (var i = 0; i < this.blockWidth; i++) {
M[i] *= 1 - eraseGate[n] * w[i];
}
}
}
}
export class Head {
static ADDITIONAL_INPUT_VALUES = 3;
memory: NTM;
w_weightings: Float64Array;
eraseGate: Float64Array;
addGate: Float64Array;
k_keys: Float64Array;
g_interpolation: number;
Y_focus: number;
s_shiftingValue: number = null;
s_shiftingVector: Float64Array;
wc_focusedWeights: Float64Array;
readVector: Float64Array;
ß_keyStrength: number;
prevFocus: number = 1;
shiftLength: number;
layer: Layer.Layer;
shiftingLayer: Layer.Layer;
constructor(memory: NTM, destinationArray?: Float64Array) {
this.memory = memory;
this.wc_focusedWeights = new Float64Array(this.memory.blocks);
this.w_weightings = new Float64Array(this.memory.blocks);
Utils.initRandomSoftmaxArray(this.w_weightings);
this.shiftLength = 3; //this.memory.blocks;
this.k_keys = new Float64Array(this.memory.blockWidth);
this.ß_keyStrength = 0;
this.eraseGate = new Float64Array(this.memory.blocks);
this.addGate = new Float64Array(this.memory.blocks);
this.readVector = destinationArray || new Float64Array(this.memory.blocks);
// Head layer
this.layer = new Layer.Layer(this.memory.blockWidth + this.memory.blocks * 3 + Head.ADDITIONAL_INPUT_VALUES, "NTM: Head layer");
this.memory.hiddenLayer.project(this.layer, Layer.Layer.connectionType.ALL_TO_ALL);
this.layer.project(this.memory.outputLayer, Layer.Layer.connectionType.ALL_TO_ALL);
// shifting layer
this.shiftingLayer = new softmaxLayer.SoftMaxLayer(this.shiftLength, "NTM: Shifting layer");
this.memory.hiddenLayer.project(this.shiftingLayer, Layer.Layer.connectionType.ALL_TO_ALL);
this.shiftingLayer.project(this.memory.hiddenLayer, Layer.Layer.connectionType.ALL_TO_ALL);
this.s_shiftingVector = this.shiftingLayer.currentActivation;
}
private readParams(activation: Synaptic.INumericArray) {
this.ß_keyStrength = activation[0];
this.g_interpolation = activation[1];
this.Y_focus = activation[2] + 1;//Squash.SOFTPLUS(activation[2]) + 1;
var startAt = 3;
for (var k = 0; k < this.k_keys.length; k++) {
this.k_keys[k] = this.layer.list[k + startAt].activation;
}
startAt += this.k_keys.length;
for (var k = 0; k < this.addGate.length; k++) {
this.addGate[k] = this.layer.list[k + startAt].activation;
}
startAt += this.addGate.length;
for (var k = 0; k < this.eraseGate.length; k++) {
this.eraseGate[k] = Squash.LOGISTIC(this.layer.list[k + startAt].activation);
}
var M = this.memory.data;
// focus by content, obtains an array of similarity indexes for each memoryBlock
for (var i = 0; i < M.length; i++)
this.wc_focusedWeights[i] = Utils.getCosineSimilarity(M[i], this.k_keys) * this.ß_keyStrength;
Utils.softMax(this.wc_focusedWeights);
// focus by location (interpolation)
Utils.interpolateArray(this.wc_focusedWeights, this.w_weightings, this.g_interpolation);
// convolutional shift
//this.doShiftings();
Utils.vectorInvertedShifting(this.wc_focusedWeights, this.s_shiftingVector);
// sharpening
Utils.sharpArray(this.w_weightings, this.wc_focusedWeights, this.Y_focus);
// since ∑ w = 1, we have to softmax the array
Utils.softMax(this.w_weightings);
/// we got wt!
}
doShiftings() {
// call this fn in case of not using a softmaxLayer for shifting
Utils.softMax(this.s_shiftingVector);
Utils.vectorInvertedShifting(this.wc_focusedWeights, this.s_shiftingVector);
}
doTimeStep() {
var activation = this.layer.activate();
this.shiftingLayer && this.shiftingLayer.activate();
this.readParams(activation);
// reading
for (var index = 0; index < this.memory.blocks; index++) {
this.readVector[index] = 0;
for (var cell = 0; cell < this.memory.blockWidth; cell++) {
this.readVector[index] += this.memory.data[index][cell] * this.w_weightings[index];
}
}
}
}
test/ntm.js
+1 -9
Ver Arquivo
@@ -70,15 +70,7 @@ describe("NTM Utils", function() {
assert.equal(sumAll(Utils.softMax(fixedArray)), 1, 'Random signed array equals 1');
});
/*
it('Generate circulant matrix', function(){
var fixedArray = new Float64Array(5);
fixedArray.set([0,0,0,0,1]);
assert.equal(Utils.vectorShifting(fixedArray, [0, 1, 0]), [0,0,0,0,1], 'Unchanged shiftings');
});
*/
it('Shifting', function(){
var fixedArray = new Float64Array(5);
test/testMemblock.html
-348
Ver Arquivo
@@ -1,348 +0,0 @@
<html>
<head>
<title>Training a neural network to complete a Discrete Sequence Recall task - Synaptic</title>
<meta charset="UTF-8">
<script src="../dist/synaptic.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.1.4/jquery.min.js" ></script>
<style>
body {
font-family: 'monaco','Open Sans', sans-serif;
position: relative;
line-height: 120%;
margin: 0px;
background: #000;
color: #0f0;
}
.container {
width: 1024px;
margin: 0 auto;
}
.title {
text-align: center;
color: rgb(125, 125, 125);
font-size: 34px;
padding-top: 50px;
height: 50px;
cursor: pointer;
}
.title b{
color:rgb(50, 50, 50);;
}
.title:hover{
opacity: .8;
}
.subtitle {
font-weight: bold;
}
p, li {
padding: 4px;
font-size: 13px;
text-align: justify;
}
.code, .log {
border-left: 5px solid rgb(220, 220, 220);
padding-left: 20px;
color: rgb(150, 150, 150);
line-height: 1.2;
white-space: pre;
font-family: "Lucida Console", Monaco, monospace;
font-size: 12px;
}
.log {
margin: 0px;
line-height: 3px;
font-size: 11px;
}
.validate {
width: 16;
height: 13;
display: inline-block;
position: relative;
top: 1px;
}
.tick {
background-image: url(img/tick.png);
}
.fail{
background-image: url(img/fail.png);
width: 12px;
height: 12px;
top: 1px;
position: relative;
}
.square{
width: 12px;
height: 12px;
border-radius: 12px;
top: 2px;
position: relative;
border: 1px solid rgb(68, 68, 68);
display: inline-block;
box-shadow: inset 0 2px 1px rgba(255,255,255,0.8);
}
.log .square{
top: -5px;
}
#footer {
position: absolute;
font-size: 11px;
bottom: 10px;
margin-left: 50%;
left: -245px;
}
a{
color: black;
font-weight: bold;
}
</style>
<script>
var LSTM = null;
var targets = [2,4];
var distractors = [3,5];
var prompts = [0, 1];
var length = 10;
var sequence = null;
var sequenceLength = 0;
var targetsCorrect = 0;
var distractorsCorrect = 0;
var indexes = [];
var positions = [];
var symbol = 0;
var symbols = targets.length + distractors.length + prompts.length;
var json = null;
var i = 0;
var canv = null;
var ctx = null;
var log = function(text, append){
var p = $('<p>', {'class' : 'log'}).html(text).appendTo($(".container")).hide().fadeIn();
if (append)
p.appendTo($("#" + append));
}
var drawMemory = function(index,rgb){
ctx.fillStyle = 'black';
ctx.fillRect(0,0,512,128);
var memory = LSTM.data;
for (var memBlock = 0; memBlock < memory.length; memBlock++) {
var mem = memory[memBlock];
for (var wM = 0; wM < mem.length; wM++) {
ctx.fillStyle = 'rgb(0,0,' + (mem[wM] * 255).toFixed() + ')';
ctx.fillRect(memBlock * 9, wM*9,8,8);
}
}
}
var train = function(){
canv = document.getElementById('canvasMem');
ctx = canv.getContext('2d');
$("#aux").remove();
$('.log').remove();
$("#log-validate").remove();
var squareTargets =[];
for (var i in targets)
squareTargets[i] = square(targets[i]);
log("Targets: " + squareTargets.join(' '));
var squareDistractors =[];
for (var i in distractors)
squareDistractors[i] = square(distractors[i]);
log("Distractors: " + squareDistractors.join(' '));
var squarePrompts =[];
for (var i in prompts)
squarePrompts[i] = square(prompts[i]);
log("Prompts: " + squarePrompts.join(' '));
log("Length: " + length);
log ("&nbsp");
log("Training DSR...");
log ("&nbsp");
$("body").animate({ scrollTop: $(document).height() }, function()
{
var inputs = symbols;
var outputs = targets.length;
var hidden = inputs * .8 | 0;
LSTM = new Architect.NTM(inputs,outputs,symbols,symbols,1,hidden);
var results = LSTM.trainer.DSR({
targets: targets,
distractors: distractors,
prompts: prompts,
length: length,
iterations: 100,
rate: .17,
schedule: {
every: 2,
do: function(obj){
drawMemory();
console.log(' Error: ' + obj.error);
}
}
});
if (results.iterations < 250000)
{
log("Trained!");
log("- Iterations: " + results.iterations);
log("- Error: " + results.error);
log("- Elapsed time: " + results.time + " ms");
log("- Success rate: " + (results.success*100).toFixed(2) + "%");
$('<div>', { id: 'log-validate', style:'height:450px;'}).appendTo(".container");
validate();
} else{
log ("&nbsp");
log ("Failed...");
log ("&nbsp");
log ("&nbsp");
console.log('failed');
train()
}
});
}
var validate = function(){
//$(document.body).css("min-height", 1349);
$("#log-validate").html('');
log ("&nbsp", "log-validate");
log("Validating...", "log-validate");
sequence = [], sequenceLength = length - prompts.length;
for (i = 0; i < sequenceLength; i ++)
{
var any = Math.random() * distractors.length | 0;
sequence.push(distractors[any]);
}
indexes = [], positions = [];
for (i = 0; i < prompts.length; i ++)
{
indexes.push(Math.random() * targets.length | 0);
positions.push(noRepeat(sequenceLength, positions));
}
positions = positions.sort();
for (i = 0; i < prompts.length; i ++)
{
sequence[positions[i]] = targets[indexes[i]];
sequence.push(prompts[i]);
}
targetsCorrect = distractorsCorrect = symbol = 0;
log ("&nbsp", "log-validate");
var squareSecuence =[];
for (var i in sequence)
squareSecuence[i] = square(sequence[i]);
log("Sequence: " + squareSecuence.join(' '), "log-validate");
next();
}
var next = function(){
// generate input from sequence
var input = [];
for (var j = 0; j < symbols; j++)
input[j] = 0;
input[sequence[symbol]] = 1;
// generate target output
var output = [];
for (j = 0; j < targets.length; j++)
output[j] = 0;
if (symbol >= sequenceLength)
{
var index = symbol - sequenceLength;
output[indexes[index]] = 1;
}
// check result
var prediction = LSTM.activate(input);
var ok = equal(prediction, output);
if (ok)
if (symbol < sequenceLength)
distractorsCorrect++;
else
targetsCorrect++;
val = value(prediction);
log((symbol+1) + ") input: " + square(sequence[symbol]) + " output: " + (val != "none" ? square(targets[val]) : "--") + (ok ? ' <span class="validate tick"></span>' : ' <span class="validate fail"></span>'), "log-validate")
symbol++;
if (symbol < sequence.length)
setTimeout(next, 100);
else {
if (distractorsCorrect + targetsCorrect == length)
{
log ("&nbsp", "log-validate");
log ("Success! <button onclick='validate();'>Try another sequence</button>", "log-validate");
$("body").animate({ scrollTop: $(document).height() }, "slow");
} else {
log ("&nbsp", "log-validate");
log ("Failed...", "log-validate");
validate();
}
}
}
var noRepeat = function(range, avoid)
{
var number = Math.random() * range | 0;
var used = false;
for (var i in avoid)
if (number == avoid[i])
used = true;
return used ? noRepeat(range, avoid) : number;
}
var equal = function(prediction, output){
for (var i in prediction)
if (Math.round(prediction[i]) != output[i])
return false;
return true;
}
var value = function(array){
var max = .5;
var res = -1;
for (var i in array)
if (array[i] > max)
{
max = array[i];
res = i;
}
return res == -1 ? 'none' : res;
}
var colors = ['white', 'black', 'green', 'yellow', 'red', 'blue', 'purple', 'orange', 'gray', 'lightblue'];
var square = function(value){
return '<span class="square" style="background-color:' + colors[value] + '">&nbsp</span>';
}
</script>
</head>
<body>
<canvas id="canvasMem" width=512 height=128></canvas>
<div class="container"></div>
<p> Start training: <button onclick="train();">START</button></p>
<p id="aux">&nbsp;</p>
</div>
</body>
</html>
test/test_NTM_CopyTask.html
-212
Ver Arquivo
@@ -1,212 +0,0 @@
<html>
<head>
<title>Training a neural network to complete a Discrete Sequence Recall task - Synaptic</title>
<meta charset="UTF-8">
<script src="../dist/synaptic.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.1.4/jquery.min.js" ></script>
<style>
body {
font-family: 'monaco','Open Sans', sans-serif;
position: relative;
line-height: 120%;
margin: 0px;
background: #000;
color: #0f0;
}
.container {
width: 1024px;
margin: 0 auto;
}
.title {
text-align: center;
color: rgb(125, 125, 125);
font-size: 34px;
padding-top: 50px;
height: 50px;
cursor: pointer;
}
.title b{
color:rgb(50, 50, 50);;
}
.title:hover{
opacity: .8;
}
.subtitle {
font-weight: bold;
}
p, li {
padding: 4px;
font-size: 13px;
text-align: justify;
}
.code, .log {
border-left: 5px solid rgb(220, 220, 220);
padding-left: 20px;
color: rgb(150, 150, 150);
line-height: 1.2;
white-space: pre;
font-family: 'monaco', "Lucida Console", Monaco, monospace;
font-size: 12px;
}
.log {
margin: 0px;
line-height: 3px;
font-size: 11px;
}
.validate {
width: 16;
height: 13;
display: inline-block;
position: relative;
top: 1px;
}
.tick {
background-image: url(img/tick.png);
}
.fail{
background-image: url(img/fail.png);
width: 12px;
height: 12px;
top: 1px;
position: relative;
}
.square{
width: 12px;
height: 12px;
border-radius: 12px;
top: 2px;
position: relative;
border: 1px solid rgb(68, 68, 68);
display: inline-block;
box-shadow: inset 0 2px 1px rgba(255,255,255,0.8);
}
.log .square{
top: -5px;
}
#footer {
position: absolute;
font-size: 11px;
bottom: 10px;
margin-left: 50%;
left: -245px;
}
a{
color: black;
font-weight: bold;
}
</style>
<script>
var seqWidth = 9;
var seqLength = 10;
var targetSequence = [];
var trainLimit = 60;
var LSTM = new Architect.NTM(seqWidth,seqWidth,4,4,1,9);
var canv = null;
var ctx = null;
for (var i = 0; i < seqLength; i++) {
var arr = new Float64Array(seqWidth);
targetSequence.push(arr);
for(var n = 0; n < seqWidth - 1; n++){
arr[n] = Math.random() > 0.5 ? 1 : 0;
}
}
var arr = new Float64Array(seqWidth);
arr[seqWidth - 1] = 1;
targetSequence.push(arr);
var log = function(text, append){
var p = $('<p>', {'class' : 'log'}).html(text).appendTo($(".container")).hide().fadeIn();
if (append)
p.appendTo($("#" + append));
}
var drawMemory = function(index,rgb){
if(index == 0){
ctx.fillStyle = 'black';
ctx.fillRect(0, 0, 512, 512);
}
var memory = LSTM.data;
for (var memBlock = 0; memBlock < memory.length; memBlock++) {
var mem = memory[memBlock];
alto = mem.length + 15;
for (var wM = 0; wM < mem.length; wM++) {
ctx.fillStyle = 'rgb(0,0,' + (mem[wM] * 255).toFixed() + ')';
ctx.fillRect(memBlock * 9, wM*9,8,8);
}
}
var focus = LSTM.heads[0].wc_focusedWeights;
for (var wM = 0; wM < focus.length; wM++) {
var r = (focus[wM] * 255).toFixed();
ctx.fillStyle = 'rgb(' + r + ',' + r + ',' + r + ')';
ctx.fillRect( (wM)*9,(memory.length + index + 1) * 9,8,8);
}
}
var train = function(){
LSTM.clean();
doTraining(0);
}
var execute = function(){
for(var i = 0; i < targetSequence.length; i++){
console.log('Result', LSTM.activate(targetSequence[i]), ' Expected:', targetSequence[i]);
}
}
var doTraining = function(num){
if(num > trainLimit) return execute();
for(var i = 0; i < targetSequence.length; i++){
LSTM.activate(targetSequence[i]);
}
LSTM.propagate(0.01, targetSequence[num % targetSequence.length]);
drawMemory(num);
console.log(num, 'of', trainLimit);
requestAnimationFrame(function(){
doTraining(num + 1);
});
}
$(function(){
canv = document.getElementById('canvasMem');
ctx = canv.getContext('2d');
drawMemory();
});
</script>
</head>
<body>
<canvas id="canvasMem" width=512 height=512></canvas>
<div class="container"></div>
<p> Start training: <button onclick="train();">START</button></p>
<p id="aux">&nbsp;</p>
</div>
</body>
</html>
tsconfig.json
+1 -2
Ver Arquivo
@@ -20,7 +20,6 @@
"src/architect/Perceptron.ts",
"src/architect/LSTM.ts",
"src/architect/Liquid.ts",
"src/architect/Hopfield.ts",
"src/architect/NTM.ts"
"src/architect/Hopfield.ts"
]
}