Comparar commits
1 Commits
| Autor | SHA1 | Data | |
|---|---|---|---|
 Agustin Mendez (Menduz)
|
f1a33e5ccc |
externo
-2
@@ -2,9 +2,7 @@ import hopfield = require('./architect/Hopfield');
|
||||
import lstm = require('./architect/LSTM');
|
||||
import lsm = require('./architect/Liquid');
|
||||
import perceptron = require('./architect/Perceptron');
|
||||
import mb = require('./architect/NTM');
|
||||
export declare var LSTM: typeof lstm.LSTM;
|
||||
export declare var Liquid: typeof lsm.Liquid;
|
||||
export declare var Hopfield: typeof hopfield.Hopfield;
|
||||
export declare var Perceptron: typeof perceptron.Perceptron;
|
||||
export declare var NTM: typeof mb.NTM;
|
||||
|
||||
externo
+1
-3
@@ -2,11 +2,9 @@ var hopfield = require('./architect/Hopfield');
|
||||
var lstm = require('./architect/LSTM');
|
||||
var lsm = require('./architect/Liquid');
|
||||
var perceptron = require('./architect/Perceptron');
|
||||
var mb = require('./architect/NTM');
|
||||
exports.LSTM = lstm.LSTM;
|
||||
exports.Liquid = lsm.Liquid;
|
||||
exports.Hopfield = hopfield.Hopfield;
|
||||
exports.Perceptron = perceptron.Perceptron;
|
||||
exports.NTM = mb.NTM;
|
||||
|
||||
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbInNyYy9hcmNoaXRlY3QudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsSUFBTyxRQUFRLFdBQVcsc0JBQXNCLENBQUMsQ0FBQztBQUNsRCxJQUFPLElBQUksV0FBVyxrQkFBa0IsQ0FBQyxDQUFDO0FBQzFDLElBQU8sR0FBRyxXQUFXLG9CQUFvQixDQUFDLENBQUM7QUFDM0MsSUFBTyxVQUFVLFdBQVcsd0JBQXdCLENBQUMsQ0FBQztBQUN0RCxJQUFPLEVBQUUsV0FBVyxpQkFBaUIsQ0FBQyxDQUFDO0FBRTVCLFlBQUksR0FBRyxJQUFJLENBQUMsSUFBSSxDQUFDO0FBQ2pCLGNBQU0sR0FBRyxHQUFHLENBQUMsTUFBTSxDQUFDO0FBQ3BCLGdCQUFRLEdBQUcsUUFBUSxDQUFDLFFBQVEsQ0FBQztBQUM3QixrQkFBVSxHQUFHLFVBQVUsQ0FBQyxVQUFVLENBQUM7QUFDbkMsV0FBRyxHQUFHLEVBQUUsQ0FBQyxHQUFHLENBQUMiLCJmaWxlIjoic3JjL2FyY2hpdGVjdC5qcyIsInNvdXJjZXNDb250ZW50IjpbImltcG9ydCBob3BmaWVsZCA9IHJlcXVpcmUoJy4vYXJjaGl0ZWN0L0hvcGZpZWxkJyk7XG5pbXBvcnQgbHN0bSA9IHJlcXVpcmUoJy4vYXJjaGl0ZWN0L0xTVE0nKTtcbmltcG9ydCBsc20gPSByZXF1aXJlKCcuL2FyY2hpdGVjdC9MaXF1aWQnKTtcbmltcG9ydCBwZXJjZXB0cm9uID0gcmVxdWlyZSgnLi9hcmNoaXRlY3QvUGVyY2VwdHJvbicpO1xuaW1wb3J0IG1iID0gcmVxdWlyZSgnLi9hcmNoaXRlY3QvTlRNJyk7XG5cbmV4cG9ydCB2YXIgTFNUTSA9IGxzdG0uTFNUTTtcbmV4cG9ydCB2YXIgTGlxdWlkID0gbHNtLkxpcXVpZDtcbmV4cG9ydCB2YXIgSG9wZmllbGQgPSBob3BmaWVsZC5Ib3BmaWVsZDtcbmV4cG9ydCB2YXIgUGVyY2VwdHJvbiA9IHBlcmNlcHRyb24uUGVyY2VwdHJvbjtcbmV4cG9ydCB2YXIgTlRNID0gbWIuTlRNOyJdLCJzb3VyY2VSb290IjoiL3NvdXJjZS8ifQ==
|
||||
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbInNyYy9hcmNoaXRlY3QudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsSUFBTyxRQUFRLFdBQVcsc0JBQXNCLENBQUMsQ0FBQztBQUNsRCxJQUFPLElBQUksV0FBVyxrQkFBa0IsQ0FBQyxDQUFDO0FBQzFDLElBQU8sR0FBRyxXQUFXLG9CQUFvQixDQUFDLENBQUM7QUFDM0MsSUFBTyxVQUFVLFdBQVcsd0JBQXdCLENBQUMsQ0FBQztBQUUzQyxZQUFJLEdBQUcsSUFBSSxDQUFDLElBQUksQ0FBQztBQUNqQixjQUFNLEdBQUcsR0FBRyxDQUFDLE1BQU0sQ0FBQztBQUNwQixnQkFBUSxHQUFHLFFBQVEsQ0FBQyxRQUFRLENBQUM7QUFDN0Isa0JBQVUsR0FBRyxVQUFVLENBQUMsVUFBVSxDQUFDIiwiZmlsZSI6InNyYy9hcmNoaXRlY3QuanMiLCJzb3VyY2VzQ29udGVudCI6WyJpbXBvcnQgaG9wZmllbGQgPSByZXF1aXJlKCcuL2FyY2hpdGVjdC9Ib3BmaWVsZCcpO1xuaW1wb3J0IGxzdG0gPSByZXF1aXJlKCcuL2FyY2hpdGVjdC9MU1RNJyk7XG5pbXBvcnQgbHNtID0gcmVxdWlyZSgnLi9hcmNoaXRlY3QvTGlxdWlkJyk7XG5pbXBvcnQgcGVyY2VwdHJvbiA9IHJlcXVpcmUoJy4vYXJjaGl0ZWN0L1BlcmNlcHRyb24nKTtcblxuZXhwb3J0IHZhciBMU1RNID0gbHN0bS5MU1RNO1xuZXhwb3J0IHZhciBMaXF1aWQgPSBsc20uTGlxdWlkO1xuZXhwb3J0IHZhciBIb3BmaWVsZCA9IGhvcGZpZWxkLkhvcGZpZWxkO1xuZXhwb3J0IHZhciBQZXJjZXB0cm9uID0gcGVyY2VwdHJvbi5QZXJjZXB0cm9uO1xuIl0sInNvdXJjZVJvb3QiOiIvc291cmNlLyJ9
|
||||
externo
+12
-267
@@ -3,14 +3,12 @@ var hopfield = require('./architect/Hopfield');
|
||||
var lstm = require('./architect/LSTM');
|
||||
var lsm = require('./architect/Liquid');
|
||||
var perceptron = require('./architect/Perceptron');
|
||||
var mb = require('./architect/NTM');
|
||||
exports.LSTM = lstm.LSTM;
|
||||
exports.Liquid = lsm.Liquid;
|
||||
exports.Hopfield = hopfield.Hopfield;
|
||||
exports.Perceptron = perceptron.Perceptron;
|
||||
exports.NTM = mb.NTM;
|
||||
|
||||
},{"./architect/Hopfield":2,"./architect/LSTM":3,"./architect/Liquid":4,"./architect/NTM":5,"./architect/Perceptron":6}],2:[function(require,module,exports){
|
||||
},{"./architect/Hopfield":2,"./architect/LSTM":3,"./architect/Liquid":4,"./architect/Perceptron":5}],2:[function(require,module,exports){
|
||||
var __extends = this.__extends || function (d, b) {
|
||||
for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p];
|
||||
function __() { this.constructor = d; }
|
||||
@@ -57,7 +55,7 @@ var Hopfield = (function (_super) {
|
||||
})(network.Network);
|
||||
exports.Hopfield = Hopfield;
|
||||
|
||||
},{"../layer":7,"../network":8,"../trainer":13}],3:[function(require,module,exports){
|
||||
},{"../layer":6,"../network":7,"../trainer":11}],3:[function(require,module,exports){
|
||||
var __extends = this.__extends || function (d, b) {
|
||||
for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p];
|
||||
function __() { this.constructor = d; }
|
||||
@@ -180,7 +178,7 @@ var LSTM = (function (_super) {
|
||||
exports.LSTM = LSTM;
|
||||
;
|
||||
|
||||
},{"../layer":7,"../network":8,"../trainer":13}],4:[function(require,module,exports){
|
||||
},{"../layer":6,"../network":7,"../trainer":11}],4:[function(require,module,exports){
|
||||
var __extends = this.__extends || function (d, b) {
|
||||
for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p];
|
||||
function __() { this.constructor = d; }
|
||||
@@ -231,190 +229,7 @@ var Liquid = (function (_super) {
|
||||
})(network.Network);
|
||||
exports.Liquid = Liquid;
|
||||
|
||||
},{"../layer":7,"../network":8,"../trainer":13}],5:[function(require,module,exports){
|
||||
var __extends = this.__extends || function (d, b) {
|
||||
for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p];
|
||||
function __() { this.constructor = d; }
|
||||
__.prototype = b.prototype;
|
||||
d.prototype = new __();
|
||||
};
|
||||
var network = require('../network');
|
||||
var trainer = require('../trainer');
|
||||
var Layer = require('../layer');
|
||||
var Squash = require('../squash');
|
||||
var _utils = require('../utils');
|
||||
var softmaxLayer = require('../softmaxLayer');
|
||||
var Utils = _utils.Utils;
|
||||
var NTM = (function (_super) {
|
||||
__extends(NTM, _super);
|
||||
function NTM(inputs, outputs, memBlocks, blockWidth, heads, hiddenSize) {
|
||||
// build the memory
|
||||
_super.call(this);
|
||||
this.heads = new Array();
|
||||
this.dirty = false;
|
||||
this.trainer = new trainer.Trainer(this);
|
||||
this.blocks = memBlocks;
|
||||
this.blockWidth = blockWidth;
|
||||
this.data = new Array(this.blocks);
|
||||
for (var index = 0; index < this.data.length; index++) {
|
||||
this.data[index] = new Float64Array(blockWidth);
|
||||
}
|
||||
this.clean();
|
||||
// build the network
|
||||
var inputLength = inputs + heads * memBlocks;
|
||||
this.inputValues = new Float64Array(inputLength);
|
||||
this.layers.input = this.inputLayer = new Layer.Layer(inputLength);
|
||||
this.hiddenLayer = new Layer.Layer(hiddenSize);
|
||||
this.layers.output = this.outputLayer = new Layer.Layer(outputs);
|
||||
this.inputLayer.project(this.hiddenLayer, Layer.Layer.connectionType.ALL_TO_ALL);
|
||||
this.hiddenLayer.project(this.outputLayer, Layer.Layer.connectionType.ALL_TO_ALL);
|
||||
var inputCounter = inputs - 1;
|
||||
for (var headIndex = 0; headIndex < heads; headIndex++) {
|
||||
this.addHead(this.inputValues.subarray(inputCounter, inputCounter + memBlocks));
|
||||
inputCounter += memBlocks;
|
||||
}
|
||||
this.optimized = false;
|
||||
}
|
||||
NTM.prototype.clean = function () {
|
||||
for (var location = 0; location < this.blocks; location++) {
|
||||
Utils.initRandomSoftmaxArray(this.data[location]);
|
||||
}
|
||||
this.dirty = false;
|
||||
};
|
||||
NTM.prototype.activate = function (input) {
|
||||
this.inputValues.set(input);
|
||||
this.inputLayer.activate(this.inputValues);
|
||||
this.hiddenLayer.activate();
|
||||
this.doTimeStep();
|
||||
return this.outputLayer.activate();
|
||||
};
|
||||
NTM.prototype.propagate = function (rate, target) {
|
||||
this.outputLayer.propagate(rate, target);
|
||||
for (var i = this.heads.length - 1; i >= 0; i--) {
|
||||
this.heads[i].shiftingLayer && this.heads[i].shiftingLayer.propagate(rate);
|
||||
this.heads[i].layer.propagate(rate);
|
||||
}
|
||||
this.hiddenLayer.propagate(rate);
|
||||
this.dirty = true;
|
||||
};
|
||||
NTM.prototype.addHead = function (subArray) {
|
||||
var head = new Head(this, subArray);
|
||||
this.heads.push(head);
|
||||
return head;
|
||||
};
|
||||
NTM.prototype.doTimeStep = function () {
|
||||
var _this = this;
|
||||
this.heads.forEach(function (head, headIndex) {
|
||||
head.doTimeStep();
|
||||
});
|
||||
// parallelizable
|
||||
this.heads.forEach(function (head, headIndex) {
|
||||
_this.doErase(head.w_weightings, head.eraseGate);
|
||||
});
|
||||
// parallelizable
|
||||
this.heads.forEach(function (head, headIndex) {
|
||||
_this.doAdd(head.w_weightings, head.addGate);
|
||||
});
|
||||
//this.data.forEach((e) => e = Utils.softMax(e))
|
||||
};
|
||||
NTM.prototype.doAdd = function (w, addGate) {
|
||||
for (var n = 0; n < this.blocks; n++) {
|
||||
var M = this.data[n];
|
||||
for (var i = 0; i < this.blockWidth; i++) {
|
||||
M[i] += addGate[n] * w[i];
|
||||
}
|
||||
}
|
||||
};
|
||||
NTM.prototype.doErase = function (w, eraseGate) {
|
||||
for (var n = 0; n < this.blocks; n++) {
|
||||
var M = this.data[n];
|
||||
for (var i = 0; i < this.blockWidth; i++) {
|
||||
M[i] *= 1 - eraseGate[n] * w[i];
|
||||
}
|
||||
}
|
||||
};
|
||||
return NTM;
|
||||
})(network.Network);
|
||||
exports.NTM = NTM;
|
||||
var Head = (function () {
|
||||
function Head(memory, destinationArray) {
|
||||
this.s_shiftingValue = null;
|
||||
this.prevFocus = 1;
|
||||
this.memory = memory;
|
||||
this.wc_focusedWeights = new Float64Array(this.memory.blocks);
|
||||
this.w_weightings = new Float64Array(this.memory.blocks);
|
||||
Utils.initRandomSoftmaxArray(this.w_weightings);
|
||||
this.shiftLength = 3; //this.memory.blocks;
|
||||
this.k_keys = new Float64Array(this.memory.blockWidth);
|
||||
this.ß_keyStrength = 0;
|
||||
this.eraseGate = new Float64Array(this.memory.blocks);
|
||||
this.addGate = new Float64Array(this.memory.blocks);
|
||||
this.readVector = destinationArray || new Float64Array(this.memory.blocks);
|
||||
// Head layer
|
||||
this.layer = new Layer.Layer(this.memory.blockWidth + this.memory.blocks * 3 + Head.ADDITIONAL_INPUT_VALUES, "NTM: Head layer");
|
||||
this.memory.hiddenLayer.project(this.layer, Layer.Layer.connectionType.ALL_TO_ALL);
|
||||
this.layer.project(this.memory.outputLayer, Layer.Layer.connectionType.ALL_TO_ALL);
|
||||
// shifting layer
|
||||
this.shiftingLayer = new softmaxLayer.SoftMaxLayer(this.shiftLength, "NTM: Shifting layer");
|
||||
this.memory.hiddenLayer.project(this.shiftingLayer, Layer.Layer.connectionType.ALL_TO_ALL);
|
||||
this.shiftingLayer.project(this.memory.hiddenLayer, Layer.Layer.connectionType.ALL_TO_ALL);
|
||||
this.s_shiftingVector = this.shiftingLayer.currentActivation;
|
||||
}
|
||||
Head.prototype.readParams = function (activation) {
|
||||
this.ß_keyStrength = activation[0];
|
||||
this.g_interpolation = activation[1];
|
||||
this.Y_focus = activation[2] + 1; //Squash.SOFTPLUS(activation[2]) + 1;
|
||||
var startAt = 3;
|
||||
for (var k = 0; k < this.k_keys.length; k++) {
|
||||
this.k_keys[k] = this.layer.list[k + startAt].activation;
|
||||
}
|
||||
startAt += this.k_keys.length;
|
||||
for (var k = 0; k < this.addGate.length; k++) {
|
||||
this.addGate[k] = this.layer.list[k + startAt].activation;
|
||||
}
|
||||
startAt += this.addGate.length;
|
||||
for (var k = 0; k < this.eraseGate.length; k++) {
|
||||
this.eraseGate[k] = Squash.LOGISTIC(this.layer.list[k + startAt].activation);
|
||||
}
|
||||
var M = this.memory.data;
|
||||
// focus by content, obtains an array of similarity indexes for each memoryBlock
|
||||
for (var i = 0; i < M.length; i++)
|
||||
this.wc_focusedWeights[i] = Utils.getCosineSimilarity(M[i], this.k_keys) * this.ß_keyStrength;
|
||||
Utils.softMax(this.wc_focusedWeights);
|
||||
// focus by location (interpolation)
|
||||
Utils.interpolateArray(this.wc_focusedWeights, this.w_weightings, this.g_interpolation);
|
||||
// convolutional shift
|
||||
//this.doShiftings();
|
||||
Utils.vectorInvertedShifting(this.wc_focusedWeights, this.s_shiftingVector);
|
||||
// sharpening
|
||||
Utils.sharpArray(this.w_weightings, this.wc_focusedWeights, this.Y_focus);
|
||||
// since ∑ w = 1, we have to softmax the array
|
||||
Utils.softMax(this.w_weightings);
|
||||
/// we got wt!
|
||||
};
|
||||
Head.prototype.doShiftings = function () {
|
||||
// call this fn in case of not using a softmaxLayer for shifting
|
||||
Utils.softMax(this.s_shiftingVector);
|
||||
Utils.vectorInvertedShifting(this.wc_focusedWeights, this.s_shiftingVector);
|
||||
};
|
||||
Head.prototype.doTimeStep = function () {
|
||||
var activation = this.layer.activate();
|
||||
this.shiftingLayer && this.shiftingLayer.activate();
|
||||
this.readParams(activation);
|
||||
// reading
|
||||
for (var index = 0; index < this.memory.blocks; index++) {
|
||||
this.readVector[index] = 0;
|
||||
for (var cell = 0; cell < this.memory.blockWidth; cell++) {
|
||||
this.readVector[index] += this.memory.data[index][cell] * this.w_weightings[index];
|
||||
}
|
||||
}
|
||||
};
|
||||
Head.ADDITIONAL_INPUT_VALUES = 3;
|
||||
return Head;
|
||||
})();
|
||||
exports.Head = Head;
|
||||
|
||||
},{"../layer":7,"../network":8,"../softmaxLayer":10,"../squash":11,"../trainer":13,"../utils":14}],6:[function(require,module,exports){
|
||||
},{"../layer":6,"../network":7,"../trainer":11}],5:[function(require,module,exports){
|
||||
var __extends = this.__extends || function (d, b) {
|
||||
for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p];
|
||||
function __() { this.constructor = d; }
|
||||
@@ -464,7 +279,7 @@ var Perceptron = (function (_super) {
|
||||
exports.Perceptron = Perceptron;
|
||||
;
|
||||
|
||||
},{"../layer":7,"../network":8,"../trainer":13}],7:[function(require,module,exports){
|
||||
},{"../layer":6,"../network":7,"../trainer":11}],6:[function(require,module,exports){
|
||||
var neuron = require('./neuron');
|
||||
var network = require('./network');
|
||||
/*******************************************************************************************
|
||||
@@ -720,7 +535,7 @@ var Layer;
|
||||
Layer.LayerConnection = LayerConnection;
|
||||
})(Layer = exports.Layer || (exports.Layer = {}));
|
||||
|
||||
},{"./network":8,"./neuron":9}],8:[function(require,module,exports){
|
||||
},{"./network":7,"./neuron":8}],7:[function(require,module,exports){
|
||||
var layer = require('./layer');
|
||||
var Squash = require('./squash');
|
||||
var _neuron = require('./neuron');
|
||||
@@ -1243,7 +1058,7 @@ var Network = (function () {
|
||||
})();
|
||||
exports.Network = Network;
|
||||
|
||||
},{"./layer":7,"./neuron":9,"./squash":11}],9:[function(require,module,exports){
|
||||
},{"./layer":6,"./neuron":8,"./squash":9}],8:[function(require,module,exports){
|
||||
/// <reference path="synaptic.ts" />
|
||||
var Squash = require('./squash');
|
||||
/******************************************************************************************
|
||||
@@ -1928,77 +1743,7 @@ var Neuron;
|
||||
})(Connection = Neuron.Connection || (Neuron.Connection = {}));
|
||||
})(Neuron = exports.Neuron || (exports.Neuron = {}));
|
||||
|
||||
},{"./squash":11}],10:[function(require,module,exports){
|
||||
var __extends = this.__extends || function (d, b) {
|
||||
for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p];
|
||||
function __() { this.constructor = d; }
|
||||
__.prototype = b.prototype;
|
||||
d.prototype = new __();
|
||||
};
|
||||
var Layer = require('./layer');
|
||||
var Squash = require('./squash');
|
||||
var _Utils = require('./utils');
|
||||
var Utils = _Utils.Utils;
|
||||
var SoftMaxLayer = (function (_super) {
|
||||
__extends(SoftMaxLayer, _super);
|
||||
function SoftMaxLayer(size, label) {
|
||||
_super.call(this, size, label);
|
||||
this.optimizable = false;
|
||||
for (var n = 0; n < this.list.length; n++) {
|
||||
this.list[n].squash = Squash.IDENTITY;
|
||||
}
|
||||
}
|
||||
SoftMaxLayer.prototype.activate = function (input) {
|
||||
if (this.currentActivation.length != this.list.length)
|
||||
this.currentActivation = new Float64Array(this.list.length);
|
||||
var activationIndex = 0;
|
||||
var sum = 0;
|
||||
var Amax = null;
|
||||
if (typeof input != 'undefined') {
|
||||
if (input.length != this.size)
|
||||
throw "INPUT size and LAYER size must be the same to activate!";
|
||||
Utils.softMax(input);
|
||||
for (var id in this.list) {
|
||||
this.list[id].readIncommingConnections(input[id]);
|
||||
if (Amax === null || this.list[id].activation > Amax)
|
||||
Amax = this.list[id].activation;
|
||||
}
|
||||
}
|
||||
else {
|
||||
for (var id in this.list) {
|
||||
this.list[id].readIncommingConnections();
|
||||
if (Amax === null || this.list[id].activation > Amax)
|
||||
Amax = this.list[id].activation;
|
||||
}
|
||||
}
|
||||
for (var n = 0; n < this.currentActivation.length; n++) {
|
||||
sum += (this.list[n].activation = Math.exp(this.list[n].activation - Amax));
|
||||
}
|
||||
for (var n = 0; n < this.currentActivation.length; n++) {
|
||||
// set the activations
|
||||
var x = this.list[n].activation / sum;
|
||||
this.list[n].activation = this.currentActivation[n] = x;
|
||||
// set the derivatives
|
||||
//x = this.list[n].activation / (sum - this.list[n].activation);
|
||||
this.list[n].derivative = x * (1 - x);
|
||||
this.list[n].updateTraces();
|
||||
}
|
||||
return this.currentActivation;
|
||||
};
|
||||
SoftMaxLayer.NormalizeConnectionWeights = function (layerConnection) {
|
||||
var sum = 0;
|
||||
for (var c = 0; c < layerConnection.list.length; c++) {
|
||||
sum += (layerConnection.list[c].weight = Math.exp(layerConnection.list[c].weight));
|
||||
}
|
||||
for (var c = 0; c < layerConnection.list.length; c++) {
|
||||
layerConnection.list[c].weight /= sum;
|
||||
}
|
||||
};
|
||||
return SoftMaxLayer;
|
||||
})(Layer.Layer);
|
||||
exports.SoftMaxLayer = SoftMaxLayer;
|
||||
|
||||
},{"./layer":7,"./squash":11,"./utils":14}],11:[function(require,module,exports){
|
||||
},{"./squash":9}],9:[function(require,module,exports){
|
||||
// squashing functions
|
||||
function LOGISTIC(x, derivate) {
|
||||
if (derivate) {
|
||||
@@ -2035,7 +1780,7 @@ function EXP(x, derivate) {
|
||||
}
|
||||
exports.EXP = EXP;
|
||||
|
||||
},{}],12:[function(require,module,exports){
|
||||
},{}],10:[function(require,module,exports){
|
||||
/*
|
||||
********************************************************************************************
|
||||
SYNAPTIC
|
||||
@@ -2088,7 +1833,7 @@ if (typeof window != "undefined")
|
||||
window['synaptic'] = Synaptic;
|
||||
module.exports = Synaptic;
|
||||
|
||||
},{"./architect":1,"./layer":7,"./network":8,"./neuron":9,"./squash":11,"./trainer":13,"./utils":14}],13:[function(require,module,exports){
|
||||
},{"./architect":1,"./layer":6,"./network":7,"./neuron":8,"./squash":9,"./trainer":11,"./utils":12}],11:[function(require,module,exports){
|
||||
/*******************************************************************************************
|
||||
TRAINER
|
||||
*******************************************************************************************/
|
||||
@@ -2625,7 +2370,7 @@ var Trainer;
|
||||
};
|
||||
})(Trainer = exports.Trainer || (exports.Trainer = {}));
|
||||
|
||||
},{}],14:[function(require,module,exports){
|
||||
},{}],12:[function(require,module,exports){
|
||||
var Utils = (function () {
|
||||
function Utils() {
|
||||
}
|
||||
@@ -2802,5 +2547,5 @@ var Utils = (function () {
|
||||
})();
|
||||
exports.Utils = Utils;
|
||||
|
||||
},{}]},{},[12]);
|
||||
},{}]},{},[10]);
|
||||
var synaptic = synaptic || Synaptic;var Neuron = synaptic.Neuron, Layer = synaptic.Layer, Network = synaptic.Network, Trainer = synaptic.Trainer, Architect = synaptic.Architect;
|
||||
externo
+2
-3
Diff do arquivo suprimido porque uma ou mais linhas são muito longas
@@ -2,10 +2,8 @@ import hopfield = require('./architect/Hopfield');
|
||||
import lstm = require('./architect/LSTM');
|
||||
import lsm = require('./architect/Liquid');
|
||||
import perceptron = require('./architect/Perceptron');
|
||||
import mb = require('./architect/NTM');
|
||||
|
||||
export var LSTM = lstm.LSTM;
|
||||
export var Liquid = lsm.Liquid;
|
||||
export var Hopfield = hopfield.Hopfield;
|
||||
export var Perceptron = perceptron.Perceptron;
|
||||
export var NTM = mb.NTM;
|
||||
@@ -1,264 +0,0 @@
|
||||
import network = require('../network');
|
||||
import trainer = require('../trainer');
|
||||
import Layer = require('../layer');
|
||||
import neuron = require('../neuron');
|
||||
import Synaptic = require('../synaptic');
|
||||
import Squash = require('../squash');
|
||||
import _utils = require('../utils');
|
||||
import softmaxLayer = require('../softmaxLayer');
|
||||
|
||||
var Utils = _utils.Utils;
|
||||
|
||||
export class NTM extends network.Network {
|
||||
trainer: trainer.Trainer;
|
||||
|
||||
data: Float64Array[];
|
||||
|
||||
blockWidth: number;
|
||||
blocks: number;
|
||||
|
||||
heads: Head[] = new Array();
|
||||
|
||||
inputValues: Float64Array;
|
||||
|
||||
inputLayer: Layer.Layer;
|
||||
hiddenLayer: Layer.Layer;
|
||||
outputLayer: Layer.Layer;
|
||||
|
||||
dirty = false;
|
||||
|
||||
constructor(inputs: number, outputs: number, memBlocks: number, blockWidth: number, heads: number, hiddenSize: number) {
|
||||
// build the memory
|
||||
|
||||
super();
|
||||
|
||||
this.trainer = new trainer.Trainer(this);
|
||||
|
||||
this.blocks = memBlocks;
|
||||
this.blockWidth = blockWidth;
|
||||
|
||||
this.data = new Array(this.blocks);
|
||||
for (var index = 0; index < this.data.length; index++) {
|
||||
this.data[index] = new Float64Array(blockWidth);
|
||||
|
||||
}
|
||||
|
||||
this.clean();
|
||||
|
||||
// build the network
|
||||
|
||||
var inputLength = inputs + heads * memBlocks;
|
||||
|
||||
this.inputValues = new Float64Array(inputLength);
|
||||
|
||||
this.layers.input = this.inputLayer = new Layer.Layer(inputLength);
|
||||
this.hiddenLayer = new Layer.Layer(hiddenSize);
|
||||
this.layers.output = this.outputLayer = new Layer.Layer(outputs);
|
||||
|
||||
|
||||
|
||||
this.inputLayer.project(this.hiddenLayer, Layer.Layer.connectionType.ALL_TO_ALL);
|
||||
this.hiddenLayer.project(this.outputLayer, Layer.Layer.connectionType.ALL_TO_ALL);
|
||||
|
||||
var inputCounter = inputs - 1;
|
||||
|
||||
for (var headIndex = 0; headIndex < heads; headIndex++) {
|
||||
this.addHead(this.inputValues.subarray(inputCounter, inputCounter + memBlocks));
|
||||
inputCounter += memBlocks;
|
||||
}
|
||||
|
||||
this.optimized = false;
|
||||
}
|
||||
|
||||
clean() {
|
||||
for (var location = 0; location < this.blocks; location++) {
|
||||
Utils.initRandomSoftmaxArray(this.data[location]);
|
||||
}
|
||||
this.dirty = false;
|
||||
}
|
||||
|
||||
activate(input: Synaptic.INumericArray) {
|
||||
this.inputValues.set(<any>input);
|
||||
|
||||
this.inputLayer.activate(this.inputValues);
|
||||
this.hiddenLayer.activate();
|
||||
|
||||
this.doTimeStep();
|
||||
|
||||
return this.outputLayer.activate();
|
||||
}
|
||||
|
||||
propagate(rate: number, target: Synaptic.INumericArray) {
|
||||
this.outputLayer.propagate(rate, target);
|
||||
for (var i = this.heads.length - 1; i >= 0; i--) {
|
||||
this.heads[i].shiftingLayer && this.heads[i].shiftingLayer.propagate(rate);
|
||||
this.heads[i].layer.propagate(rate);
|
||||
}
|
||||
this.hiddenLayer.propagate(rate);
|
||||
this.dirty = true;
|
||||
}
|
||||
|
||||
addHead(subArray: Float64Array): Head {
|
||||
var head = new Head(this, subArray);
|
||||
this.heads.push(head);
|
||||
return head;
|
||||
}
|
||||
|
||||
doTimeStep() {
|
||||
this.heads.forEach((head, headIndex) => {
|
||||
head.doTimeStep();
|
||||
});
|
||||
|
||||
// parallelizable
|
||||
this.heads.forEach((head, headIndex) => {
|
||||
this.doErase(head.w_weightings, head.eraseGate);
|
||||
});
|
||||
|
||||
// parallelizable
|
||||
this.heads.forEach((head, headIndex) => {
|
||||
this.doAdd(head.w_weightings, head.addGate);
|
||||
});
|
||||
|
||||
//this.data.forEach((e) => e = Utils.softMax(e))
|
||||
}
|
||||
|
||||
doAdd(w: Synaptic.INumericArray, addGate: Synaptic.INumericArray) {
|
||||
for (var n = 0; n < this.blocks; n++) {
|
||||
var M = this.data[n];
|
||||
for (var i = 0; i < this.blockWidth; i++) {
|
||||
M[i] += addGate[n] * w[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
doErase(w: Synaptic.INumericArray, eraseGate: Synaptic.INumericArray) {
|
||||
for (var n = 0; n < this.blocks; n++) {
|
||||
var M = this.data[n];
|
||||
for (var i = 0; i < this.blockWidth; i++) {
|
||||
M[i] *= 1 - eraseGate[n] * w[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
export class Head {
|
||||
static ADDITIONAL_INPUT_VALUES = 3;
|
||||
|
||||
memory: NTM;
|
||||
|
||||
w_weightings: Float64Array;
|
||||
eraseGate: Float64Array;
|
||||
addGate: Float64Array;
|
||||
k_keys: Float64Array;
|
||||
g_interpolation: number;
|
||||
Y_focus: number;
|
||||
s_shiftingValue: number = null;
|
||||
s_shiftingVector: Float64Array;
|
||||
wc_focusedWeights: Float64Array;
|
||||
readVector: Float64Array;
|
||||
ß_keyStrength: number;
|
||||
prevFocus: number = 1;
|
||||
|
||||
shiftLength: number;
|
||||
|
||||
layer: Layer.Layer;
|
||||
shiftingLayer: Layer.Layer;
|
||||
|
||||
constructor(memory: NTM, destinationArray?: Float64Array) {
|
||||
this.memory = memory;
|
||||
this.wc_focusedWeights = new Float64Array(this.memory.blocks);
|
||||
this.w_weightings = new Float64Array(this.memory.blocks);
|
||||
|
||||
Utils.initRandomSoftmaxArray(this.w_weightings);
|
||||
|
||||
this.shiftLength = 3; //this.memory.blocks;
|
||||
|
||||
|
||||
this.k_keys = new Float64Array(this.memory.blockWidth);
|
||||
this.ß_keyStrength = 0;
|
||||
this.eraseGate = new Float64Array(this.memory.blocks);
|
||||
this.addGate = new Float64Array(this.memory.blocks);
|
||||
this.readVector = destinationArray || new Float64Array(this.memory.blocks);
|
||||
|
||||
// Head layer
|
||||
this.layer = new Layer.Layer(this.memory.blockWidth + this.memory.blocks * 3 + Head.ADDITIONAL_INPUT_VALUES, "NTM: Head layer");
|
||||
this.memory.hiddenLayer.project(this.layer, Layer.Layer.connectionType.ALL_TO_ALL);
|
||||
this.layer.project(this.memory.outputLayer, Layer.Layer.connectionType.ALL_TO_ALL);
|
||||
|
||||
// shifting layer
|
||||
this.shiftingLayer = new softmaxLayer.SoftMaxLayer(this.shiftLength, "NTM: Shifting layer");
|
||||
this.memory.hiddenLayer.project(this.shiftingLayer, Layer.Layer.connectionType.ALL_TO_ALL);
|
||||
this.shiftingLayer.project(this.memory.hiddenLayer, Layer.Layer.connectionType.ALL_TO_ALL);
|
||||
this.s_shiftingVector = this.shiftingLayer.currentActivation;
|
||||
}
|
||||
|
||||
private readParams(activation: Synaptic.INumericArray) {
|
||||
|
||||
this.ß_keyStrength = activation[0];
|
||||
this.g_interpolation = activation[1];
|
||||
this.Y_focus = activation[2] + 1;//Squash.SOFTPLUS(activation[2]) + 1;
|
||||
|
||||
var startAt = 3;
|
||||
for (var k = 0; k < this.k_keys.length; k++) {
|
||||
this.k_keys[k] = this.layer.list[k + startAt].activation;
|
||||
}
|
||||
|
||||
startAt += this.k_keys.length;
|
||||
for (var k = 0; k < this.addGate.length; k++) {
|
||||
this.addGate[k] = this.layer.list[k + startAt].activation;
|
||||
}
|
||||
|
||||
startAt += this.addGate.length;
|
||||
for (var k = 0; k < this.eraseGate.length; k++) {
|
||||
this.eraseGate[k] = Squash.LOGISTIC(this.layer.list[k + startAt].activation);
|
||||
}
|
||||
|
||||
var M = this.memory.data;
|
||||
|
||||
// focus by content, obtains an array of similarity indexes for each memoryBlock
|
||||
for (var i = 0; i < M.length; i++)
|
||||
this.wc_focusedWeights[i] = Utils.getCosineSimilarity(M[i], this.k_keys) * this.ß_keyStrength;
|
||||
|
||||
Utils.softMax(this.wc_focusedWeights);
|
||||
|
||||
// focus by location (interpolation)
|
||||
Utils.interpolateArray(this.wc_focusedWeights, this.w_weightings, this.g_interpolation);
|
||||
|
||||
// convolutional shift
|
||||
//this.doShiftings();
|
||||
Utils.vectorInvertedShifting(this.wc_focusedWeights, this.s_shiftingVector);
|
||||
|
||||
// sharpening
|
||||
Utils.sharpArray(this.w_weightings, this.wc_focusedWeights, this.Y_focus);
|
||||
|
||||
// since ∑ w = 1, we have to softmax the array
|
||||
Utils.softMax(this.w_weightings);
|
||||
|
||||
/// we got wt!
|
||||
}
|
||||
|
||||
doShiftings() {
|
||||
// call this fn in case of not using a softmaxLayer for shifting
|
||||
|
||||
Utils.softMax(this.s_shiftingVector);
|
||||
Utils.vectorInvertedShifting(this.wc_focusedWeights, this.s_shiftingVector);
|
||||
}
|
||||
|
||||
doTimeStep() {
|
||||
var activation = this.layer.activate();
|
||||
|
||||
this.shiftingLayer && this.shiftingLayer.activate();
|
||||
|
||||
this.readParams(activation);
|
||||
|
||||
// reading
|
||||
for (var index = 0; index < this.memory.blocks; index++) {
|
||||
this.readVector[index] = 0;
|
||||
for (var cell = 0; cell < this.memory.blockWidth; cell++) {
|
||||
this.readVector[index] += this.memory.data[index][cell] * this.w_weightings[index];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+1
-9
@@ -70,15 +70,7 @@ describe("NTM Utils", function() {
|
||||
|
||||
assert.equal(sumAll(Utils.softMax(fixedArray)), 1, 'Random signed array equals 1');
|
||||
});
|
||||
/*
|
||||
it('Generate circulant matrix', function(){
|
||||
var fixedArray = new Float64Array(5);
|
||||
|
||||
fixedArray.set([0,0,0,0,1]);
|
||||
|
||||
assert.equal(Utils.vectorShifting(fixedArray, [0, 1, 0]), [0,0,0,0,1], 'Unchanged shiftings');
|
||||
});
|
||||
*/
|
||||
|
||||
it('Shifting', function(){
|
||||
var fixedArray = new Float64Array(5);
|
||||
|
||||
|
||||
@@ -1,348 +0,0 @@
|
||||
<html>
|
||||
<head>
|
||||
<title>Training a neural network to complete a Discrete Sequence Recall task - Synaptic</title>
|
||||
<meta charset="UTF-8">
|
||||
|
||||
<script src="../dist/synaptic.js"></script>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.1.4/jquery.min.js" ></script>
|
||||
<style>
|
||||
|
||||
body {
|
||||
font-family: 'monaco','Open Sans', sans-serif;
|
||||
position: relative;
|
||||
line-height: 120%;
|
||||
margin: 0px;
|
||||
background: #000;
|
||||
color: #0f0;
|
||||
}
|
||||
.container {
|
||||
width: 1024px;
|
||||
margin: 0 auto;
|
||||
}
|
||||
.title {
|
||||
text-align: center;
|
||||
color: rgb(125, 125, 125);
|
||||
font-size: 34px;
|
||||
padding-top: 50px;
|
||||
height: 50px;
|
||||
cursor: pointer;
|
||||
}
|
||||
.title b{
|
||||
color:rgb(50, 50, 50);;
|
||||
}
|
||||
.title:hover{
|
||||
opacity: .8;
|
||||
}
|
||||
.subtitle {
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
p, li {
|
||||
padding: 4px;
|
||||
font-size: 13px;
|
||||
text-align: justify;
|
||||
}
|
||||
.code, .log {
|
||||
border-left: 5px solid rgb(220, 220, 220);
|
||||
padding-left: 20px;
|
||||
color: rgb(150, 150, 150);
|
||||
line-height: 1.2;
|
||||
white-space: pre;
|
||||
font-family: "Lucida Console", Monaco, monospace;
|
||||
font-size: 12px;
|
||||
}
|
||||
.log {
|
||||
margin: 0px;
|
||||
line-height: 3px;
|
||||
font-size: 11px;
|
||||
}
|
||||
.validate {
|
||||
width: 16;
|
||||
height: 13;
|
||||
display: inline-block;
|
||||
position: relative;
|
||||
top: 1px;
|
||||
}
|
||||
.tick {
|
||||
background-image: url(img/tick.png);
|
||||
}
|
||||
.fail{
|
||||
background-image: url(img/fail.png);
|
||||
width: 12px;
|
||||
height: 12px;
|
||||
top: 1px;
|
||||
position: relative;
|
||||
}
|
||||
.square{
|
||||
width: 12px;
|
||||
height: 12px;
|
||||
border-radius: 12px;
|
||||
top: 2px;
|
||||
position: relative;
|
||||
border: 1px solid rgb(68, 68, 68);
|
||||
display: inline-block;
|
||||
box-shadow: inset 0 2px 1px rgba(255,255,255,0.8);
|
||||
}
|
||||
.log .square{
|
||||
top: -5px;
|
||||
}
|
||||
#footer {
|
||||
position: absolute;
|
||||
font-size: 11px;
|
||||
bottom: 10px;
|
||||
margin-left: 50%;
|
||||
left: -245px;
|
||||
}
|
||||
a{
|
||||
color: black;
|
||||
font-weight: bold;
|
||||
}
|
||||
</style>
|
||||
<script>
|
||||
|
||||
var LSTM = null;
|
||||
|
||||
var targets = [2,4];
|
||||
var distractors = [3,5];
|
||||
var prompts = [0, 1];
|
||||
var length = 10;
|
||||
|
||||
var sequence = null;
|
||||
var sequenceLength = 0;
|
||||
var targetsCorrect = 0;
|
||||
var distractorsCorrect = 0;
|
||||
var indexes = [];
|
||||
var positions = [];
|
||||
var symbol = 0;
|
||||
var symbols = targets.length + distractors.length + prompts.length;
|
||||
var json = null;
|
||||
|
||||
var i = 0;
|
||||
|
||||
var canv = null;
|
||||
var ctx = null;
|
||||
|
||||
var log = function(text, append){
|
||||
var p = $('<p>', {'class' : 'log'}).html(text).appendTo($(".container")).hide().fadeIn();
|
||||
if (append)
|
||||
p.appendTo($("#" + append));
|
||||
}
|
||||
|
||||
var drawMemory = function(index,rgb){
|
||||
ctx.fillStyle = 'black';
|
||||
ctx.fillRect(0,0,512,128);
|
||||
|
||||
var memory = LSTM.data;
|
||||
|
||||
for (var memBlock = 0; memBlock < memory.length; memBlock++) {
|
||||
var mem = memory[memBlock];
|
||||
for (var wM = 0; wM < mem.length; wM++) {
|
||||
ctx.fillStyle = 'rgb(0,0,' + (mem[wM] * 255).toFixed() + ')';
|
||||
ctx.fillRect(memBlock * 9, wM*9,8,8);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var train = function(){
|
||||
canv = document.getElementById('canvasMem');
|
||||
ctx = canv.getContext('2d');
|
||||
|
||||
$("#aux").remove();
|
||||
$('.log').remove();
|
||||
$("#log-validate").remove();
|
||||
|
||||
var squareTargets =[];
|
||||
for (var i in targets)
|
||||
squareTargets[i] = square(targets[i]);
|
||||
log("Targets: " + squareTargets.join(' '));
|
||||
|
||||
var squareDistractors =[];
|
||||
for (var i in distractors)
|
||||
squareDistractors[i] = square(distractors[i]);
|
||||
log("Distractors: " + squareDistractors.join(' '));
|
||||
|
||||
var squarePrompts =[];
|
||||
for (var i in prompts)
|
||||
squarePrompts[i] = square(prompts[i]);
|
||||
log("Prompts: " + squarePrompts.join(' '));
|
||||
|
||||
log("Length: " + length);
|
||||
log (" ");
|
||||
|
||||
log("Training DSR...");
|
||||
log (" ");
|
||||
|
||||
$("body").animate({ scrollTop: $(document).height() }, function()
|
||||
{
|
||||
var inputs = symbols;
|
||||
var outputs = targets.length;
|
||||
var hidden = inputs * .8 | 0;
|
||||
|
||||
LSTM = new Architect.NTM(inputs,outputs,symbols,symbols,1,hidden);
|
||||
|
||||
|
||||
|
||||
var results = LSTM.trainer.DSR({
|
||||
targets: targets,
|
||||
distractors: distractors,
|
||||
prompts: prompts,
|
||||
length: length,
|
||||
iterations: 100,
|
||||
rate: .17,
|
||||
schedule: {
|
||||
every: 2,
|
||||
do: function(obj){
|
||||
drawMemory();
|
||||
console.log(' Error: ' + obj.error);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
if (results.iterations < 250000)
|
||||
{
|
||||
log("Trained!");
|
||||
log("- Iterations: " + results.iterations);
|
||||
log("- Error: " + results.error);
|
||||
log("- Elapsed time: " + results.time + " ms");
|
||||
log("- Success rate: " + (results.success*100).toFixed(2) + "%");
|
||||
$('<div>', { id: 'log-validate', style:'height:450px;'}).appendTo(".container");
|
||||
validate();
|
||||
} else{
|
||||
log (" ");
|
||||
log ("Failed...");
|
||||
log (" ");
|
||||
log (" ");
|
||||
console.log('failed');
|
||||
train()
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
var validate = function(){
|
||||
//$(document.body).css("min-height", 1349);
|
||||
$("#log-validate").html('');
|
||||
log (" ", "log-validate");
|
||||
log("Validating...", "log-validate");
|
||||
|
||||
sequence = [], sequenceLength = length - prompts.length;
|
||||
for (i = 0; i < sequenceLength; i ++)
|
||||
{
|
||||
var any = Math.random() * distractors.length | 0;
|
||||
sequence.push(distractors[any]);
|
||||
}
|
||||
indexes = [], positions = [];
|
||||
for (i = 0; i < prompts.length; i ++)
|
||||
{
|
||||
indexes.push(Math.random() * targets.length | 0);
|
||||
positions.push(noRepeat(sequenceLength, positions));
|
||||
}
|
||||
positions = positions.sort();
|
||||
for (i = 0; i < prompts.length; i ++)
|
||||
{
|
||||
sequence[positions[i]] = targets[indexes[i]];
|
||||
sequence.push(prompts[i]);
|
||||
}
|
||||
targetsCorrect = distractorsCorrect = symbol = 0;
|
||||
|
||||
log (" ", "log-validate");
|
||||
var squareSecuence =[];
|
||||
for (var i in sequence)
|
||||
squareSecuence[i] = square(sequence[i]);
|
||||
log("Sequence: " + squareSecuence.join(' '), "log-validate");
|
||||
next();
|
||||
}
|
||||
|
||||
var next = function(){
|
||||
// generate input from sequence
|
||||
var input = [];
|
||||
for (var j = 0; j < symbols; j++)
|
||||
input[j] = 0;
|
||||
input[sequence[symbol]] = 1;
|
||||
|
||||
// generate target output
|
||||
var output = [];
|
||||
for (j = 0; j < targets.length; j++)
|
||||
output[j] = 0;
|
||||
|
||||
if (symbol >= sequenceLength)
|
||||
{
|
||||
var index = symbol - sequenceLength;
|
||||
output[indexes[index]] = 1;
|
||||
}
|
||||
|
||||
// check result
|
||||
var prediction = LSTM.activate(input);
|
||||
var ok = equal(prediction, output);
|
||||
if (ok)
|
||||
if (symbol < sequenceLength)
|
||||
distractorsCorrect++;
|
||||
else
|
||||
targetsCorrect++;
|
||||
val = value(prediction);
|
||||
log((symbol+1) + ") input: " + square(sequence[symbol]) + " output: " + (val != "none" ? square(targets[val]) : "--") + (ok ? ' <span class="validate tick"></span>' : ' <span class="validate fail"></span>'), "log-validate")
|
||||
|
||||
symbol++;
|
||||
|
||||
if (symbol < sequence.length)
|
||||
setTimeout(next, 100);
|
||||
else {
|
||||
if (distractorsCorrect + targetsCorrect == length)
|
||||
{
|
||||
log (" ", "log-validate");
|
||||
log ("Success! <button onclick='validate();'>Try another sequence</button>", "log-validate");
|
||||
$("body").animate({ scrollTop: $(document).height() }, "slow");
|
||||
} else {
|
||||
log (" ", "log-validate");
|
||||
log ("Failed...", "log-validate");
|
||||
validate();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var noRepeat = function(range, avoid)
|
||||
{
|
||||
var number = Math.random() * range | 0;
|
||||
var used = false;
|
||||
for (var i in avoid)
|
||||
if (number == avoid[i])
|
||||
used = true;
|
||||
return used ? noRepeat(range, avoid) : number;
|
||||
}
|
||||
var equal = function(prediction, output){
|
||||
for (var i in prediction)
|
||||
if (Math.round(prediction[i]) != output[i])
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
var value = function(array){
|
||||
var max = .5;
|
||||
var res = -1;
|
||||
for (var i in array)
|
||||
if (array[i] > max)
|
||||
{
|
||||
max = array[i];
|
||||
res = i;
|
||||
}
|
||||
return res == -1 ? 'none' : res;
|
||||
}
|
||||
|
||||
var colors = ['white', 'black', 'green', 'yellow', 'red', 'blue', 'purple', 'orange', 'gray', 'lightblue'];
|
||||
var square = function(value){
|
||||
return '<span class="square" style="background-color:' + colors[value] + '"> </span>';
|
||||
}
|
||||
|
||||
</script>
|
||||
</head>
|
||||
<body>
|
||||
<canvas id="canvasMem" width=512 height=128></canvas>
|
||||
|
||||
<div class="container"></div>
|
||||
|
||||
<p> Start training: <button onclick="train();">START</button></p>
|
||||
<p id="aux"> </p>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,212 +0,0 @@
|
||||
<html>
|
||||
<head>
|
||||
<title>Training a neural network to complete a Discrete Sequence Recall task - Synaptic</title>
|
||||
<meta charset="UTF-8">
|
||||
|
||||
<script src="../dist/synaptic.js"></script>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.1.4/jquery.min.js" ></script>
|
||||
<style>
|
||||
|
||||
body {
|
||||
font-family: 'monaco','Open Sans', sans-serif;
|
||||
position: relative;
|
||||
line-height: 120%;
|
||||
margin: 0px;
|
||||
background: #000;
|
||||
color: #0f0;
|
||||
}
|
||||
.container {
|
||||
width: 1024px;
|
||||
margin: 0 auto;
|
||||
}
|
||||
.title {
|
||||
text-align: center;
|
||||
color: rgb(125, 125, 125);
|
||||
font-size: 34px;
|
||||
padding-top: 50px;
|
||||
height: 50px;
|
||||
cursor: pointer;
|
||||
}
|
||||
.title b{
|
||||
color:rgb(50, 50, 50);;
|
||||
}
|
||||
.title:hover{
|
||||
opacity: .8;
|
||||
}
|
||||
.subtitle {
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
p, li {
|
||||
padding: 4px;
|
||||
font-size: 13px;
|
||||
text-align: justify;
|
||||
}
|
||||
.code, .log {
|
||||
border-left: 5px solid rgb(220, 220, 220);
|
||||
padding-left: 20px;
|
||||
color: rgb(150, 150, 150);
|
||||
line-height: 1.2;
|
||||
white-space: pre;
|
||||
font-family: 'monaco', "Lucida Console", Monaco, monospace;
|
||||
font-size: 12px;
|
||||
}
|
||||
.log {
|
||||
margin: 0px;
|
||||
line-height: 3px;
|
||||
font-size: 11px;
|
||||
}
|
||||
.validate {
|
||||
width: 16;
|
||||
height: 13;
|
||||
display: inline-block;
|
||||
position: relative;
|
||||
top: 1px;
|
||||
}
|
||||
.tick {
|
||||
background-image: url(img/tick.png);
|
||||
}
|
||||
.fail{
|
||||
background-image: url(img/fail.png);
|
||||
width: 12px;
|
||||
height: 12px;
|
||||
top: 1px;
|
||||
position: relative;
|
||||
}
|
||||
.square{
|
||||
width: 12px;
|
||||
height: 12px;
|
||||
border-radius: 12px;
|
||||
top: 2px;
|
||||
position: relative;
|
||||
border: 1px solid rgb(68, 68, 68);
|
||||
display: inline-block;
|
||||
box-shadow: inset 0 2px 1px rgba(255,255,255,0.8);
|
||||
}
|
||||
.log .square{
|
||||
top: -5px;
|
||||
}
|
||||
#footer {
|
||||
position: absolute;
|
||||
font-size: 11px;
|
||||
bottom: 10px;
|
||||
margin-left: 50%;
|
||||
left: -245px;
|
||||
}
|
||||
a{
|
||||
color: black;
|
||||
font-weight: bold;
|
||||
}
|
||||
</style>
|
||||
<script>
|
||||
|
||||
var seqWidth = 9;
|
||||
var seqLength = 10;
|
||||
var targetSequence = [];
|
||||
var trainLimit = 60;
|
||||
|
||||
var LSTM = new Architect.NTM(seqWidth,seqWidth,4,4,1,9);
|
||||
|
||||
var canv = null;
|
||||
var ctx = null;
|
||||
|
||||
for (var i = 0; i < seqLength; i++) {
|
||||
var arr = new Float64Array(seqWidth);
|
||||
targetSequence.push(arr);
|
||||
for(var n = 0; n < seqWidth - 1; n++){
|
||||
arr[n] = Math.random() > 0.5 ? 1 : 0;
|
||||
}
|
||||
}
|
||||
|
||||
var arr = new Float64Array(seqWidth);
|
||||
arr[seqWidth - 1] = 1;
|
||||
targetSequence.push(arr);
|
||||
|
||||
|
||||
var log = function(text, append){
|
||||
var p = $('<p>', {'class' : 'log'}).html(text).appendTo($(".container")).hide().fadeIn();
|
||||
if (append)
|
||||
p.appendTo($("#" + append));
|
||||
}
|
||||
|
||||
var drawMemory = function(index,rgb){
|
||||
if(index == 0){
|
||||
ctx.fillStyle = 'black';
|
||||
ctx.fillRect(0, 0, 512, 512);
|
||||
}
|
||||
|
||||
var memory = LSTM.data;
|
||||
|
||||
for (var memBlock = 0; memBlock < memory.length; memBlock++) {
|
||||
var mem = memory[memBlock];
|
||||
alto = mem.length + 15;
|
||||
|
||||
for (var wM = 0; wM < mem.length; wM++) {
|
||||
ctx.fillStyle = 'rgb(0,0,' + (mem[wM] * 255).toFixed() + ')';
|
||||
ctx.fillRect(memBlock * 9, wM*9,8,8);
|
||||
}
|
||||
}
|
||||
|
||||
var focus = LSTM.heads[0].wc_focusedWeights;
|
||||
|
||||
for (var wM = 0; wM < focus.length; wM++) {
|
||||
var r = (focus[wM] * 255).toFixed();
|
||||
ctx.fillStyle = 'rgb(' + r + ',' + r + ',' + r + ')';
|
||||
ctx.fillRect( (wM)*9,(memory.length + index + 1) * 9,8,8);
|
||||
}
|
||||
}
|
||||
|
||||
var train = function(){
|
||||
|
||||
|
||||
LSTM.clean();
|
||||
|
||||
doTraining(0);
|
||||
|
||||
}
|
||||
|
||||
var execute = function(){
|
||||
for(var i = 0; i < targetSequence.length; i++){
|
||||
console.log('Result', LSTM.activate(targetSequence[i]), ' Expected:', targetSequence[i]);
|
||||
}
|
||||
}
|
||||
|
||||
var doTraining = function(num){
|
||||
if(num > trainLimit) return execute();
|
||||
|
||||
for(var i = 0; i < targetSequence.length; i++){
|
||||
LSTM.activate(targetSequence[i]);
|
||||
}
|
||||
|
||||
LSTM.propagate(0.01, targetSequence[num % targetSequence.length]);
|
||||
|
||||
|
||||
|
||||
|
||||
drawMemory(num);
|
||||
console.log(num, 'of', trainLimit);
|
||||
|
||||
requestAnimationFrame(function(){
|
||||
doTraining(num + 1);
|
||||
});
|
||||
}
|
||||
|
||||
$(function(){
|
||||
canv = document.getElementById('canvasMem');
|
||||
ctx = canv.getContext('2d');
|
||||
drawMemory();
|
||||
});
|
||||
|
||||
|
||||
</script>
|
||||
</head>
|
||||
<body>
|
||||
<canvas id="canvasMem" width=512 height=512></canvas>
|
||||
|
||||
<div class="container"></div>
|
||||
|
||||
<p> Start training: <button onclick="train();">START</button></p>
|
||||
<p id="aux"> </p>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
+1
-2
@@ -20,7 +20,6 @@
|
||||
"src/architect/Perceptron.ts",
|
||||
"src/architect/LSTM.ts",
|
||||
"src/architect/Liquid.ts",
|
||||
"src/architect/Hopfield.ts",
|
||||
"src/architect/NTM.ts"
|
||||
"src/architect/Hopfield.ts"
|
||||
]
|
||||
}
|
||||
Referência em uma Nova Issue
Bloquear um usuário