Comparar commits

..

7 Commits

Autor SHA1 Mensagem Data
Brian Broll 3676d4c340 WIP Added istanbul 2016-10-02 09:04:17 -05:00
Brian Broll 16668468f4 Added boolean type check for attributes. Fixes #832 (#834) 2016-09-28 21:31:51 -05:00
Brian Broll ffae88a168 Set the exec status to 'running' on job restart. Fixes #831 (#833) 2016-09-28 21:27:45 -05:00
Brian Broll 25f5759c17 Cancel job if the exec has been canceled. Fixes #820 (#830)
* Cancel job if the exec has been canceled. Fixes #820

* WIP #820 Added test for stopping watching on cancel

* WIP #820 Increased the timeout for import torch
2016-09-27 19:28:57 -05:00
Brian Broll 7a0eae386f Added run.sh script to execution files. Fixes #818 (#829)
* WIP Added basic run.sh script creation

* WIP #818. Added baseURL... still needs work

* WIP #818 Added prompt for DEEPFORGE_URL and fixed wget issue
2016-09-27 15:01:51 -05:00
Brian Broll af5d74483a Added read-only tier of changes being applied. Fixes #826 (#827)
* Added read-only tier of changes being applied. Fixes #826

WIP #826 Updated currentChanges name. Added debugging

WIP #826 Added get/setAttr test for ExecJob

WIP #826 Added get/set attr tests

WIP #826 Added logs to try to figure out what on earth is going wrong

WIP #826 Fixed typo in test

WIP #826 Removed debugging logging

WIP #826 Enabled all tests

* WIP #826 Fixed code climate issue
2016-09-26 19:18:20 -05:00
Brian Broll 9bf7632aba Update README.md 2016-09-24 15:24:43 -05:00
10 arquivos alterados com 403 adições e 45 exclusões
+7 -1
Ver Arquivo
@@ -7,7 +7,13 @@
**Notice**: DeepForge is still a work in progress and is also lacking significant documentation! That being said, any contributions and/or feedback is greatly appreciated (and feel free to always ask any questions on the gitter)!
# DeepForge
DeepForge is an open-source visual development environment for deep learning. Currently, it supports Convolutional Neural Networks but we are planning on supporting additional deep learning classifiers such as RNNs and LSTMs. Additional features include real-time collaborative editing and version control.
DeepForge is an open-source visual development environment for deep learning. Currently, it supports Convolutional Neural Networks, RNNs and LSTMs as well as the creation of custom layers. Additional features include:
- Graphical architecture editor
- Training/testing pipeline creation
- Distributed pipeline execution
- Real-time pipeline feedback
- Collaborative editing
- Automatic version control.
## Quick Start
Simply run the following command to install deepforge with its dependencies:
+1
Ver Arquivo
@@ -20,6 +20,7 @@ config.seedProjects.basePaths.push(__dirname + '/../src/seeds/devPipelineTests')
config.seedProjects.basePaths.push(__dirname + '/../src/seeds/project');
config.seedProjects.basePaths.push(__dirname + '/../src/seeds/cifar10');
config.seedProjects.basePaths.push(__dirname + '/../src/seeds/xor');
config.seedProjects.basePaths.push(__dirname + '/../src/seeds/devProject');
+2
Ver Arquivo
@@ -9,6 +9,7 @@
"local": "node ./bin/start-local.js",
"worker": "node ./bin/start-worker.js",
"test": "mkdir ./test-tmp; node ./node_modules/mocha/bin/mocha --recursive test",
"test-cover": "node node_modules/istanbul/lib/cli.js --hook-run-in-context cover node_modules/mocha/bin/_mocha -- -R spec test/**/*",
"watch-test": "./node_modules/nodemon/bin/nodemon.js --exec 'node ./node_modules/mocha/bin/mocha --recursive test'",
"build-nn": "node ./utils/nn-parser.js"
},
@@ -35,6 +36,7 @@
"chai": "^3.0.0",
"jszip": "^2.5.0",
"mocha": "^2.2.5",
"istanbul": "^0.4.5",
"mockery": "^1.7.0"
}
}
+97 -37
Ver Arquivo
@@ -57,7 +57,9 @@ define([
this._oldMetadataByName = {}; // name -> id
this.lastAppliedCmd = {};
this.canceled = false;
this.changes = {};
this.currentChanges = {}; // read-only changes being applied
this.creations = {};
this.deletions = [];
this.createIdToMetadataId = {};
@@ -76,6 +78,18 @@ define([
ExecuteJob.prototype = Object.create(PluginBase.prototype);
ExecuteJob.prototype.constructor = ExecuteJob;
ExecuteJob.prototype.configure = function () {
var result = PluginBase.prototype.configure.apply(this, arguments);
this.logManager = new JobLogsClient({
logger: this.logger,
port: this.gmeConfig.server.port,
branchName: this.branchName,
projectId: this.projectId
});
return result;
};
/**
* Main function for the plugin to execute. This will perform the execution.
* Notes:
@@ -88,19 +102,21 @@ define([
ExecuteJob.prototype.main = function (callback) {
// Check the activeNode to make sure it is a valid node
var type = this.core.getMetaType(this.activeNode),
typeName = type && this.getAttribute(type, 'name');
typeName = type && this.getAttribute(type, 'name'),
execNode,
status;
if (typeName !== 'Job') {
return callback(`Cannot execute ${typeName} (expected Job)`, this.result);
}
// Get the gmeConfig...
this.logManager = new JobLogsClient({
logger: this.logger,
port: this.gmeConfig.server.port,
branchName: this.branchName,
projectId: this.projectId
});
// Set the parent execution to 'running'
execNode = this.core.getParent(this.activeNode);
status = this.getAttribute(execNode, 'status');
if (status !== 'running') {
this.setAttribute(execNode, 'status', 'running');
}
this._callback = callback;
this.currentForkName = null;
this.prepare()
@@ -135,13 +151,7 @@ define([
ExecuteJob.prototype.createNode = function (baseType, parent) {
var id = this.getCreateId(),
parentId;
if (this.isCreateId(parent)) {
parentId = parent;
} else {
parentId = this.core.getPath(parent);
}
parentId = this.isCreateId(parent) ? parent : this.core.getPath(parent);
this.logger.info(`Creating ${id} of type ${baseType} in ${parentId}`);
assert(this.META[baseType], `Cannot create node w/ unrecognized type: ${baseType}`);
@@ -183,8 +193,7 @@ define([
};
ExecuteJob.prototype.getAttribute = function (node, attr) {
var nodeId,
base;
var nodeId;
assert(this.deletions.indexOf(nodeId) === -1,
`Cannot get ${attr} from deleted node ${nodeId}`);
@@ -193,27 +202,34 @@ define([
if (this.isCreateId(node)) {
nodeId = node;
assert(this.creations[nodeId], `Creation node not updated: ${nodeId}`);
// Set the node to the base so it falls back to an
// existing node if the attr info isn't in the diff
node = this.META[this.creations[nodeId].base];
} else {
nodeId = this.core.getPath(node);
}
// Check the changes; fallback on actual node
if (this.changes[nodeId] && this.changes[nodeId][attr] !== undefined) {
// If deleted the attribute, get the default (inherited) value
if (this.changes[nodeId][attr] === null) {
base = this.core.getBase(node);
return this.getAttribute(base, attr);
}
return this.changes[nodeId][attr];
// Check the most recent changes, then the currentChanges, then the model
var value = this._getValueFrom(nodeId, attr, node, this.changes) ||
this._getValueFrom(nodeId, attr, node, this.currentChanges);
if (value) {
return value;
}
return this.core.getAttribute(node, attr);
};
ExecuteJob.prototype._getValueFrom = function (nodeId, attr, node, changes) {
var base;
if (changes[nodeId] && changes[nodeId][attr] !== undefined) {
// If deleted the attribute, get the default (inherited) value
if (changes[nodeId][attr] === null) {
base = this.isCreateId(nodeId) ? node : this.core.getBase(node);
return this.getAttribute(base, attr);
}
return changes[nodeId][attr];
}
};
ExecuteJob.prototype._applyNodeChanges = function (node, changes) {
var attr,
value;
@@ -223,6 +239,7 @@ define([
attr = changes[i][0];
value = changes[i][1];
if (value !== null) {
this.logger.info(`Setting ${attr} to ${value} (${this.core.getPath(node)})`);
this.core.setAttribute(node, attr, value);
} else {
this.core.delAttribute(node, attr);
@@ -248,6 +265,7 @@ define([
promise;
this.logger.info('Collecting changes to apply in commit');
for (var i = nodeIds.length; i--;) {
changes = [];
attrs = Object.keys(this.changes[nodeIds[i]]);
@@ -264,7 +282,9 @@ define([
promises.push(promise);
}
this.currentChanges = this.changes;
this.changes = {};
// Need to differentiate between read/write changes.
this.logger.info(`About to apply changes for ${promises.length} nodes`);
return Q.all(promises)
.then(nodes => {
@@ -273,6 +293,9 @@ define([
assert(nodes[i], `node is ${nodes[i]} (${nodeIds[i]})`);
this._applyNodeChanges(nodes[i], changesFor[id]);
}
// Local model is now up-to-date. No longer need currentChanges
this.currentChanges = {};
});
};
@@ -437,7 +460,24 @@ define([
.then(activeObject => this.activeNode = activeObject)
.then(() => {
var metaNames = Object.keys(this.META);
return Q.all(metaNames.map(name => this.updateMetaNode(name)));
})
.then(() => {
var mdNodes,
mdIds;
mdIds = Object.keys(this._metadata)
.filter(id => !this.isCreateId(this._metadata[id]));
mdNodes = mdIds.map(id => this.core.getPath(this._metadata[id]))
.map(nodeId => this.core.loadByPath(this.rootNode, nodeId));
return Q.all(mdNodes).then(nodes => {
for (var i = nodes.length; i--;) {
this._metadata[mdIds[i]] = nodes[i];
}
});
});
};
@@ -694,20 +734,32 @@ define([
})
.then(mds => {
// Record the large files
var inputData = {};
var inputData = {},
runsh = '# Bash script to download data files and run job\n' +
'if [ -z "$DEEPFORGE_URL" ]; then\n echo "Please set DEEPFORGE_URL and' +
' re-run:"\n echo "" \n echo " DEEPFORGE_URL=http://my.' +
'deepforge.server.com:8080 bash run.sh"\n echo ""\n exit 1\nfi\n';
mds.forEach((metadata, i) => {
// add the hashes for each input
var input = inputs[i],
hash = files.inputAssets[input];
hash = files.inputAssets[input],
dataPath = 'inputs/' + input + '/data',
url = this.blobClient.getRelativeDownloadURL(hash);
inputData['inputs/' + input + '/data'] = {
inputData[dataPath] = {
req: hash,
cache: metadata.content
};
// Add to the run.sh file
runsh += `wget $DEEPFORGE_URL${url} -O ${dataPath}\n`;
});
delete files.inputAssets;
files['input-data.json'] = JSON.stringify(inputData, null, 2);
runsh += 'th init.lua';
files['run.sh'] = runsh;
// Add pointer assets
Object.keys(files.ptrAssets)
@@ -1087,8 +1139,8 @@ define([
};
ExecuteJob.prototype.createAttributeFile = function (node, files) {
var skip = ['code'],
numRegex = /^\d+\.?\d*((e|e-)\d+)?$/,
var skip = ['code', 'stdout', 'execFiles', 'jobId', 'secret'],
numOrBool = /^(-?\d+\.?\d*((e|e-)\d+)?|(true|false))$/,
table;
this.logger.info('Creating attributes file...');
@@ -1096,7 +1148,7 @@ define([
.filter(attr => skip.indexOf(attr) === -1)
.map(name => {
var value = this.getAttribute(node, name);
if (!numRegex.test(value)) {
if (!numOrBool.test(value)) {
value = `"${value}"`;
}
return [name, value];
@@ -1142,6 +1194,11 @@ define([
});
};
ExecuteJob.prototype.isExecutionCanceled = function () {
var execNode = this.core.getParent(this.activeNode);
return this.getAttribute(execNode, 'status') === 'canceled';
};
ExecuteJob.prototype.watchOperation = function (executor, hash, op, job) {
var jobId = this.core.getPath(job),
opId = this.core.getPath(op),
@@ -1150,7 +1207,7 @@ define([
name = this.getAttribute(job, 'name');
// If canceled, stop the operation
if (this.canceled) {
if (this.canceled || this.isExecutionCanceled()) {
secret = this.getAttribute(job, 'secret');
if (secret) {
executor.cancelJob(hash, secret);
@@ -1176,6 +1233,7 @@ define([
last = stdout.lastIndexOf('\n'),
result,
lastLine,
next = Q(),
msg;
// parse deepforge commands
@@ -1189,13 +1247,15 @@ define([
if (output) {
// Send notification to all clients watching the branch
this.logManager.appendTo(jobId, output)
next = next
.then(() => this.logManager.appendTo(jobId, output))
.then(() => this.notifyStdoutUpdate(jobId));
}
if (result.hasMetadata) {
msg = `Updated graph/image output for ${name}`;
return this.save(msg);
next = next.then(() => this.save(msg));
}
return next;
});
}
})
@@ -37,6 +37,7 @@ define([
this._currentSave = Q();
this.changes = {};
this.currentChanges = {}; // read-only changes being applied
this.creations = {};
this.deletions = [];
this.createIdToMetadataId = {};
@@ -188,6 +189,10 @@ define([
});
};
ExecutePipeline.prototype.isExecutionCanceled = function () {
return this.getAttribute(this.activeNode, 'status') === 'canceled';
};
ExecutePipeline.prototype.isInputData = function (node) {
var prnt = this.core.getParent(node);
return this.core.isTypeOf(prnt, this.META.Inputs);
+2 -2
Ver Arquivo
@@ -12,7 +12,7 @@ var JobLogManager = function(logger, config) {
};
JobLogManager.prototype._getFilePath = function(jInfo) {
this.logger.info(`getting file path for ${jInfo.job} in ${jInfo.project} on ${jInfo.branch}`);
this.logger.debug(`getting file path for ${jInfo.job} in ${jInfo.project} on ${jInfo.branch}`);
var jobId = jInfo.job.replace(/\//g, '_'),
filename = `${jobId}.txt`;
@@ -123,7 +123,7 @@ JobLogManager.prototype.appendTo = function(jobInfo, logs) {
JobLogManager.prototype.getLog = function(jobInfo) {
var filename = this._getFilePath(jobInfo);
this.logger.info(`Getting log content to ${filename}`);
this.logger.info(`Getting log content from ${filename}`);
return this.exists(jobInfo)
.then(exists => {
if (exists) {
Arquivo binário não exibido.
+285 -4
Ver Arquivo
@@ -10,12 +10,14 @@ describe('ExecuteJob', function () {
PluginCliManager = testFixture.WebGME.PluginCliManager,
projectName = 'testProject',
pluginName = 'ExecuteJob',
manager = new PluginCliManager(null, logger, gmeConfig),
project,
gmeAuth,
storage,
commitHash;
before(function (done) {
this.timeout(10000);
testFixture.clearDBAndGetGMEAuth(gmeConfig, projectName)
.then(function (gmeAuth_) {
gmeAuth = gmeAuth_;
@@ -25,7 +27,7 @@ describe('ExecuteJob', function () {
})
.then(function () {
var importParam = {
projectSeed: testFixture.path.join(testFixture.SEED_DIR, 'EmptyProject.webgmex'),
projectSeed: testFixture.path.join(testFixture.DF_SEED_DIR, 'devProject', 'devProject.webgmex'),
projectName: projectName,
branchName: 'master',
logger: logger,
@@ -51,9 +53,7 @@ describe('ExecuteJob', function () {
});
it('should verify activeNode is "Job"', function (done) {
var manager = new PluginCliManager(null, logger, gmeConfig),
pluginConfig = {
},
var pluginConfig = {},
context = {
project: project,
commitHash: commitHash,
@@ -68,4 +68,285 @@ describe('ExecuteJob', function () {
done();
});
});
////////// Helper Functions //////////
var plugin,
node,
preparePlugin = function(done) {
var context = {
project: project,
commitHash: commitHash,
branchName: 'test',
activeNode: '/K/R/p' // hello world job
};
return manager.initializePlugin(pluginName)
.then(plugin_ => {
plugin = plugin_;
return manager.configurePlugin(plugin, {}, context);
})
.then(() => node = plugin.activeNode)
.nodeify(done);
};
////////// END Helper Functions //////////
// Race condition checks w/ saving...
describe('get/set', function() {
beforeEach(preparePlugin);
it('should get correct attribute after set', function() {
plugin.setAttribute(node, 'status', 'queued');
var attrValue = plugin.getAttribute(node, 'status');
expect(attrValue).to.equal('queued');
});
it('should get correct attribute before updating nodes', function(done) {
// Run setAttribute on some node
plugin.setAttribute(node, 'status', 'queued');
// Check that the value is correct before applying node changes
var updateNodes = plugin.updateNodes;
plugin.updateNodes = function() {
var attrValue = plugin.getAttribute(node, 'status');
expect(attrValue).to.equal('queued');
return updateNodes.apply(this, arguments);
};
plugin.save().nodeify(done);
});
it('should get correct attribute (from new node) before updating nodes', function(done) {
// Run setAttribute on some node
var graphTmp = plugin.createNode('pipeline.Graph', node),
newVal = 'testGraph',
id = 'testId';
// Get the
plugin.setAttribute(graphTmp, 'name', newVal);
plugin._metadata[id] = graphTmp;
plugin.createIdToMetadataId[graphTmp] = id;
// Check that the value is correct before applying node changes
var updateNodes = plugin.updateNodes;
plugin.updateNodes = function() {
var graph = plugin._metadata[id],
attrValue = plugin.getAttribute(graph, 'name');
expect(attrValue).to.equal(newVal);
return updateNodes.apply(this, arguments);
};
plugin.save().nodeify(done);
});
it('should get correct attribute after save', function(done) {
// Run setAttribute on some node
plugin.setAttribute(node, 'status', 'queued');
// Check that the value is correct before applying node changes
plugin.save()
.then(() => {
var attrValue = plugin.getAttribute(node, 'status');
expect(attrValue).to.equal('queued');
})
.nodeify(done);
});
it('should get correct attribute while applying node changes', function(done) {
// Run setAttribute on some node
plugin.setAttribute(node, 'status', 'queued');
// Check that the value is correct before applying node changes
var oldApplyChanges = plugin._applyNodeChanges;
plugin._applyNodeChanges = function() {
var attrValue = plugin.getAttribute(node, 'status');
expect(attrValue).to.equal('queued');
return oldApplyChanges.apply(this, arguments);
};
plugin.save().nodeify(done);
});
});
describe('createNode', function() {
beforeEach(preparePlugin);
it('should update _metadata after applying changes', function(done) {
// Run setAttribute on some node
var graphTmp = plugin.createNode('pipeline.Graph', node),
id = 'testId';
plugin._metadata[id] = graphTmp;
plugin.createIdToMetadataId[graphTmp] = id;
// Check that the value is correct before applying node changes
var applyModelChanges = plugin.applyModelChanges;
plugin.applyModelChanges = function() {
return applyModelChanges.apply(this, arguments)
.then(() => {
var graph = plugin._metadata[id];
expect(graph).to.not.equal(graphTmp);
});
};
plugin.save().nodeify(done);
});
it('should update _metadata in updateNodes', function(done) {
var id = 'testId';
plugin._metadata[id] = node;
node.old = true;
plugin.updateNodes()
.then(() => {
var graph = plugin._metadata[id];
expect(graph.old).to.not.equal(true);
})
.nodeify(done);
});
// Check that it gets the correct value from a newly created node after
// it has been saved/created
it('should get changed attribute', function(done) {
// Run setAttribute on some node
var graphTmp = plugin.createNode('pipeline.Graph', node),
id = 'testId';
plugin._metadata[id] = node;
plugin.createIdToMetadataId[graphTmp] = id;
plugin.setAttribute(graphTmp, 'name', 'firstName');
// Check that the value is correct before applying node changes
plugin.save()
.then(() => {
var graph = plugin._metadata[id],
val = plugin.getAttribute(graph, 'name');
expect(val).to.equal('firstName');
})
.nodeify(done);
});
it('should get inherited attribute', function(done) {
// Run setAttribute on some node
var graphTmp = plugin.createNode('pipeline.Graph', node),
id = 'testId',
val;
// Check that the value is correct before applying node changes
plugin._metadata[id] = node;
plugin.createIdToMetadataId[graphTmp] = id;
val = plugin.getAttribute(graphTmp, 'name');
expect(val).to.equal('Graph');
plugin.save()
.then(() => {
var graph = plugin._metadata[id];
val = plugin.getAttribute(graph, 'name');
expect(val).to.equal('Graph');
})
.nodeify(done);
});
});
// Canceling
describe('cancel', function() {
beforeEach(preparePlugin);
it('should stop the job if the execution is canceled', function(done) {
var job = node,
hash = 'abc123',
exec = {
cancelJob: jobHash => expect(jobHash).equal(hash)
};
plugin.setAttribute(node, 'secret', 'abc');
plugin.isExecutionCanceled = () => true;
plugin.onOperationCanceled = () => done();
plugin.watchOperation(exec, hash, job, job);
});
it('should stop the job if a job is canceled', function(done) {
var job = node,
hash = 'abc123',
exec = {
cancelJob: jobHash => expect(jobHash).equal(hash)
};
plugin.setAttribute(job, 'secret', 'abc');
plugin.canceled = true;
plugin.onOperationCanceled = () => done();
plugin.watchOperation(exec, hash, job, job);
});
it('should set exec to running', function(done) {
var job = node,
execNode = plugin.core.getParent(job);
// Set the execution to canceled
plugin.setAttribute(execNode, 'status', 'canceled');
plugin.prepare = () => {
var status = plugin.getAttribute(execNode, 'status');
expect(status).to.not.equal('canceled');
return {then: () => done()};
};
plugin.main();
});
});
describe('exec files', function() {
describe('attribute file', function() {
var boolString = /['"](true|false)['"]/g;
beforeEach(preparePlugin);
it('should not quote true (s) boolean values', function() {
var files = {},
content,
matches;
plugin.setAttribute(node, 'debug', 'true');
plugin.createAttributeFile(node, files);
content = files['attributes.lua'];
matches = content.match(boolString);
expect(matches).to.equal(null);
});
it('should not quote true boolean values', function() {
var files = {},
content,
matches;
plugin.setAttribute(node, 'debug', true);
plugin.createAttributeFile(node, files);
content = files['attributes.lua'];
matches = content.match(boolString);
expect(matches).to.equal(null);
});
it('should not quote false (s) boolean values', function() {
var files = {},
content,
matches;
plugin.setAttribute(node, 'debug', 'false');
plugin.createAttributeFile(node, files);
content = files['attributes.lua'];
matches = content.match(boolString);
expect(matches).to.equal(null);
});
it('should not quote false boolean values', function() {
var files = {},
content,
matches;
plugin.setAttribute(node, 'debug', false);
plugin.createAttributeFile(node, files);
content = files['attributes.lua'];
matches = content.match(boolString);
expect(matches).to.equal(null);
});
});
});
});
+1 -1
Ver Arquivo
@@ -14,7 +14,7 @@ var testFixture = require('../../globals'),
];
describe('ImportTorch', function () {
this.timeout(5000);
this.timeout(7500);
var gmeConfig = testFixture.getGmeConfig(),
Q = testFixture.Q,
GraphChecker = testFixture.requirejs('deepforge/GraphChecker'),
+3
Ver Arquivo
@@ -280,6 +280,9 @@
},
"xor": {
"src": "src/seeds/xor"
},
"devProject": {
"src": "src/seeds/devProject"
}
},
"routers": {