v0.18.0

* WIP #763 basic working example

* WIP #763 applying layout position to the nodes

* WIP #763 Ignoring the klayjs lib

.codeclimate.yml

@@ -34,3 +34,4 @@ exclude_paths:
 - src/visualizers/widgets/TextEditor/lib/
 - src/visualizers/widgets/PipelineIndex/styles/PipelineIndex.css
 - src/visualizers/widgets/LineGraph/lib/
+- src/visualizers/widgets/PipelineEditor/klay.js

.npmignore

@@ -1,40 +0,0 @@
-src/worker/tmp
-.env
-*.swp
-*.swo
-**.sass-cache
-# Logs
-logs
-*.log
-
-# Runtime data
-pids
-*.pid
-*.seed
-
-# Directory for instrumented libs generated by jscoverage/JSCover
-lib-cov
-
-# Coverage directory used by tools like istanbul
-coverage
-
-# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
-.grunt
-
-# node-waf configuration
-.lock-wscript
-
-# Compiled binary addons (http://nodejs.org/api/addons.html)
-build/Release
-
-# Dependency directory
-# https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git
-node_modules
-tmp/
-test-tmp/
-blob-local-storage/
-src/seeds/nn/hash.txt
-src/seeds/pipeline/hash.txt
-
-# npm specific things
-images/

README.md

@@ -4,10 +4,16 @@
 [![Join the chat at https://gitter.im/dfst/deepforge](https://badges.gitter.im/dfst/deepforge.svg)](https://gitter.im/dfst/deepforge?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 [![Stories in Ready](https://badge.waffle.io/dfst/deepforge.png?label=ready&title=Ready)](https://waffle.io/dfst/deepforge)
 
-**Notice**: DeepForge is still a work in progress and is also lacking significant documentation! That being said, any contributions and/or feedback is greatly appreciated (and feel free to always ask any questions on the gitter)!
+**Notice**: DeepForge is still a work in progress and in beta! That being said, any contributions and/or feedback is greatly appreciated. If you have any questions, check out the [wiki](https://github.com/dfst/deepforge/wiki/) or drop me a line on the gitter!
 
 # DeepForge
-DeepForge is an open-source visual development environment for deep learning. Currently, it supports Convolutional Neural Networks but we are planning on supporting additional deep learning classifiers such as RNNs and LSTMs. Additional features include real-time collaborative editing and version control.
+DeepForge is an open-source visual development environment for deep learning. Currently, it supports Convolutional Neural Networks, RNNs and LSTMs as well as the creation of custom layers. Additional features include:
+- Graphical architecture editor
+- Training/testing pipeline creation
+- Distributed pipeline execution
+- Real-time pipeline feedback
+- Collaborative editing
+- Automatic version control.
 
 ## Quick Start
 Simply run the following command to install deepforge with its dependencies:
@@ -22,9 +28,9 @@ Or, if you already have NodeJS (v6) installed, simply run
 npm install -g deepforge
 ```
 
-Next, start deepforge with `deepforge start`!
+Finally, start deepforge with `deepforge start`and navigate to [http://localhost:8888](http://localhost:8888) to start using DeepForge! For more, detailed instructions, check out the [wiki](https://github.com/dfst/deepforge/wiki/Installation-Guide).
 
-Finally, navigate to [http://localhost:8888](http://localhost:8888) to start using DeepForge! For more, detailed instructions, check out the [wiki](https://github.com/dfst/deepforge/wiki/Installation-Guide).
+**Note**: running deepforge w/ `deepforge start` will also require [MongoDB](https://www.mongodb.com/download-center?jmp=nav#community) to be installed locally.
 
 Also, be sure to check out the other available features of the `deepforge` cli; it can be used to update, manage your torch installation, uninstall deepforge and run individual components!
 

config/components.json

@@ -46,8 +46,8 @@
     "CHFLayout": {
         "panels": [
             {
-                "id": "Header",
-                "panel": "BreadcrumbHeader/BreadcrumbHeaderPanel",
+                "id": "WorkerHeader",
+                "panel": "WorkerHeader/WorkerHeaderPanel",
                 "container": "header",
                 "DEBUG_ONLY": false
             },

config/config.webgme.js

@@ -20,6 +20,7 @@ config.seedProjects.basePaths.push(__dirname + '/../src/seeds/devPipelineTests')
 config.seedProjects.basePaths.push(__dirname + '/../src/seeds/project');
 config.seedProjects.basePaths.push(__dirname + '/../src/seeds/cifar10');
 config.seedProjects.basePaths.push(__dirname + '/../src/seeds/xor');
+config.seedProjects.basePaths.push(__dirname + '/../src/seeds/devProject');
 
 
 
@@ -31,6 +32,7 @@ config.visualization.panelPaths.push(__dirname + '/../src/visualizers/panels');
 
 
 config.rest.components['execution/logs'] = __dirname + '/../src/routers/JobLogsAPI/JobLogsAPI.js';
+config.rest.components['job/origins'] = __dirname + '/../src/routers/JobOriginAPI/JobOriginAPI.js';
 
 // Visualizer descriptors
 config.visualization.visualizerDescriptors.push(__dirname + '/../src/visualizers/Visualizers.json');

install.sh

@@ -42,10 +42,10 @@ detect_profile() {
 detect_profile
 
 set_node_version() {
-    # Install nodejs v6.2.0
-    echo "Installing NodeJS v6.2.0"
-    nvm install v6.2.0
-    nvm alias default v6.2.0
+    # Install nodejs v6.2.1
+    echo "Installing NodeJS v6.2.1"
+    nvm install v6.2.1
+    nvm alias default v6.2.1
 
     # Install npm@2
     npm install npm@2 -g

package.json

@@ -12,7 +12,7 @@
     "watch-test": "./node_modules/nodemon/bin/nodemon.js --exec 'node ./node_modules/mocha/bin/mocha --recursive test'",
     "build-nn": "node ./utils/nn-parser.js"
   },
-  "version": "0.15.0",
+  "version": "0.18.0",
   "dependencies": {
     "commander": "^2.9.0",
     "dotenv": "^2.0.0",
@@ -20,6 +20,7 @@
     "express": "^4.14.0",
     "lodash.difference": "^4.1.2",
     "lodash.merge": "^4.5.1",
+    "mongodb": "^2.2.10",
     "nodemon": "^1.9.2",
     "q": "1.4.1",
     "rimraf": "^2.4.0",

src/common/api/APIClient.js

@@ -0,0 +1,75 @@
+/*globals define*/
+define([
+    'q',
+    'superagent'
+], function(
+    Q,
+    superagent
+) {
+    'use strict';
+
+    // Wrap the ability to read, update, and delete logs using the JobLogsAPI
+    var APIClient = function(params) {
+        params = params || {};
+
+        this.logger = this.logger || params.logger.fork('APIClient');
+
+        // Get the server url
+        this.token = params.token;
+        this.origin = this._getServerUrl(params);
+        this.relativeUrl = this.relativeUrl || '';
+        this.url = this.origin + this.relativeUrl;
+
+        this.logger.debug(`Setting url to ${this.url}`);
+
+        this.branch = params.branchName;
+        this.project = params.projectId;
+        this._modifiedJobs = [];
+
+        this.logger.debug(`Using <project>:<branch>: "${this.project}"/"${this.branch}"`);
+        this.logger.info('ctor finished');
+    };
+
+    APIClient.prototype._getServerUrl = function(params) {
+        if (typeof window !== 'undefined') {
+            return window.location.origin;
+        }
+
+        // If not in browser, set using the params
+        var server = params.server || '127.0.0.1',
+            port = params.port || '80',
+            protocol = params.httpsecure ? 'https' : 'http';  // default is http
+
+        return params.origin || `${protocol}://${server}:${port}`;
+    };
+
+    APIClient.prototype.getUrl = function() {
+        return this.url;
+    };
+
+    APIClient.prototype._request = function(method, jobId, content) {
+        var deferred = Q.defer(),
+            req = superagent[method](this.getUrl(jobId));
+
+        this.logger.info(`sending ${method} request to ${this.getUrl(jobId)}`);
+        if (this.token) {
+            req.set('Authorization', 'Bearer ' + this.token);
+        }
+
+        if (content) {
+            req = req.send(content);
+        }
+
+        req.end((err, res) => {
+            if (err || res.status > 399) {
+                return deferred.reject(res || err);
+            }
+
+            return deferred.resolve(res);
+        });
+
+        return deferred.promise;
+    };
+
+    return APIClient;
+});

src/common/api/JobLogsClient.js

@@ -1,8 +1,10 @@
 /* globals define */
 define([
+    './APIClient',
     'q',
     'superagent'
 ], function(
+    APIClient,
     Q,
     superagent
 ) {
@@ -12,40 +14,22 @@ define([
     var JobLogsClient = function(params) {
         params = params || {};
 
-        this.logger = params.logger.fork('JobLogsClient');
-
-        // Get the server url
-        this.token = params.token;
-        this.origin = this._getServerUrl(params);
         this.relativeUrl = '/execution/logs';
-        this.url = this.origin + this.relativeUrl;
-
-        this.logger.debug(`Setting url to ${this.url}`);
+        this.logger = params.logger.fork('JobLogsClient');
+        APIClient.call(this, params);
 
         // Get the project, branch name
         if (!(params.branchName && params.projectId)) {
             throw Error('"branchName" and "projectId" required');
         }
-        this.branch = params.branchName;
-        this.project = params.projectId;
+
         this._modifiedJobs = [];
 
         this.logger.debug(`Using <project>:<branch>: "${this.project}"/"${this.branch}"`);
         this.logger.info('ctor finished');
     };
 
-    JobLogsClient.prototype._getServerUrl = function(params) {
-        if (typeof window !== 'undefined') {
-            return window.location.origin;
-        }
-
-        // If not in browser, set using the params
-        var server = params.server || '127.0.0.1',
-            port = params.port || '80',
-            protocol = params.httpsecure ? 'https' : 'http';  // default is http
-
-        return params.origin || `${protocol}://${server}:${port}`;
-    };
+    JobLogsClient.prototype = Object.create(APIClient.prototype);
 
     // This method could be optimized - it could make a log of requests
     JobLogsClient.prototype.fork = function(forkName) {
@@ -87,45 +71,21 @@ define([
         ].join('/');
     };
 
-    JobLogsClient.prototype._logRequest = function(method, jobId, content) {
-        var deferred = Q.defer(),
-            req = superagent[method](this.getUrl(jobId));
-
-        this.logger.info(`sending ${method} request to ${this.getUrl(jobId)}`);
-        if (this.token) {
-            req.set('Authorization', 'Bearer ' + this.token);
-        }
-
-        if (content) {
-            req = req.send(content);
-        }
-
-        req.end((err, res) => {
-            if (err || res.status > 399) {
-                return deferred.reject(err || res.status);
-            }
-
-            return deferred.resolve(res);
-        });
-
-        return deferred.promise;
-    };
-
     JobLogsClient.prototype.appendTo = function(jobId, text) {
         this._modifiedJobs.push(jobId);
         this.logger.info(`Appending logs to ${jobId}`);
-        return this._logRequest('patch', jobId, {patch: text});
+        return this._request('patch', jobId, {patch: text});
     };
 
     JobLogsClient.prototype.getLog = function(jobId) {
         this.logger.info(`Getting logs for ${jobId}`);
-        return this._logRequest('get', jobId)
+        return this._request('get', jobId)
             .then(res => res.text);
     };
 
     JobLogsClient.prototype.deleteLog = function(jobId) {
         this.logger.info(`Deleting logs for ${jobId}`);
-        return this._logRequest('delete', jobId);
+        return this._request('delete', jobId);
     };
 
     return JobLogsClient;

src/common/api/JobOriginClient.js

@@ -0,0 +1,60 @@
+/* globals define */
+define([
+    './APIClient'
+], function(
+    APIClient
+) {
+    'use strict';
+
+    var JobOriginClient = function(params) {
+        this.relativeUrl = '/job/origins/';
+        this.logger = params.logger.fork('JobOriginClient');
+        APIClient.call(this, params);
+    };
+
+    JobOriginClient.prototype = Object.create(APIClient.prototype);
+
+    // - Record the origin
+    // - Look up the origin
+    // - Delete record
+    JobOriginClient.prototype.getUrl = function(hash) {
+        return this.url + hash;
+    };
+
+    JobOriginClient.prototype.record = function(hash, info) {
+        var jobInfo = {
+            hash: hash,
+            nodeId: info.nodeId,
+            job: info.job,
+            project: info.project || this.project,
+            branch: info.branch || this.branch,
+            execution: info.execution
+        };
+
+        return this._request('post', hash, jobInfo)
+            .catch(err => {
+                throw err.text || err;
+            });
+    };
+
+    JobOriginClient.prototype.getOrigin = function(hash) {
+        return this._request('get', hash)
+            .then(res => JSON.parse(res.text))
+            .catch(res => {
+                if (res.status && res.status === 404) {
+                    return null;
+                }
+                throw res;
+            });
+    };
+
+    JobOriginClient.prototype.fork = function(hash, forkName) {
+        return this._request('patch', hash, {branch: forkName});
+    };
+
+    JobOriginClient.prototype.deleteRecord = function(hash) {
+        return this._request('delete', hash);
+    };
+
+    return JobOriginClient;
+});

src/common/plugin/LocalExecutor.js

@@ -20,13 +20,13 @@ define([
                 var output = cntrs
                     .find(cntr => {
                         var metaNode = this.core.getMetaType(cntr),
-                            metaName = this.core.getAttribute(metaNode, 'name');
+                            metaName = this.getAttribute(metaNode, 'name');
                         return metaName === 'Outputs';
                     });
                 return this.core.loadChildren(output);
             })
             .then(dataNodes => {
-                hash = this.core.getAttribute(dataNodes[0], 'data');
+                hash = this.getAttribute(dataNodes[0], 'data');
                 return this.getOutputs(node);
             })
             .then(outputTuples => {
@@ -48,7 +48,7 @@ define([
         var hash,
             typeId = this.core.getPointerPath(node, 'type'),
             type,
-            artifactName = this.core.getAttribute(node, 'artifactName');
+            artifactName = this.getAttribute(node, 'artifactName');
 
         return this.core.loadByPath(this.rootNode, typeId)
             .then(_type => {
@@ -58,25 +58,20 @@ define([
             .then(saveDir => this.core.loadChildren(saveDir))
             .then(artifacts => {
                 return artifacts.find(artifact =>
-                    this.core.getAttribute(artifact, 'name') === artifactName &&
+                    this.getAttribute(artifact, 'name') === artifactName &&
                         this.isMetaTypeOf(artifact, type));
             })
             .then(matchingArtifact => {
-                hash = matchingArtifact && this.core.getAttribute(matchingArtifact, 'data');
+                hash = matchingArtifact && this.getAttribute(matchingArtifact, 'data');
                 // If no hash, just continue (the subsequent ops will receive 'nil')
                 if (!hash) {
                     return this.onOperationComplete(node);
                 } else {
                     return this.getOutputs(node)
                         .then(outputPairs => {
-                            var outputs = outputPairs.map(pair => pair[2]),
-                                paths;
-
-                            paths = outputs.map(output => this.core.getPath(output));
+                            var outputs = outputPairs.map(pair => pair[2]);
                             // Get the 'data' hash and store it in the output data ports
-                            this.logger.info(`Loading blob data (${hash}) to ${paths.map(p => `"${p}"`)}`);
-
-                            outputs.forEach(output => this.core.setAttribute(output, 'data', hash));
+                            outputs.forEach(output => this.setAttribute(output, 'data', hash));
 
                             this.onOperationComplete(node);
                         });
@@ -99,7 +94,7 @@ define([
 
                 if (containers.length > 1) {
                     saveDir = containers.find(c =>
-                        this.core.getAttribute(c, 'name').toLowerCase().indexOf('artifacts') > -1
+                        this.getAttribute(c, 'name').toLowerCase().indexOf('artifacts') > -1
                     ) || containers[0];
                 }
 
@@ -121,8 +116,8 @@ define([
             .then(artifacts => {
                 currNameHashPairs = artifacts
                     .map(node => [
-                        this.core.getAttribute(node, 'name'),
-                        this.core.getAttribute(node, 'data')
+                        this.getAttribute(node, 'name'),
+                        this.getAttribute(node, 'data')
                     ]);
                 return this.getInputs(node);
             })
@@ -142,9 +137,9 @@ define([
 
                 // Remove nodes that already exist
                 dataNodes = allDataNodes.filter(dataNode => {
-                    var hash = this.core.getAttribute(dataNode, 'data'),
+                    var hash = this.getAttribute(dataNode, 'data'),
                         name = this.core.getOwnAttribute(node, 'saveName') ||
-                            this.core.getAttribute(dataNode, 'name');
+                            this.getAttribute(dataNode, 'name');
 
                     return !(currNameHashPairs
                         .find(pair => pair[0] === name && pair[1] === hash));
@@ -156,10 +151,10 @@ define([
                         newName = this.core.getOwnAttribute(node, 'saveName');
                     if (newName) {
                         newNodes.forEach(node =>
-                            this.core.setAttribute(node, 'name', newName)
+                            this.setAttribute(node, 'name', newName)
                         );
                     }
-                    var hashes = dataNodes.map(n => this.core.getAttribute(n, 'data'));
+                    var hashes = dataNodes.map(n => this.getAttribute(n, 'data'));
                     this.logger.info(`saving hashes: ${hashes.map(h => `"${h}"`)}`);
                 } else if (allDataNodes.length === 0) {
                     this.logger.warn('No data nodes found!');

src/common/viz/Execute.js

@@ -102,15 +102,9 @@ define([
             .fail(err => this.logger.error(`Job cancel failed: ${err}`));
     };
 
-    Execute.prototype.stopJob = function(job, silent) {
-        var jobId;
-
-        job = job || this.client.getNode(this._currentNodeId);
-        jobId = job.getId();
-
-        this.silentStopJob(job);
-
+    Execute.prototype._setJobStopped = function(jobId, silent) {
         if (!silent) {
+            var name = this.client.getNode(jobId).getAttribute('name');
             this.client.startTransaction(`Stopping "${name}" job`);
         }
 
@@ -123,6 +117,16 @@ define([
         }
     };
 
+    Execute.prototype.stopJob = function(job, silent) {
+        var jobId;
+
+        job = job || this.client.getNode(this._currentNodeId);
+        jobId = job.getId();
+
+        this.silentStopJob(job);
+        this._setJobStopped(jobId, silent);
+    };
+
 
     Execute.prototype.loadChildren = function(id) {
         var deferred = Q.defer(),
@@ -165,14 +169,17 @@ define([
     };
 
     Execute.prototype._stopExecution = function(execNode, inTransaction) {
-        var msg = `Canceling ${execNode.getAttribute('name')} execution`;
+        var msg = `Canceling ${execNode.getAttribute('name')} execution`,
+            jobIds;
 
         if (!inTransaction) {
             this.client.startTransaction(msg);
         }
 
-        this._silentStopExecution(execNode);
+        jobIds = this._silentStopExecution(execNode);
+
         this.client.setAttributes(execNode.getId(), 'status', 'canceled');
+        jobIds.forEach(jobId => this._setJobStopped(jobId, true));
 
         if (!inTransaction) {
             this.client.completeTransaction();
@@ -180,11 +187,13 @@ define([
     };
 
     Execute.prototype._silentStopExecution = function(execNode) {
-        var jobIds = execNode.getChildrenIds();
+        var runningJobIds = execNode.getChildrenIds()
+            .map(id => this.client.getNode(id))
+            .filter(job => this.isRunning(job));  // get running jobs
 
-        jobIds.map(id => this.client.getNode(id))
-            .filter(job => this.isRunning(job))  // get running jobs
-            .forEach(job => this.silentStopJob(job));  // stop them
+        runningJobIds.forEach(job => this.silentStopJob(job));  // stop them
+
+        return runningJobIds;
     };
 
     return Execute;

src/plugins/ExecuteJob/ExecuteJob.js

@@ -2,19 +2,22 @@
 /*jshint node:true, browser:true*/
 
 define([
+    'common/util/assert',
     'common/storage/constants',
     'text!./metadata.json',
     'executor/ExecutorClient',
     'plugin/PluginBase',
     'deepforge/plugin/LocalExecutor',
     'deepforge/plugin/PtrCodeGen',
-    'deepforge/JobLogsClient',
+    'deepforge/api/JobLogsClient',
+    'deepforge/api/JobOriginClient',
     'deepforge/Constants',
     'deepforge/utils',
     './templates/index',
     'q',
     'underscore'
 ], function (
+    assert,
     STORAGE_CONSTANTS,
     pluginMetadata,
     ExecutorClient,
@@ -22,6 +25,7 @@ define([
     LocalExecutor,  // DeepForge operation primitives
     PtrCodeGen,
     JobLogsClient,
+    JobOriginClient,
     CONSTANTS,
     utils,
     Templates,
@@ -33,7 +37,9 @@ define([
     pluginMetadata = JSON.parse(pluginMetadata);
 
     var OUTPUT_INTERVAL = 1500,
-        STDOUT_FILE = 'job_stdout.txt';
+        STDOUT_FILE = 'job_stdout.txt',
+        CREATE_PREFIX = 'created_node_',
+        INDEX = 1;
 
     /**
      * Initializes a new instance of ExecuteJob.
@@ -53,7 +59,12 @@ define([
         this._oldMetadataByName = {};  // name -> id
         this.lastAppliedCmd = {};
         this.canceled = false;
+
         this.changes = {};
+        this.currentChanges = {};  // read-only changes being applied
+        this.creations = {};
+        this.deletions = [];
+        this.createIdToMetadataId = {};
         this.logManager = null;
     };
 
@@ -69,6 +80,20 @@ define([
     ExecuteJob.prototype = Object.create(PluginBase.prototype);
     ExecuteJob.prototype.constructor = ExecuteJob;
 
+    ExecuteJob.prototype.configure = function () {
+        var result = PluginBase.prototype.configure.apply(this, arguments),
+            params = {
+                logger: this.logger,
+                port: this.gmeConfig.server.port,
+                branchName: this.branchName,
+                projectId: this.projectId
+            };
+
+        this.logManager = new JobLogsClient(params);
+        this.originManager = new JobOriginClient(params);
+        return result;
+    };
+
     /**
      * Main function for the plugin to execute. This will perform the execution.
      * Notes:
@@ -81,19 +106,21 @@ define([
     ExecuteJob.prototype.main = function (callback) {
         // Check the activeNode to make sure it is a valid node
         var type = this.core.getMetaType(this.activeNode),
-            typeName = type && this.getAttribute(type, 'name');
+            typeName = type && this.getAttribute(type, 'name'),
+            execNode,
+            status;
 
         if (typeName !== 'Job') {
             return callback(`Cannot execute ${typeName} (expected Job)`, this.result);
         }
 
-        // Get the gmeConfig...
-        this.logManager = new JobLogsClient({
-            logger: this.logger,
-            port: this.gmeConfig.server.port,
-            branchName: this.branchName,
-            projectId: this.projectId
-        });
+        // Set the parent execution to 'running'
+        execNode = this.core.getParent(this.activeNode);
+        status = this.getAttribute(execNode, 'status');
+        if (status !== 'running') {
+            this.setAttribute(execNode, 'status', 'running');
+        }
+
         this._callback = callback;
         this.currentForkName = null;
         this.prepare()
@@ -117,12 +144,51 @@ define([
         });
     };
 
+    //////////////////////////// Safe Save ////////////////////////////
+    ExecuteJob.prototype.getCreateId = function () {
+        return CREATE_PREFIX + (++INDEX);
+    };
+
+    ExecuteJob.prototype.isCreateId = function (id) {
+        return (typeof id === 'string') && (id.indexOf(CREATE_PREFIX) === 0);
+    };
+
+    ExecuteJob.prototype.createNode = function (baseType, parent) {
+        var id = this.getCreateId(),
+            parentId = this.isCreateId(parent) ? parent : this.core.getPath(parent);
+
+        this.logger.info(`Creating ${id} of type ${baseType} in ${parentId}`);
+        assert(this.META[baseType], `Cannot create node w/ unrecognized type: ${baseType}`);
+        this.creations[id] = {
+            base: baseType,
+            parent: parentId
+        };
+        return id;
+    };
+
+    ExecuteJob.prototype.deleteNode = function (nodeId) {
+        this.deletions.push(nodeId);
+    };
+
     ExecuteJob.prototype.delAttribute = function (node, attr) {
         return this.setAttribute(node, attr, null);
     };
 
     ExecuteJob.prototype.setAttribute = function (node, attr, value) {
-        var nodeId = this.core.getPath(node);
+        var nodeId;
+
+        if (this.isCreateId(node)) {
+            nodeId = node;
+        } else {
+            nodeId = this.core.getPath(node);
+            assert(typeof nodeId === 'string', `Cannot set attribute of ${nodeId}`);
+        }
+
+        if (value !== null) {
+            this.logger.info(`Setting ${attr} of ${nodeId} to ${value}`);
+        } else {
+            this.logger.info(`Deleting ${attr} of ${nodeId}`);
+        }
 
         if (!this.changes[nodeId]) {
             this.changes[nodeId] = {};
@@ -131,32 +197,53 @@ define([
     };
 
     ExecuteJob.prototype.getAttribute = function (node, attr) {
-        var nodeId = this.core.getPath(node),
-            base,
-            baseId;
+        var nodeId;
 
-        // Check the changes; fallback on actual node
-        if (this.changes[nodeId] && this.changes[nodeId][attr] !== undefined) {
-            // If deleted the attribute, get the default (inherited) value
-            if (this.changes[nodeId][attr] === null) {
-                base = this.core.getBase(node);
-                baseId = this.core.getPath(base);
-                return this.getAttribute(baseId, attr);
-            }
-            return this.changes[nodeId][attr];
+        assert(this.deletions.indexOf(nodeId) === -1,
+            `Cannot get ${attr} from deleted node ${nodeId}`);
+
+        // Check if it was newly created
+        if (this.isCreateId(node)) {
+            nodeId = node;
+            assert(this.creations[nodeId], `Creation node not updated: ${nodeId}`);
+            node = this.META[this.creations[nodeId].base];
+        } else {
+            nodeId = this.core.getPath(node);
+        }
+
+        // Check the most recent changes, then the currentChanges, then the model
+        var value = this._getValueFrom(nodeId, attr, node, this.changes) ||
+            this._getValueFrom(nodeId, attr, node, this.currentChanges);
+
+        if (value) {
+            return value;
         }
 
         return this.core.getAttribute(node, attr);
     };
 
+    ExecuteJob.prototype._getValueFrom = function (nodeId, attr, node, changes) {
+        var base;
+        if (changes[nodeId] && changes[nodeId][attr] !== undefined) {
+            // If deleted the attribute, get the default (inherited) value
+            if (changes[nodeId][attr] === null) {
+                base = this.isCreateId(nodeId) ? node : this.core.getBase(node);
+                return this.getAttribute(base, attr);
+            }
+            return changes[nodeId][attr];
+        }
+    };
+
     ExecuteJob.prototype._applyNodeChanges = function (node, changes) {
         var attr,
             value;
 
+        this.logger.info(`About to apply changes for ${this.core.getPath(node)}`);
         for (var i = changes.length; i--;) {
             attr = changes[i][0];
             value = changes[i][1];
             if (value !== null) {
+                this.logger.info(`Setting ${attr} to ${value} (${this.core.getPath(node)})`);
                 this.core.setAttribute(node, attr, value);
             } else {
                 this.core.delAttribute(node, attr);
@@ -165,6 +252,12 @@ define([
         return node;
     };
 
+    ExecuteJob.prototype.applyModelChanges = function () {
+        return this.applyCreations()
+            .then(() => this.applyChanges())
+            .then(() => this.applyDeletions());
+    };
+
     ExecuteJob.prototype.applyChanges = function () {
         var nodeIds = Object.keys(this.changes),
             attrs,
@@ -175,6 +268,8 @@ define([
             id,
             promise;
 
+        this.logger.info('Collecting changes to apply in commit');
+
         for (var i = nodeIds.length; i--;) {
             changes = [];
             attrs = Object.keys(this.changes[nodeIds[i]]);
@@ -183,27 +278,165 @@ define([
                 changes.push([attrs[a], value]);
             }
             changesFor[nodeIds[i]] = changes;
+
+            assert(changes, `changes are invalid for ${nodeIds[i]}: ${changes}`);
+            assert(!this.isCreateId(nodeIds[i]),
+                `Creation id not resolved to actual id: ${nodeIds[i]}`);
             promise = this.core.loadByPath(this.rootNode, nodeIds[i]);
             promises.push(promise);
         }
 
+        this.currentChanges = this.changes;
         this.changes = {};
+        // Need to differentiate between read/write changes.
+        this.logger.info(`About to apply changes for ${promises.length} nodes`);
+        return Q.all(promises)
+            .then(nodes => {
+                for (var i = nodes.length; i--;) {
+                    id = this.core.getPath(nodes[i]);
+                    assert(nodes[i], `node is ${nodes[i]} (${nodeIds[i]})`);
+                    this._applyNodeChanges(nodes[i], changesFor[id]);
+                }
+
+                // Local model is now up-to-date. No longer need currentChanges
+                this.currentChanges = {};
+            });
+    };
+
+    ExecuteJob.prototype.applyCreations = function () {
+        var nodeIds = Object.keys(this.creations),
+            tiers = this.createCreationTiers(nodeIds),
+            creations = this.creations,
+            newIds = {},
+            promise = Q(),
+            tier;
+
+        this.logger.info('Applying node creations');
+        for (var i = 0; i < tiers.length; i++) {
+            tier = tiers[i];
+            // Chain the promises, loading each tier sequentially
+            promise = promise.then(this.applyCreationTier.bind(this, creations, newIds, tier));
+        }
+
+        this.creations = {};
+        return promise;
+    };
+
+    ExecuteJob.prototype.applyCreationTier = function (creations, newIds, tier) {
+        var promises = [],
+            parentId,
+            node;
+
+        for (var j = tier.length; j--;) {
+            node = creations[tier[j]];
+            assert(node, `Could not find create info for ${tier[j]}`);
+            parentId = newIds[node.parent] || node.parent;
+            promises.push(this.applyCreation(tier[j], node.base, parentId));
+        }
         return Q.all(promises).then(nodes => {
-            for (var i = nodes.length; i--;) {
-                id = this.core.getPath(nodes[i]);
-                this._applyNodeChanges(nodes[i], changesFor[id]);
+            // Record the newIds so they can be used to resolve creation ids
+            // in subsequent tiers
+            for (var i = tier.length; i--;) {
+                newIds[tier[i]] = this.core.getPath(nodes[i]);
             }
         });
     };
 
+    // Figure out the dependencies between nodes to create.
+    // eg, if newId1 is to be created in newId2, then newId2 will
+    // be in an earlier tier than newId1. Essentially a topo-sort
+    // on a tree structure
+    ExecuteJob.prototype.createCreationTiers = function (nodeIds) {
+        var tiers = [],
+            prevTier = {},
+            tier = {},
+            id,
+            prevLen,
+            i;
+
+        // Create first tier (created inside existing nodes)
+        for (i = nodeIds.length; i--;) {
+            id = nodeIds[i];
+            if (!this.isCreateId(this.creations[id].parent)) {
+                tier[id] = true;
+                nodeIds.splice(i, 1);
+            }
+        }
+        prevTier = tier;
+        tiers.push(Object.keys(tier));
+
+        // Now, each tier consists of the nodes to be created inside a
+        // node from the previous tier
+        while (nodeIds.length) {
+            prevLen = nodeIds.length;
+            tier = {};
+            for (i = nodeIds.length; i--;) {
+                id = nodeIds[i];
+                if (prevTier[this.creations[id].parent]) {
+                    tier[id] = true;
+                    nodeIds.splice(i, 1);
+                }
+            }
+            prevTier = tier;
+            tiers.push(Object.keys(tier));
+            // Every iteration should find at least one node
+            assert(prevLen > nodeIds.length,
+                `Created empty create tier! Remaining: ${nodeIds.join(', ')}`);
+        }
+
+        return tiers;
+    };
+
+    ExecuteJob.prototype.applyCreation = function (tmpId, baseType, parentId) {
+        var base = this.META[baseType],
+            nodeId,
+            id;
+
+        this.logger.info(`Applying creation of ${tmpId} (${baseType}) in ${parentId}`);
+
+        assert(!this.isCreateId(parentId),
+            `Did not resolve parent id: ${parentId} for ${tmpId}`);
+        assert(base, `Invalid base type: ${baseType}`);
+        return this.core.loadByPath(this.rootNode, parentId)
+            .then(parent => this.core.createNode({base, parent}))
+            .then(node => {  // Update the _metadata records
+                id = this.createIdToMetadataId[tmpId];
+                delete this.createIdToMetadataId[tmpId];
+                this._metadata[id] = node;
+
+                // Update creations
+                nodeId = this.core.getPath(node);
+                if (this.changes[tmpId]) {
+                    assert(!this.changes[nodeId],
+                        `Newly created node cannot already have changes! (${nodeId})`);
+                    this.changes[nodeId] = this.changes[tmpId];
+                    delete this.changes[tmpId];
+                }
+                return node;
+            });
+    };
+
+    ExecuteJob.prototype.applyDeletions = function () {
+        var deletions = this.deletions;
+
+        this.deletions = [];
+        return Q.all(deletions.map(id => this.core.loadByPath(this.rootNode, id)))
+            .then(nodes => {
+                for (var i = nodes.length; i--;) {
+                    this.core.deleteNode(nodes[i]);
+                }
+            });
+    };
+
     // Override 'save' to notify the user on fork
     ExecuteJob.prototype.save = function (msg) {
         var name = this.getAttribute(this.activeNode, 'name');
+
         return this.updateForkName(name)
-            .then(() => this.applyChanges())
+            .then(() => this.applyModelChanges())
             .then(() => PluginBase.prototype.save.call(this, msg))
             .then(result => {
-                var msg;
+                this.logger.info(`Save finished w/ status: ${result.status}`);
                 if (result.status === STORAGE_CONSTANTS.FORKED) {
                     msg = `"${name}" execution has forked to "${result.forkName}"`;
                     this.currentForkName = result.forkName;
@@ -228,9 +461,37 @@ define([
                 this.rootNode = rootObject;
                 return this.core.loadByPath(rootObject,activeId);
             })
-            .then(activeObject => this.activeNode = activeObject);
+            .then(activeObject => this.activeNode = activeObject)
+            .then(() => {
+                var metaNames = Object.keys(this.META);
+
+                return Q.all(metaNames.map(name => this.updateMetaNode(name)));
+            })
+            .then(() => {
+                var mdNodes,
+                    mdIds;
+
+                mdIds = Object.keys(this._metadata)
+                    .filter(id => !this.isCreateId(this._metadata[id]));
+
+                mdNodes = mdIds.map(id => this.core.getPath(this._metadata[id]))
+                    .map(nodeId => this.core.loadByPath(this.rootNode, nodeId));
+
+                return Q.all(mdNodes).then(nodes => {
+                    for (var i = nodes.length; i--;) {
+                        this._metadata[mdIds[i]] = nodes[i];
+                    }
+                });
+            });
     };
 
+    ExecuteJob.prototype.updateMetaNode = function (name) {
+        var id = this.core.getPath(this.META[name]);
+        return this.core.loadByPath(this.rootNode, id).then(node => this.META[name] = node);
+    };
+
+    //////////////////////////// END Safe Save ////////////////////////////
+
     ExecuteJob.prototype.getConnections = function (nodes) {
         var conns = [];
         for (var i = nodes.length; i--;) {
@@ -279,7 +540,8 @@ define([
             idsToDelete = [],
             type,
             base,
-            child;
+            child,
+            i;
 
         this.lastAppliedCmd[nodeId] = 0;
         this._oldMetadataByName[nodeId] = {};
@@ -287,7 +549,7 @@ define([
         return this.core.loadChildren(job)
             .then(jobChildren => {
                 // Remove any metadata nodes
-                for (var i = jobChildren.length; i--;) {
+                for (i = jobChildren.length; i--;) {
                     child = jobChildren[i];
                     if (this.isMetaTypeOf(child, this.META.Metadata)) {
                         id = this.core.getPath(child);
@@ -310,18 +572,22 @@ define([
                 }
 
                 // make the deletion ids relative to the job node
-                idsToDelete = idsToDelete.map(id => id.replace(nodeId, ''));
-                return Q.all(idsToDelete.map(id => this.core.loadByPath(job, id)));
-            })
-            .then(nodes => nodes.forEach(node => this.core.deleteNode(node)));
+                this.logger.debug(`About to delete ${idsToDelete.length}: ${idsToDelete.join(', ')}`);
+                for (i = idsToDelete.length; i--;) {
+                    this.deleteNode(idsToDelete[i]);
+                }
+            });
     };
 
     ExecuteJob.prototype.clearOldMetadata = function (job) {
         var nodeId = this.core.getPath(job),
-            nodeIds = Object.keys(this._markForDeletion[nodeId]);
+            nodeIds = Object.keys(this._markForDeletion[nodeId]),
+            node;
 
+        this.logger.debug(`About to delete ${nodeIds.length}: ${nodeIds.join(', ')}`);
         for (var i = nodeIds.length; i--;) {
-            this.core.deleteNode(this._markForDeletion[nodeId][nodeIds[i]]);
+            node = this._markForDeletion[nodeId][nodeIds[i]];
+            this.deleteNode(this.core.getPath(node));
         }
         delete this.lastAppliedCmd[nodeId];
         delete this._markForDeletion[nodeId];
@@ -472,20 +738,32 @@ define([
                 })
                 .then(mds => {
                     // Record the large files
-                    var inputData = {};
+                    var inputData = {},
+                        runsh = '# Bash script to download data files and run job\n' +
+                        'if [ -z "$DEEPFORGE_URL" ]; then\n  echo "Please set DEEPFORGE_URL and' +
+                        ' re-run:"\n  echo ""  \n  echo "  DEEPFORGE_URL=http://my.' +
+                        'deepforge.server.com:8080 bash run.sh"\n  echo ""\n exit 1\nfi\n';
+
                     mds.forEach((metadata, i) => {
                         // add the hashes for each input
                         var input = inputs[i], 
-                            hash = files.inputAssets[input];
+                            hash = files.inputAssets[input],
+                            dataPath = 'inputs/' + input + '/data',
+                            url = this.blobClient.getRelativeDownloadURL(hash);
 
-                        inputData['inputs/' + input + '/data'] = {
+                        inputData[dataPath] = {
                             req: hash,
                             cache: metadata.content
                         };
+
+                        // Add to the run.sh file
+                        runsh += `wget $DEEPFORGE_URL${url} -O ${dataPath}\n`;
                     });
 
                     delete files.inputAssets;
                     files['input-data.json'] = JSON.stringify(inputData, null, 2);
+                    runsh += 'th init.lua';
+                    files['run.sh'] = runsh;
 
                     // Add pointer assets
                     Object.keys(files.ptrAssets)
@@ -589,12 +867,26 @@ define([
                 if (info.secret) {  // o.w. it is a cached job!
                     this.setAttribute(job, 'secret', info.secret);
                 }
-                return this.watchOperation(executor, hash, opNode, job);
+                return this.recordJobOrigin(hash, job);
             })
+            .then(() => this.watchOperation(executor, hash, opNode, job))
             .catch(err => this.logger.error(`Could not execute "${name}": ${err}`));
 
     };
 
+    ExecuteJob.prototype.recordJobOrigin = function (hash, job) {
+        var execNode = this.core.getParent(job),
+            info;
+
+        info = {
+            hash: hash,
+            nodeId: this.core.getPath(job),
+            job: this.getAttribute(job, 'name'),
+            execution: this.getAttribute(execNode, 'name')
+        };
+        return this.originManager.record(hash, info);
+    };
+
     ExecuteJob.prototype.createOperationFiles = function (node) {
         var files = {};
         // For each operation, generate the output files:
@@ -865,8 +1157,8 @@ define([
     };
 
     ExecuteJob.prototype.createAttributeFile = function (node, files) {
-        var skip = ['code'],
-            numRegex = /^\d+\.?\d*((e|e-)\d+)?$/,
+        var skip = ['code', 'stdout', 'execFiles', 'jobId', 'secret'],
+            numOrBool = /^(-?\d+\.?\d*((e|e-)\d+)?|(true|false))$/,
             table;
 
         this.logger.info('Creating attributes file...');
@@ -874,7 +1166,7 @@ define([
             .filter(attr => skip.indexOf(attr) === -1)
             .map(name => {
                 var value = this.getAttribute(node, name);
-                if (!numRegex.test(value)) {
+                if (!numOrBool.test(value)) {
                     value = `"${value}"`;
                 }
                 return [name, value];
@@ -920,6 +1212,11 @@ define([
         });
     };
 
+    ExecuteJob.prototype.isExecutionCanceled = function () {
+        var execNode = this.core.getParent(this.activeNode);
+        return this.getAttribute(execNode, 'status') === 'canceled';
+    };
+
     ExecuteJob.prototype.watchOperation = function (executor, hash, op, job) {
         var jobId = this.core.getPath(job),
             opId = this.core.getPath(op),
@@ -928,7 +1225,7 @@ define([
             name = this.getAttribute(job, 'name');
 
         // If canceled, stop the operation
-        if (this.canceled) {
+        if (this.canceled || this.isExecutionCanceled()) {
             secret = this.getAttribute(job, 'secret');
             if (secret) {
                 executor.cancelJob(hash, secret);
@@ -954,6 +1251,7 @@ define([
                                 last = stdout.lastIndexOf('\n'),
                                 result,
                                 lastLine,
+                                next = Q(),
                                 msg;
 
                             // parse deepforge commands
@@ -967,13 +1265,15 @@ define([
 
                             if (output) {
                                 // Send notification to all clients watching the branch
-                                this.logManager.appendTo(jobId, output)
+                                next = next
+                                    .then(() => this.logManager.appendTo(jobId, output))
                                     .then(() => this.notifyStdoutUpdate(jobId));
                             }
                             if (result.hasMetadata) {
                                 msg = `Updated graph/image output for ${name}`;
-                                return this.save(msg);
+                                next = next.then(() => this.save(msg));
                             }
+                            return next;
                         });
                 }
             })
@@ -1186,14 +1486,12 @@ define([
         // Check if the graph already exists
         graph = this._getExistingMetadata(jobId, 'Graph', name);
         if (!graph) {
-            graph = this.core.createNode({
-                base: this.META.Graph,
-                parent: job
-            });
+            graph = this.createNode('Graph', job);
 
             if (name) {
                 this.setAttribute(graph, 'name', name);
             }
+            this.createIdToMetadataId[graph] = id;
         }
 
         this._metadata[id] = graph;
@@ -1201,25 +1499,25 @@ define([
 
     ExecuteJob.prototype[CONSTANTS.GRAPH_PLOT] = function (job, id, x, y) {
         var jobId = this.core.getPath(job),
-            nonNum = /[^\d\.]*/g,
-            graph,
+            nonNum = /[^\d-\.]*/g,
+            line,
             points;
             
 
         id = jobId + '/' + id;
         this.logger.info(`Adding point ${x}, ${y} to ${id}`);
-        graph = this._metadata[id];
-        if (!graph) {
-            this.logger.warn(`Can't add point to non-existent graph: ${id}`);
+        line = this._metadata[id];
+        if (!line) {
+            this.logger.warn(`Can't add point to non-existent line: ${id}`);
             return;
         }
 
         // Clean the points by removing and special characters
         x = x.replace(nonNum, '');
         y = y.replace(nonNum, '');
-        points = this.getAttribute(graph, 'points');
+        points = this.getAttribute(line, 'points');
         points += `${x},${y};`;
-        this.setAttribute(graph, 'points', points);
+        this.setAttribute(line, 'points', points);
     };
 
     ExecuteJob.prototype[CONSTANTS.GRAPH_CREATE_LINE] = function (job, graphId, id) {
@@ -1230,12 +1528,10 @@ define([
 
         // Create a 'line' node in the given Graph metadata node
         name = name.replace(/\s+$/, '');
-        line = this.core.createNode({
-            base: this.META.Line,
-            parent: graph
-        });
+        line = this.createNode('Line', graph);
         this.setAttribute(line, 'name', name);
         this._metadata[jobId + '/' + id] = line;
+        this.createIdToMetadataId[line] = jobId + '/' + id;
     };
 
     ExecuteJob.prototype[CONSTANTS.IMAGE.BASIC] =
@@ -1265,11 +1561,9 @@ define([
             imageNode = this._getExistingMetadata(jobId, 'Image', name);
             if (!imageNode) {
                 this.logger.info(`Creating image ${id} named ${name}`);
-                imageNode = this.core.createNode({
-                    base: this.META.Image,
-                    parent: job
-                });
+                imageNode = this.createNode('Image', job);
                 this.setAttribute(imageNode, 'name', name);
+                this.createIdToMetadataId[imageNode] = id;
             }
             this._metadata[id] = imageNode;
         }

src/plugins/ExecutePipeline/ExecutePipeline.js

@@ -4,7 +4,6 @@
 define([
     'plugin/CreateExecution/CreateExecution/CreateExecution',
     'plugin/ExecuteJob/ExecuteJob/ExecuteJob',
-    'deepforge/JobLogsClient',
     'common/storage/constants',
     'common/core/constants',
     'q',
@@ -13,7 +12,6 @@ define([
 ], function (
     CreateExecution,
     ExecuteJob,
-    JobLogsClient,
     STORAGE_CONSTANTS,
     CONSTANTS,
     Q,
@@ -37,6 +35,10 @@ define([
 
         this._currentSave = Q();
         this.changes = {};
+        this.currentChanges = {};  // read-only changes being applied
+        this.creations = {};
+        this.deletions = [];
+        this.createIdToMetadataId = {};
         this.initRun();
     };
 
@@ -114,13 +116,6 @@ define([
             return callback('Current node is not a Pipeline or Execution!', this.result);
         }
 
-        // Get the gmeConfig...
-        this.logManager = new JobLogsClient({
-            logger: this.logger,
-            port: this.gmeConfig.server.port,
-            branchName: this.branchName,
-            projectId: this.projectId
-        });
         this._callback = callback;
         this.currentForkName = null;
 
@@ -149,7 +144,7 @@ define([
         // before continuing
         this._currentSave = this._currentSave
             .then(() => this.updateForkName(this.pipelineName))
-            .then(() => this.applyChanges())
+            .then(() => this.applyModelChanges())
             .then(() => CreateExecution.prototype.save.call(this, msg))
             .then(result => {
                 var msg;
@@ -185,6 +180,10 @@ define([
         });
     };
 
+    ExecutePipeline.prototype.isExecutionCanceled = function () {
+        return this.getAttribute(this.activeNode, 'status') === 'canceled';
+    };
+
     ExecutePipeline.prototype.isInputData = function (node) {
         var prnt = this.core.getParent(node);
         return this.core.isTypeOf(prnt, this.META.Inputs);

src/routers/JobLogsAPI/JobLogManager.js

@@ -12,7 +12,7 @@ var JobLogManager = function(logger, config) {
 };
 
 JobLogManager.prototype._getFilePath = function(jInfo) {
-    this.logger.info(`getting file path for ${jInfo.job} in ${jInfo.project} on ${jInfo.branch}`);
+    this.logger.debug(`getting file path for ${jInfo.job} in ${jInfo.project} on ${jInfo.branch}`);
     var jobId = jInfo.job.replace(/\//g, '_'),
         filename = `${jobId}.txt`;
 
@@ -71,7 +71,7 @@ JobLogManager.prototype.migrate = function(migrationInfo, jobIds) {
     }
 
     // Copy the job files and evaluate each of the finish functions
-    this.logger.info('migrating from ' + migrationInfo.srcBranch + ' to '+ migrationInfo.dstBranch);
+    this.logger.debug('migrating from ' + migrationInfo.srcBranch + ' to '+ migrationInfo.dstBranch);
     return Q.all(jobIds.map(jobId => {
         src = this._getFilePath({
             project: migrationInfo.project,
@@ -112,7 +112,7 @@ JobLogManager.prototype.appendTo = function(jobInfo, logs) {
         branchDirname = path.dirname(filename),
         projDirname = path.dirname(branchDirname);
 
-    this.logger.info(`Appending content to ${filename}`);
+    this.logger.debug(`Appending content to ${filename}`);
     // Make directory if needed
     return this.mkdirIfNeeded(this.rootDir)
         .then(() => this.mkdirIfNeeded(projDirname))
@@ -123,7 +123,7 @@ JobLogManager.prototype.appendTo = function(jobInfo, logs) {
 JobLogManager.prototype.getLog = function(jobInfo) {
     var filename = this._getFilePath(jobInfo);
 
-    this.logger.info(`Getting log content to ${filename}`);
+    this.logger.info(`Getting log content from ${filename}`);
     return this.exists(jobInfo)
         .then(exists => {
             if (exists) {
@@ -142,7 +142,7 @@ JobLogManager.prototype.delete = function(jobInfo) {
                 this.logger.debug(`Removing file ${filename}`);
                 return Q.nfcall(fs.unlink, filename);
             }
-            this.logger.info(`${filename} doesn't exist. No need to delete...`);
+            this.logger.debug(`${filename} doesn't exist. No need to delete...`);
         });
 };
 

src/routers/JobOriginAPI/JobOriginAPI.js

@@ -0,0 +1,161 @@
+/*jshint node:true*/
+
+/**
+ * This is an API to record the executor job hash to originating project, execution and job
+ */
+
+'use strict';
+
+// This contains
+var express = require('express'),
+    MONGO_COLLECTION = 'JobOrigins',
+    Q = require('q'),
+    router = express.Router(),
+    mongodb = require('mongodb');
+
+/**
+ * Called when the server is created but before it starts to listening to incoming requests.
+ * N.B. gmeAuth, safeStorage and workerManager are not ready to use until the start function is called.
+ * (However inside an incoming request they are all ensured to have been initialized.)
+ *
+ * @param {object} middlewareOpts - Passed by the webgme server.
+ * @param {GmeConfig} middlewareOpts.gmeConfig - GME config parameters.
+ * @param {GmeLogger} middlewareOpts.logger - logger
+ * @param {function} middlewareOpts.ensureAuthenticated - Ensures the user is authenticated.
+ * @param {function} middlewareOpts.getUserId - If authenticated retrieves the userId from the request.
+ * @param {object} middlewareOpts.gmeAuth - Authorization module.
+ * @param {object} middlewareOpts.safeStorage - Accesses the storage and emits events (PROJECT_CREATED, COMMIT..).
+ * @param {object} middlewareOpts.workerManager - Spawns and keeps track of "worker" sub-processes.
+ */
+// When testing, use in memory storage...
+function initialize(middlewareOpts) {
+    var logger = middlewareOpts.logger.fork('JobOriginAPI'),
+        gmeConfig = middlewareOpts.gmeConfig,
+        ensureAuthenticated = middlewareOpts.ensureAuthenticated,
+        REQUIRED_FIELDS = ['hash', 'project', 'execution', 'job', 'nodeId', 'branch'],
+        mongo;
+
+    logger.debug('initializing ...');
+
+    // Ensure authenticated can be used only after this rule.
+    router.use('*', function (req, res, next) {
+        // This header ensures that any failures with authentication won't redirect.
+        res.setHeader('X-WebGME-Media-Type', 'webgme.v1');
+        next();
+    });
+
+    // Use ensureAuthenticated if the routes require authentication. (Can be set explicitly for each route.)
+    router.use('*', ensureAuthenticated);
+
+    // Connect to mongo...
+    Q.nfcall(mongodb.MongoClient.connect, gmeConfig.mongo.uri, gmeConfig.mongo.options)
+        .then(db => {
+            mongo = db.collection(MONGO_COLLECTION);
+            logger.debug('Connected to mongo!');
+        })
+        .catch(err => {
+            logger.error(`Could not connect to mongo: ${err}`);
+            throw err;
+        });
+
+    router.get('/:jobHash', function (req, res/*, next*/) {
+        var hash = req.params.jobHash,
+            jobInfo = {};
+
+        mongo.findOne({hash: hash})
+            .then(result => {
+                if (result) {
+                    // Filter the result object
+                    for (var i = REQUIRED_FIELDS.length; i--;) {
+                        jobInfo[REQUIRED_FIELDS[i]] = result[REQUIRED_FIELDS[i]];
+                    }
+                    return res.json(jobInfo);
+                }
+                res.sendStatus(404);
+            })
+            .catch(err => {
+                logger.error(`Storing job info failed: ${err}`);
+                res.status(500).send(err);
+            });
+    });
+
+    router.post('/:jobHash', function (req, res/*, next*/) {
+        var hash = req.params.jobHash,
+            jobInfo = {
+                hash: hash,
+                project: req.body.project,
+                execution: req.body.execution,
+                branch: req.body.branch,
+                job: req.body.job,  // job name
+                nodeId: req.body.nodeId
+            };
+
+        // Check that none of the fields are undefined
+        logger.debug(`Storing job info for ${hash}`);
+        var fields = REQUIRED_FIELDS;
+        for (var i = fields.length; i--;) {
+            if (!jobInfo[fields[i]]) {
+                return res.status(400).send(`Missing required field: ${fields[i]}`);
+            }
+        }
+
+        return mongo.insertOne(jobInfo)
+            .then(() => res.sendStatus(201))
+            .catch(err => {
+                logger.error(`Storing job info failed: ${err}`);
+                res.status(500).send(err.toString());
+            });
+    });
+
+    router.delete('/:jobHash', function (req, res/*, next*/) {
+        var hash = req.params.jobHash;
+
+        mongo.findOneAndDelete({hash: hash})
+            .then(() => res.sendStatus(204));
+    });
+
+    // on fork
+    router.patch('/:jobHash', function (req, res) {
+        var hash = req.params.jobHash;
+
+        if (!req.body.branch) {
+            return res.status(400).send('Missing "branch" field');
+        }
+
+        return mongo.findOneAndUpdate({hash: hash}, {$set: {branch: req.body.branch}})
+            .then(() => {
+                logger.debug('Finished updateOne!');
+                res.sendStatus(200);
+            })
+            .catch(err => {
+                logger.error(`Job update failed: ${err}`);
+                res.status(500).send(err);
+            });
+    });
+
+    logger.debug('ready');
+}
+
+/**
+ * Called before the server starts listening.
+ * @param {function} callback
+ */
+function start(callback) {
+    callback();
+}
+
+/**
+ * Called after the server stopped listening.
+ * @param {function} callback
+ */
+function stop(callback) {
+    callback();
+}
+
+
+module.exports = {
+    initialize: initialize,
+    router: router,
+    start: start,
+    stop: stop
+};

src/visualizers/Visualizers.json

@@ -100,5 +100,11 @@
     "title": "ExecutionIndex",
     "panel": "panels/ExecutionIndex/ExecutionIndexPanel",
     "DEBUG_ONLY": false
+  },
+  {
+    "id": "WorkerHeader",
+    "title": "WorkerHeader",
+    "panel": "panels/WorkerHeader/WorkerHeaderPanel",
+    "DEBUG_ONLY": false
   }
-]
+]

src/visualizers/panels/LogViewer/LogViewerControl.js

@@ -5,7 +5,7 @@
 // if the job is running, get the logs from the log-storage
 define([
     'q',
-    'deepforge/JobLogsClient',
+    'deepforge/api/JobLogsClient',
     'js/Constants',
     'deepforge/Constants',
     'panels/TextEditor/TextEditorControl'
@@ -82,7 +82,14 @@ define([
     };
 
     LogViewerControl.prototype._getRunningLogs = function (id) {
-        var logManager = new JobLogsClient({
+        var logManager;
+
+        if (!this._client.getActiveBranchName() || !this._client.getActiveProjectId()) {
+            // Logs are only stored for a given branch
+            return Q().then(() => '');
+        }
+
+        logManager = new JobLogsClient({
             logger: this._logger,
             projectId: this._client.getActiveProjectId(),
             branchName: this._client.getActiveBranchName()

src/visualizers/panels/WorkerHeader/ProjectNavigatorController.js

@@ -0,0 +1,106 @@
+/* globals define, WebGMEGlobal */
+define([
+    'js/Dialogs/Projects/ProjectsDialog',
+    './WorkerDialog',
+    'js/Panels/Header/ProjectNavigatorController'
+], function(
+    ProjectsDialog,
+    WorkerDialog,
+    GMEProjectNavigatorController
+) {
+    'use strict';
+    var ProjectNavigatorController = function() {
+        GMEProjectNavigatorController.apply(this, arguments);
+    };
+
+    ProjectNavigatorController.prototype = Object.create(GMEProjectNavigatorController.prototype);
+
+    ProjectNavigatorController.prototype.initialize = function () {
+        var self = this,
+            newProject,
+            manageProjects,
+            manageWorkers;
+
+
+        // initialize model structure for view
+        self.$scope.navigator = {
+            items: [],
+            separator: true
+        };
+
+
+        manageProjects = function (/*data*/) {
+            var pd = new ProjectsDialog(self.gmeClient);
+            pd.show();
+        };
+        newProject = function (data) {
+            var pd = new ProjectsDialog(self.gmeClient, true, data.newType);
+            pd.show();
+        };
+        self.userId = WebGMEGlobal.userInfo._id;
+
+        manageWorkers = function() {
+            // Create the worker dialog
+            var pd = new WorkerDialog(self.logger);
+            pd.show();
+        };
+
+        // initialize root menu
+        // projects id is mandatory
+        if (self.config.disableProjectActions === false) {
+            self.root.menu = [
+                {
+                    id: 'top',
+                    items: [
+                        {
+                            id: 'manageProject',
+                            label: 'Manage projects ...',
+                            iconClass: 'glyphicon glyphicon-folder-open',
+                            action: manageProjects,
+                            actionData: {}
+                        },
+                        {
+                            id: 'newProject',
+                            label: 'New project ...',
+                            disabled: WebGMEGlobal.userInfo.canCreate !== true,
+                            iconClass: 'glyphicon glyphicon-plus',
+                            action: newProject,
+                            actionData: {newType: 'seed'}
+                        },
+                        {
+                            id: 'importProject',
+                            label: 'Import project ...',
+                            disabled: WebGMEGlobal.userInfo.canCreate !== true,
+                            iconClass: 'glyphicon glyphicon-import',
+                            action: newProject,
+                            actionData: {newType: 'import'}
+                        },
+                        {
+                            id: 'manageWorkers',
+                            label: 'View workers ...',
+                            iconClass: 'glyphicon glyphicon-cloud',
+                            action: manageWorkers
+                        }
+                    ]
+                },
+                {
+                    id: 'projects',
+                    label: 'Recent projects',
+                    totalItems: 20,
+                    items: [],
+                    showAllItems: manageProjects
+                }
+            ];
+        }
+
+        self.initWithClient();
+
+        // only root is selected by default
+        self.$scope.navigator = {
+            items: self.config.disableProjectActions ? [] : [self.root],
+            separator: true
+        };
+    };
+
+    return ProjectNavigatorController;
+});

src/visualizers/panels/WorkerHeader/WorkerDialog.js

@@ -0,0 +1,239 @@
+/* globals define, $ */
+define([
+    'q',
+    'superagent',
+    'deepforge/viz/Utils',
+    'deepforge/api/JobOriginClient',
+    'text!./WorkerModal.html',
+    'text!./WorkerTemplate.html.ejs',
+    'text!./WorkerJobItem.html',
+    'css!./WorkerModal.css'
+], function(
+    Q,
+    superagent,
+    utils,
+    JobOriginClient,
+    WorkerHtml,
+    WorkerTemplate,
+    WorkerJobItem
+) {
+    'use strict';
+
+    var WORKER_ENDPOINT = '/rest/executor/worker',
+        JOBS_ENDPOINT = '/rest/executor';
+
+    var WorkerDialog = function(logger) {
+        this.workerDict = {};
+        this.workers = {};
+        this.runningWorkers = [];
+        this.jobsDict = {};
+        this.jobs = {};
+        this.active = false;
+        this.logger = logger.fork('WorkerDialog');
+        this.originManager = new JobOriginClient({
+            logger: this.logger
+        });
+    };
+
+    WorkerDialog.prototype.initialize = function() {
+        this._dialog = $(WorkerHtml);
+        this._table = this._dialog.find('.worker-list');
+        this.$noJobs = this._dialog.find('.no-jobs-msg');
+        this.$noWorkers = this._dialog.find('.no-workers-msg');
+        this._isShowingJobs = false;
+        this._isShowingWorkers = true;
+        this._queue = this._dialog.find('.job-queue-list');
+        this._dialog.modal('show');
+        this._dialog.on('hidden.bs.modal', () => this.active = false);
+    };
+
+    WorkerDialog.prototype.show = function() {
+        this.active = true;
+        this.update();
+        this.initialize();
+    };
+
+    WorkerDialog.prototype.get = function(url) {
+        var deferred = Q.defer();
+
+        superagent.get(url)
+            .end((err, res) => {
+                if (err) {
+                    return deferred.reject(err);
+                }
+                deferred.resolve(JSON.parse(res.text));
+            });
+
+        return deferred.promise;
+    };
+
+    WorkerDialog.prototype.update = function() {
+        // Poll the workers
+        return Q.all([
+            this.get(WORKER_ENDPOINT).then(workers => this.updateWorkers(workers)),
+            this.get(JOBS_ENDPOINT).then(jobs => this.updateJobs(jobs))
+        ]).then(() => {
+            if (this.active) {
+                setTimeout(this.update.bind(this), 1000);
+            }
+        })
+        .catch(err => this.logger.error('Update failed:', err));
+    };
+
+    WorkerDialog.prototype.updateWorkers = function(workerDict) {
+        var ids = Object.keys(workerDict),
+            oldWorkerIds,
+            visibleWorkers = false,
+            i;
+
+        this.runningWorkers = [];
+        for (i = ids.length; i--;) {
+            this.updateWorker(workerDict[ids[i]]);
+            visibleWorkers = true;
+            delete this.workerDict[ids[i]];
+        }
+        this.toggleNoWorkersMsg(!visibleWorkers);
+
+        // Clear old workers
+        oldWorkerIds = Object.keys(this.workerDict);
+        for (i = oldWorkerIds.length; i--;) {
+            this.removeWorker(oldWorkerIds[i]);
+        }
+
+        this.workerDict = workerDict;
+    };
+
+    WorkerDialog.prototype.updateWorker = function(worker) {
+        var row = this.workers[worker.clientId] || $(WorkerTemplate),
+            clazz;
+
+        worker.lastSeen = utils.getDisplayTime(worker.lastSeen*1000);
+        worker.status = worker.jobs.length ? 'RUNNING' : 'READY';
+
+        clazz = worker.status === 'RUNNING' ? 'warning' : 'success';
+        row[0].className = clazz;
+
+        row.find('.lastSeen').text(worker.lastSeen);
+        row.find('.clientId').text(worker.clientId);
+        row.find('.status').text(worker.status);
+        if (!this.workers[worker.clientId]) {
+            this._table.append(row);
+            this.workers[worker.clientId] = row;
+        }
+
+        if (worker.status === 'RUNNING') {
+            this.runningWorkers.push(worker);
+        }
+    };
+
+    WorkerDialog.prototype.removeWorker = function(workerId) {
+        this.workers[workerId].remove();
+        delete this.workers[workerId];
+    };
+
+    WorkerDialog.prototype.updateJobs = function(jobsDict) {
+        var allJobIds = Object.keys(jobsDict),
+            hasJobs = false,
+            id;
+
+        this.jobsDict = jobsDict;
+        for (var i = allJobIds.length; i--;) {
+            id = allJobIds[i];
+            if (this.jobs[id] || !this.isFinished(id)) {
+                hasJobs = this.updateJobItem(id) || hasJobs;
+            }
+        }
+        this.setNoJobsMessage(!hasJobs);  // hide if no queue
+    };
+
+    WorkerDialog.prototype.setNoJobsMessage = function(visible) {
+        var visibility = visible ? 'inherit' : 'none',
+            wasVisible = !this._isShowingJobs;
+
+        if (visible !== wasVisible) {
+            this.$noJobs.css('display', visibility);
+            this._isShowingJobs = !visible;
+        }
+    };
+
+    WorkerDialog.prototype.toggleNoWorkersMsg = function(visible) {
+        var visibility = visible ? 'inherit' : 'none';
+
+        if (visible !== this._isShowingWorkers) {
+            this.$noWorkers.css('display', visibility);
+            this._isShowingWorkers = visible;
+        }
+    };
+
+    WorkerDialog.prototype.isFinished = function(jobId) {
+        return this.jobsDict[jobId].status === 'FAILED_TO_EXECUTE' ||
+            this.jobsDict[jobId].status === 'SUCCESS' ||
+            this.jobsDict[jobId].status === 'CANCELED';
+    };
+
+    WorkerDialog.prototype.updateJobItemName = function(jobId) {
+        return this.originManager.getOrigin(jobId)
+            .then(info => {
+                var job = this.jobs[jobId],
+                    project = info.project.replace(/^guest\+/, '');
+
+                if (job && this.active) {
+                    if (info.branch !== 'master') {
+                        project += ' (' + info.branch + ')';
+                    }
+                    job.find('.job-id').text(info.job);
+                    job.find('.execution').text(info.execution);
+                    job.find('.project').text(project);
+                }
+            });
+    };
+
+    WorkerDialog.prototype.getWorkerWithJob = function(jobId) {
+        var jobs;
+
+        for (var i = this.runningWorkers.length; i--;) {
+            jobs = this.runningWorkers[i].jobs;
+            for (var j = jobs.length; j--;) {
+                if (jobs[j].hash === jobId) {
+                    return this.runningWorkers[i].clientId;
+                }
+            }
+        }
+
+        return 'unknown';
+    };
+
+    WorkerDialog.prototype.updateJobItem = function(jobId) {
+        var job = this.jobs[jobId] || $(WorkerJobItem),
+            info = this.jobsDict[jobId],
+            createdTime = new Date(info.createTime).getTime(),
+            clazz = utils.ClassForJobStatus[info.status.toLowerCase()],
+            status = info.status;
+
+        job[0].className = `job-tag ${clazz}`;
+
+        // Add the worker id if running
+        if (info.status.toLowerCase() === 'running') {
+            var workerId = this.getWorkerWithJob(jobId);
+            status += ' (' + workerId + ')';
+        }
+        job.find('.status').text(status);
+
+        if (!this.jobs[jobId]) {
+            job.find('.job-id').text('Loading');
+            job.find('.createdAt').text(utils.getDisplayTime(createdTime));
+            this.updateJobItemName(jobId);
+            this._queue.append(job);
+            this.jobs[jobId] = job;
+        }
+
+        if (this.isFinished(jobId)) {
+            job.remove();
+            delete this.jobs[jobId];
+            return false;
+        }
+        return true;
+    };
+
+    return WorkerDialog;
+});

src/visualizers/panels/WorkerHeader/WorkerHeaderPanel.js

@@ -0,0 +1,41 @@
+/*globals define, angular, _,*/
+/*jshint browser: true*/
+
+define([
+    'panels/BreadcrumbHeader/BreadcrumbHeaderPanel',
+    'js/Widgets/UserProfile/UserProfileWidget',
+    'js/Widgets/ConnectedUsers/ConnectedUsersWidget',
+    'js/Panels/Header/DefaultToolbar',
+    'panels/BreadcrumbHeader/NodePathNavigator',
+    'js/Toolbar/Toolbar',
+    './ProjectNavigatorController'
+], function (
+    BreadcrumbHeader,
+    UserProfileWidget,
+    ConnectedUsersWidget,
+    DefaultToolbar,
+    NodePathNavigator,
+    Toolbar,
+    ProjectNavigatorController
+) {
+    'use strict';
+
+    var HeaderPanel;
+
+    HeaderPanel = function (layoutManager, params) {
+        BreadcrumbHeader.call(this, layoutManager, params);
+    };
+
+    //inherit from PanelBaseWithHeader
+    _.extend(HeaderPanel.prototype, BreadcrumbHeader.prototype);
+
+    HeaderPanel.prototype._initialize = function () {
+        BreadcrumbHeader.prototype._initialize.call(this);
+        var app = angular.module('gmeApp');
+
+        app.controller('ProjectNavigatorController', ['$scope', 'gmeClient', '$timeout', '$window', '$http',
+            ProjectNavigatorController]);
+    };
+
+    return HeaderPanel;
+});

src/visualizers/panels/WorkerHeader/WorkerJobItem.html

@@ -0,0 +1,7 @@
+<tr>
+    <td class="job-id"></td>
+    <td class="execution">unknown</td>
+    <td class="project">unknown</td>
+    <td class="createdAt">unknown</td>
+    <td class="status">unknown</td>
+</tr>

src/visualizers/panels/WorkerHeader/WorkerModal.css

@@ -0,0 +1,47 @@
+.worker-modal .modal-content .modal-header span {
+  font-size: 28px;
+  vertical-align: middle;
+}
+
+.worker-modal .modal-content .modal-header .header-icon {
+  height: 28px;
+  width: 28px;
+  margin-right: 1ex;
+  display: inline-block;
+  vertical-align: middle;
+  background-size: 28px 28px;
+}
+
+.worker-modal .modal-content .modal-body th {
+  text-align: left;
+}
+
+.worker-modal .job-tag {
+  margin-right: 5px;
+}
+
+.worker-modal .queue-title {
+  float: left;
+  margin-right: 5px;
+  font-size: 1.4em;
+  color: #555555;
+  font-weight: bold;
+  width: 100%;
+  text-align: center;
+}
+
+.worker-modal .no-jobs-msg {
+  font-style: italic;
+  font-size: 1.3em;
+  color: #777;
+}
+
+.worker-modal .no-workers-msg {
+  font-style: italic;
+  font-size: 1.3em;
+  color: #777;
+}
+
+.worker-modal .job-queue {
+  padding-top: 1em;
+}

src/visualizers/panels/WorkerHeader/WorkerModal.html

@@ -0,0 +1,60 @@
+<div class="worker-modal modal fade in" tabindex="-1" role="dialog">
+    <div class="modal-dialog modal-lg">
+        <div class="modal-content">
+            <div class="modal-header">
+                <button type="button" class="close" data-dismiss="modal">x</button>
+                <i class="header-icon gme-icon"></i>
+                <span>Overview</span>
+            </div>
+            <div class="modal-body">
+                <div>
+                    <div class="queue-title">Connected Workers</div>
+                    <table class="table table-projects">
+                        <thead>
+                            <tr class="with-children">
+                                <th class="title-owner">Worker Id
+                                    <!--<i class="glyphicon glyphicon-sort-by-attributes-alt sorted in-order"></i>
+                                    <i class="glyphicon glyphicon-sort-by-attributes sorted rev-order"></i>-->
+                                </th>
+                                <th class="title-name">Last Seen
+                                    <!--<i class="glyphicon glyphicon-sort-by-attributes-alt sorted in-order"></i>
+                                    <i class="glyphicon glyphicon-sort-by-attributes sorted rev-order"></i>-->
+                                </th>
+                                <th class="title-modified">Status
+                                    <!--<i class="glyphicon glyphicon-sort-by-attributes-alt sorted in-order"></i>
+                                    <i class="glyphicon glyphicon-sort-by-attributes sorted rev-order"></i>-->
+                                </th>
+                            </tr>
+                            <!--<tr class="with-no-children">
+                                <th>No projects in this group...</th>
+                            </tr>-->
+                        </thead>
+                        <tbody class="worker-list">
+                            <tr><td class="no-workers-msg">No Connected Workers...</td></tr>
+                        </tbody>
+                    </table>
+                </div>
+                <div class="job-queue">
+                    <div class="queue-title">Job Queue</div>
+                    <table class="table table-projects">
+                        <thead>
+                            <tr class="with-children">
+                                <th>Job</th>
+                                <th>Execution</th>
+                                <th>Project</th>
+                                <th>Creation Date</th>
+                                <th>Status</th>
+                            </tr>
+                            <!--<tr class="with-no-children">-->
+                                <!--<th>No queued jobs</th>-->
+                            <!--</tr>-->
+                        </thead>
+                        <tbody class="job-queue-list">
+                            <tr><td class="no-jobs-msg">No Running Jobs...</td></tr>
+                        </tbody>
+                    </table>
+                </div>
+            </div>
+        </div>
+    </div>
+</div>

src/visualizers/panels/WorkerHeader/WorkerTemplate.html.ejs

@@ -0,0 +1,5 @@
+<tr>
+    <td class="clientId"></td>
+    <td class="lastSeen">unknown</td>
+    <td class="status">unknown</td>
+</tr>

src/visualizers/widgets/ArchEditor/ArchEditorWidget.js

@@ -28,6 +28,7 @@ define([
     ArchEditorWidget = function (logger, container) {
         EasyDAGWidget.call(this, logger, container);
         this.$el.addClass(WIDGET_CLASS);
+        this._emptyMsg = 'Click to add a new layer';
     };
 
     _.extend(ArchEditorWidget.prototype, EasyDAGWidget.prototype);

src/visualizers/widgets/PipelineEditor/Connection.js

@@ -17,16 +17,5 @@ define([
 
     _.extend(Connection.prototype, EasyDAGConn.prototype);
 
-    Connection.prototype.setStartPoint = function(point) {
-        // Update 'this.points' to start at the given point
-        this.points[0] = point;
-    };
-
-    Connection.prototype.setEndPoint = function(point) {
-        // Update 'this.points' to end at the given point
-        var last = this.points.length-1;
-        this.points[last] = point;
-    };
-
     return Connection;
 });

src/visualizers/widgets/PipelineEditor/PipelineEditorWidget.js

@@ -1,4 +1,4 @@
-/*globals WebGMEGlobal, $, define*/
+/*globals WebGMEGlobal, $, define, $klay*/
 /*jshint browser: true*/
 
 define([
@@ -12,6 +12,7 @@ define([
     './Connection',
     './SelectionManager',
     'underscore',
+    './klay',
     'css!./styles/PipelineEditorWidget.css'
 ], function (
     CONSTANTS,
@@ -186,9 +187,6 @@ define([
             this.showPorts(match.nodeId, match.portIds, isOutput)
         );
 
-        // Show the 'add' button
-        // TODO
-
         this.PORT_STATE = STATE.CONNECTING;
     };
 
@@ -209,27 +207,6 @@ define([
     PipelineEditorWidget.prototype.refreshExtras =
         PipelineEditorWidget.prototype.updateEmptyMsg;
 
-    PipelineEditorWidget.prototype.refreshConnections = function() {
-        // Update the connections to they first update their start/end points
-        var connIds = Object.keys(this.connections),
-            src,
-            dst,
-            conn;
-
-        for (var i = connIds.length; i--;) {
-            conn = this.connections[connIds[i]];
-
-            // Update the start/end point
-            src = this.items[conn.src];
-            conn.setStartPoint(src.getPortLocation(conn.srcPort));
-
-            dst = this.items[conn.dst];
-            conn.setEndPoint(dst.getPortLocation(conn.dstPort, true));
-            
-            conn.redraw();
-        }
-    };
-
     //////////////////// Action Overrides ////////////////////
 
     PipelineEditorWidget.prototype.onAddItemSelected = function(item, selected) {
@@ -443,5 +420,114 @@ define([
         this.updateThumbnail(svg.outerHTML);
     }, 1000);
 
+    // Changing the layout to klayjs
+    PipelineEditorWidget.prototype.refreshScreen = function() {
+        if (!this.active) {
+            return;
+        }
+
+        // WRITE UPDATES
+        // Update the locations of all the nodes
+
+        var graph = {
+            id: 'root',
+            properties: {
+                direction: 'DOWN',
+                'de.cau.cs.kieler.spacing': 25,
+                'de.cau.cs.kieler.edgeRouting': 'ORTHOGONAL'
+                //'de.cau.cs.kieler.klay.layered.nodePlace': 'INTERACTIVE'
+            },
+            edges: [],
+            children: []
+        };
+
+        graph.children = Object.keys(this.items).map(itemId => {
+            var item = this.items[itemId],
+                ports;
+
+            ports = item.inputs.map(p => this._getPortInfo(item, p, true))
+                .concat(item.outputs.map(p => this._getPortInfo(item, p)));
+            return {
+                id: itemId,
+                height: item.height,
+                width: item.width,
+                ports: ports,
+                properties: {
+                    'de.cau.cs.kieler.portConstraints': 'FIXED_POS'
+                }
+            };
+        });
+
+        graph.edges = Object.keys(this.connections).map(connId => {
+            var conn = this.connections[connId];
+            return {
+                id: connId,
+                source: conn.src,
+                target: conn.dst,
+                sourcePort: conn.srcPort,
+                targetPort: conn.dstPort
+            };
+        });
+
+        $klay.layout({
+            graph: graph,
+            success: graph => {
+                this.resultGraph = graph;
+                this.queueFns([
+                    this.applyLayout.bind(this, graph),
+                    this.updateTranslation.bind(this),
+                    this.refreshItems.bind(this),
+                    this.refreshConnections.bind(this),
+                    this.selectionManager.redraw.bind(this.selectionManager),
+                    this.updateContainerWidth.bind(this),
+                    this.refreshExtras.bind(this)
+                ]);
+            }
+        });
+    };
+
+    PipelineEditorWidget.prototype._getPortInfo = function(item, port, isInput) {
+        var position = item.decorator.getPortLocation(port.id, isInput),
+            side = isInput ? 'NORTH' : 'SOUTH';
+
+        position.y += (item.height/2) - 1;
+        return {
+            id: port.id,
+            width: 1,  // Ports are rendered outside the node in this library;
+            height: 1,  // we want it to look like it goes right up to the node
+            properties: {
+                'de.cau.cs.kieler.portSide': side
+            },
+            x: position.x,
+            y: position.y
+        };
+    };
+
+    PipelineEditorWidget.prototype.applyLayout = function (graph) {
+        var id,
+            item,
+            lItem,  // layout item
+            i;
+
+        for (i = graph.children.length; i--;) {
+            // update the x, y
+            lItem = graph.children[i];
+            id = lItem.id;
+            item = this.items[id];
+            item.x = lItem.x + item.width/2;
+            item.y = lItem.y + item.height/2;
+        }
+
+        for (i = graph.edges.length; i--;) {
+            // update the connection.points
+            lItem = graph.edges[i];
+            id = lItem.id;
+            item = this.connections[id];
+            item.points = lItem.bendPoints || [];
+            item.points.unshift(lItem.sourcePoint);
+            item.points.push(lItem.targetPoint);
+        }
+    };
+
     return PipelineEditorWidget;
 });

test/common/api/JobLogsClient.spec.js

@@ -1,4 +1,4 @@
-var testFixture = require('../globals'),
+var testFixture = require('../../globals'),
     expect = testFixture.expect,
     assert = require('assert'),
     path = testFixture.path,
@@ -7,7 +7,7 @@ var testFixture = require('../globals'),
     server = testFixture.WebGME.standaloneServer(gmeConfig),
     Logger = require('webgme/src/server/logger'),
     logger = Logger.createWithGmeConfig('gme', gmeConfig, true),
-    JobLogsClient = testFixture.requirejs('deepforge/JobLogsClient'),
+    JobLogsClient = testFixture.requirejs('deepforge/api/JobLogsClient'),
     rm_rf = require('rimraf'),
     exists = require('exists-file');
 

test/common/api/JobOriginClient.spec.js

@@ -0,0 +1,81 @@
+var testFixture = require('../../globals'),
+    expect = testFixture.expect,
+    gmeConfig = testFixture.getGmeConfig(),
+    server = testFixture.WebGME.standaloneServer(gmeConfig),
+    Logger = require('webgme/src/server/logger'),
+    logger = Logger.createWithGmeConfig('gme', gmeConfig, true),
+    JobOriginClient = testFixture.requirejs('deepforge/api/JobOriginClient');
+
+describe('JobOriginClient', function() {
+    var client = new JobOriginClient({
+            logger: logger,
+            origin: server.getUrl(),
+            projectId: 'testProject',
+            branchName: 'master'
+        }),
+        hashes = {},
+        getJobInfo = function() {
+            var hash = 'hashOrigin'+Math.ceil(Math.random()*100000);
+
+            while (hashes[hash]) {
+                hash = 'hashOrigin'+Math.ceil(Math.random()*100000);
+            }
+            hashes[hash] = true;
+
+            return {
+                hash: hash,
+                job: 'SomeJob',
+                execution: 'train_execution',
+                nodeId: 'K/6/1'
+            };
+        };
+
+    before(function(done) {
+        server.start(done);
+    });
+
+    after(function(done) {
+        server.stop(done);
+    });
+
+    it('should store job info', function(done) {
+        var job = getJobInfo();
+
+        client.record(job.hash, job)
+            .nodeify(done);
+    });
+
+    it('should read job info', function(done) {
+        var job = getJobInfo();
+
+        client.record(job.hash, job)
+            .then(() => client.getOrigin(job.hash))
+            .then(jobInfo => {
+                Object.keys(job).forEach(key => {
+                    expect(jobInfo[key]).equal(job[key]);
+                });
+            })
+            .nodeify(done);
+    });
+
+    it('should delete job info', function(done) {
+        var job = getJobInfo();
+
+        client.record(job.hash, job)
+            .then(() => client.deleteRecord(job.hash))
+            .then(() => client.getOrigin(job.hash))
+            .then(res => expect(res).equal(null))
+            .nodeify(done);
+    });
+
+    it('should update job branch on fork', function(done) {
+        var job = getJobInfo(),
+            newBranch = 'newBranch';
+
+        client.record(job.hash, job)
+            .then(() => client.fork(job.hash, newBranch))
+            .then(() => client.getOrigin(job.hash))
+            .then(res => expect(res.branch).equal(newBranch))
+            .nodeify(done);
+    });
+});

test/plugins/ExecuteJob/ExecuteJob.spec.js

@@ -10,12 +10,14 @@ describe('ExecuteJob', function () {
         PluginCliManager = testFixture.WebGME.PluginCliManager,
         projectName = 'testProject',
         pluginName = 'ExecuteJob',
+        manager = new PluginCliManager(null, logger, gmeConfig),
         project,
         gmeAuth,
         storage,
         commitHash;
 
     before(function (done) {
+        this.timeout(10000);
         testFixture.clearDBAndGetGMEAuth(gmeConfig, projectName)
             .then(function (gmeAuth_) {
                 gmeAuth = gmeAuth_;
@@ -25,7 +27,7 @@ describe('ExecuteJob', function () {
             })
             .then(function () {
                 var importParam = {
-                    projectSeed: testFixture.path.join(testFixture.SEED_DIR, 'EmptyProject.webgmex'),
+                    projectSeed: testFixture.path.join(testFixture.DF_SEED_DIR, 'devProject', 'devProject.webgmex'),
                     projectName: projectName,
                     branchName: 'master',
                     logger: logger,
@@ -51,9 +53,7 @@ describe('ExecuteJob', function () {
     });
 
     it('should verify activeNode is "Job"', function (done) {
-        var manager = new PluginCliManager(null, logger, gmeConfig),
-            pluginConfig = {
-            },
+        var pluginConfig = {},
             context = {
                 project: project,
                 commitHash: commitHash,
@@ -68,4 +68,285 @@ describe('ExecuteJob', function () {
             done();
         });
     });
+
+    ////////// Helper Functions //////////
+    var plugin,
+        node,
+        preparePlugin = function(done) {
+            var context = {
+                project: project,
+                commitHash: commitHash,
+                branchName: 'test',
+                activeNode: '/K/R/p'  // hello world job
+            };
+
+            return manager.initializePlugin(pluginName)
+                .then(plugin_ => {
+                    plugin = plugin_;
+                    return manager.configurePlugin(plugin, {}, context);
+                })
+                .then(() => node = plugin.activeNode)
+                .nodeify(done);
+        };
+
+    ////////// END Helper Functions //////////
+
+    // Race condition checks w/ saving...
+    describe('get/set', function() {
+        beforeEach(preparePlugin);
+
+        it('should get correct attribute after set', function() {
+            plugin.setAttribute(node, 'status', 'queued');
+            var attrValue = plugin.getAttribute(node, 'status');
+            expect(attrValue).to.equal('queued');
+        });
+
+        it('should get correct attribute before updating nodes', function(done) {
+            // Run setAttribute on some node
+            plugin.setAttribute(node, 'status', 'queued');
+
+            // Check that the value is correct before applying node changes
+            var updateNodes = plugin.updateNodes;
+            plugin.updateNodes = function() {
+                var attrValue = plugin.getAttribute(node, 'status');
+                expect(attrValue).to.equal('queued');
+                return updateNodes.apply(this, arguments);
+            };
+            plugin.save().nodeify(done);
+        });
+
+        it('should get correct attribute (from new node) before updating nodes', function(done) {
+            // Run setAttribute on some node
+            var graphTmp = plugin.createNode('pipeline.Graph', node),
+                newVal = 'testGraph',
+                id = 'testId';
+
+            // Get the 
+            plugin.setAttribute(graphTmp, 'name', newVal);
+            plugin._metadata[id] = graphTmp;
+            plugin.createIdToMetadataId[graphTmp] = id;
+
+            // Check that the value is correct before applying node changes
+            var updateNodes = plugin.updateNodes;
+            plugin.updateNodes = function() {
+                var graph = plugin._metadata[id],
+                    attrValue = plugin.getAttribute(graph, 'name');
+
+                expect(attrValue).to.equal(newVal);
+                return updateNodes.apply(this, arguments);
+            };
+            plugin.save().nodeify(done);
+        });
+
+        it('should get correct attribute after save', function(done) {
+            // Run setAttribute on some node
+            plugin.setAttribute(node, 'status', 'queued');
+
+            // Check that the value is correct before applying node changes
+            plugin.save()
+                .then(() => {
+                    var attrValue = plugin.getAttribute(node, 'status');
+                    expect(attrValue).to.equal('queued');
+                })
+                .nodeify(done);
+        });
+
+        it('should get correct attribute while applying node changes', function(done) {
+            // Run setAttribute on some node
+            plugin.setAttribute(node, 'status', 'queued');
+
+            // Check that the value is correct before applying node changes
+            var oldApplyChanges = plugin._applyNodeChanges;
+            plugin._applyNodeChanges = function() {
+                var attrValue = plugin.getAttribute(node, 'status');
+                expect(attrValue).to.equal('queued');
+                return oldApplyChanges.apply(this, arguments);
+            };
+            plugin.save().nodeify(done);
+        });
+    });
+
+    describe('createNode', function() {
+        beforeEach(preparePlugin);
+
+        it('should update _metadata after applying changes', function(done) {
+            // Run setAttribute on some node
+            var graphTmp = plugin.createNode('pipeline.Graph', node),
+                id = 'testId';
+
+            plugin._metadata[id] = graphTmp;
+            plugin.createIdToMetadataId[graphTmp] = id;
+
+            // Check that the value is correct before applying node changes
+            var applyModelChanges = plugin.applyModelChanges;
+            plugin.applyModelChanges = function() {
+                return applyModelChanges.apply(this, arguments)
+                    .then(() => {
+                        var graph = plugin._metadata[id];
+                        expect(graph).to.not.equal(graphTmp);
+                    });
+            };
+            plugin.save().nodeify(done);
+        });
+
+        it('should update _metadata in updateNodes', function(done) {
+            var id = 'testId';
+
+            plugin._metadata[id] = node;
+            node.old = true;
+            plugin.updateNodes()
+                    .then(() => {
+                        var graph = plugin._metadata[id];
+                        expect(graph.old).to.not.equal(true);
+                    })
+                    .nodeify(done);
+        });
+
+        // Check that it gets the correct value from a newly created node after
+        // it has been saved/created
+        it('should get changed attribute', function(done) {
+            // Run setAttribute on some node
+            var graphTmp = plugin.createNode('pipeline.Graph', node),
+                id = 'testId';
+
+            plugin._metadata[id] = node;
+            plugin.createIdToMetadataId[graphTmp] = id;
+
+            plugin.setAttribute(graphTmp, 'name', 'firstName');
+
+            // Check that the value is correct before applying node changes
+            plugin.save()
+                .then(() => {
+                    var graph = plugin._metadata[id],
+                        val = plugin.getAttribute(graph, 'name');
+                    expect(val).to.equal('firstName');
+                })
+                .nodeify(done);
+        });
+
+        it('should get inherited attribute', function(done) {
+            // Run setAttribute on some node
+            var graphTmp = plugin.createNode('pipeline.Graph', node),
+                id = 'testId',
+                val;
+
+            // Check that the value is correct before applying node changes
+            plugin._metadata[id] = node;
+            plugin.createIdToMetadataId[graphTmp] = id;
+
+            val = plugin.getAttribute(graphTmp, 'name');
+            expect(val).to.equal('Graph');
+
+            plugin.save()
+                .then(() => {
+                    var graph = plugin._metadata[id];
+
+                    val = plugin.getAttribute(graph, 'name');
+
+                    expect(val).to.equal('Graph');
+                })
+                .nodeify(done);
+        });
+    });
+
+    // Canceling
+    describe('cancel', function() {
+        beforeEach(preparePlugin);
+
+        it('should stop the job if the execution is canceled', function(done) {
+            var job = node,
+                hash = 'abc123',
+                exec = {
+                    cancelJob: jobHash => expect(jobHash).equal(hash)
+                };
+
+            plugin.setAttribute(node, 'secret', 'abc');
+            plugin.isExecutionCanceled = () => true;
+            plugin.onOperationCanceled = () => done();
+            plugin.watchOperation(exec, hash, job, job);
+        });
+
+        it('should stop the job if a job is canceled', function(done) {
+            var job = node,
+                hash = 'abc123',
+                exec = {
+                    cancelJob: jobHash => expect(jobHash).equal(hash)
+                };
+
+            plugin.setAttribute(job, 'secret', 'abc');
+            plugin.canceled = true;
+            plugin.onOperationCanceled = () => done();
+            plugin.watchOperation(exec, hash, job, job);
+        });
+
+        it('should set exec to running', function(done) {
+            var job = node,
+                execNode = plugin.core.getParent(job);
+
+            // Set the execution to canceled
+            plugin.setAttribute(execNode, 'status', 'canceled');
+            plugin.prepare = () => {
+                var status = plugin.getAttribute(execNode, 'status');
+                expect(status).to.not.equal('canceled');
+                return {then: () => done()};
+            };
+            plugin.main();
+        });
+    });
+
+    describe('exec files', function() {
+        describe('attribute file', function() {
+            var boolString = /['"](true|false)['"]/g;
+
+            beforeEach(preparePlugin);
+
+            it('should not quote true (s) boolean values', function() {
+                var files = {},
+                    content,
+                    matches;
+
+                plugin.setAttribute(node, 'debug', 'true');
+                plugin.createAttributeFile(node, files);
+                content = files['attributes.lua'];
+                matches = content.match(boolString);
+                expect(matches).to.equal(null);
+            });
+
+            it('should not quote true boolean values', function() {
+                var files = {},
+                    content,
+                    matches;
+
+                plugin.setAttribute(node, 'debug', true);
+                plugin.createAttributeFile(node, files);
+                content = files['attributes.lua'];
+                matches = content.match(boolString);
+                expect(matches).to.equal(null);
+            });
+
+            it('should not quote false (s) boolean values', function() {
+                var files = {},
+                    content,
+                    matches;
+
+                plugin.setAttribute(node, 'debug', 'false');
+                plugin.createAttributeFile(node, files);
+                content = files['attributes.lua'];
+                matches = content.match(boolString);
+                expect(matches).to.equal(null);
+            });
+
+            it('should not quote false boolean values', function() {
+                var files = {},
+                    content,
+                    matches;
+
+                plugin.setAttribute(node, 'debug', false);
+                plugin.createAttributeFile(node, files);
+                content = files['attributes.lua'];
+                matches = content.match(boolString);
+                expect(matches).to.equal(null);
+            });
+        });
+    });
 });

test/plugins/ImportTorch/ImportTorch.spec.js

@@ -14,7 +14,7 @@ var testFixture = require('../../globals'),
     ];
 
 describe('ImportTorch', function () {
-    this.timeout(5000);
+    this.timeout(7500);
     var gmeConfig = testFixture.getGmeConfig(),
         Q = testFixture.Q,
         GraphChecker = testFixture.requirejs('deepforge/GraphChecker'),

test/routers/JobLogsAPI/JobLogsAPI.spec.js

@@ -1,4 +1,4 @@
-var testFixture = require('../globals'),
+var testFixture = require('../../globals'),
     superagent = testFixture.superagent,
     Q = testFixture.Q,
     expect = testFixture.expect,

test/routers/JobOriginAPI/JobOriginAPI.spec.js

@@ -0,0 +1,115 @@
+var testFixture = require('../../globals'),
+    superagent = testFixture.superagent,
+    expect = testFixture.expect,
+    gmeConfig = testFixture.getGmeConfig(),
+    server = testFixture.WebGME.standaloneServer(gmeConfig),
+    mntPt = 'job/origins';
+
+describe('JobOriginAPI', function() {
+    var hashes = {},
+        getUrl = function(hash) {
+            return [
+                server.getUrl(),
+                mntPt,
+                hash
+            ].join('/');
+        },
+        getJobInfo = function() {
+            var hash = 'hash'+Math.ceil(Math.random()*100000);
+
+            while (hashes[hash]) {
+                hash = 'hash'+Math.ceil(Math.random()*100000);
+            }
+            hashes[hash] = true;
+
+            return {
+                hash: hash,
+                job: 'SomeJob',
+                branch: 'master',
+                execution: 'train_execution',
+                project: 'guest+example',
+                nodeId: 'K/6/1'
+            };
+        };
+
+    before(function(done) {
+        server.start(done);
+    });
+
+    after(function(done) {
+        server.stop(done);
+    });
+
+    it('should store job info', function(done) {
+        var job = getJobInfo();
+
+        superagent.post(getUrl(job.hash))
+            .send(job)
+            .end(function (err, res) {
+                expect(res.status).equal(201, err);
+                done();
+            });
+    });
+
+    it('should read job info', function(done) {
+        var job = getJobInfo(),
+            url = getUrl(job.hash);
+
+        superagent.post(url)
+            .send(job)
+            .end(function (err, res) {
+                expect(res.status).equal(201, err);
+                superagent.get(url)
+                    .end((err, res) => {
+                        var jobInfo = JSON.parse(res.text);
+                        Object.keys(jobInfo).forEach(key => {
+                            expect(jobInfo[key]).equal(job[key]);
+                        });
+                        done();
+                    });
+            });
+    });
+
+    it('should delete job info', function(done) {
+        var job = getJobInfo(),
+            url = getUrl(job.hash);
+
+        superagent.post(url)
+            .send(job)
+            .end(function (err, res) {
+                expect(res.status).equal(201, err);
+                superagent.delete(url).end(err => {
+                    expect(err).equal(null);
+                    superagent.get(url)
+                        .end((err, res) => {
+                            expect(res.status).equal(404, err);
+                            done();
+                        });
+                });
+            });
+    });
+
+    it('should update job branch', function(done) {
+        var job = getJobInfo(),
+            url = getUrl(job.hash);
+
+        superagent.post(url)  // create the job
+            .send(job)
+            .end(function (err, res) {
+                expect(res.status).equal(201, err);
+                superagent.patch(url)  // update the branch
+                    .send({branch: 'newBranch'})
+                    .end(err => {
+                        expect(err).equal(null);
+
+                        superagent.get(url)  // check the new version
+                            .end((err, res) => {
+                                var info = JSON.parse(res.text);
+                                expect(info.branch).equal('newBranch');
+                                expect(res.status).equal(200, err);
+                                done();
+                            });
+                    });
+            });
+    });
+});

webgme-setup.json

@@ -230,6 +230,13 @@
         "panel": "src/visualizers/panels/ExecutionIndex",
         "secondary": false,
         "widget": "src/visualizers/widgets/ExecutionIndex"
+      },
+      "WorkerHeader": {
+        "src": "panels/WorkerHeader/WorkerHeaderPanel",
+        "title": "WorkerHeader",
+        "panel": "src/visualizers/panels/WorkerHeader",
+        "secondary": false,
+        "widget": "src/visualizers/widgets/WorkerHeader"
       }
     },
     "addons": {},
@@ -280,12 +287,19 @@
       },
       "xor": {
         "src": "src/seeds/xor"
+      },
+      "devProject": {
+        "src": "src/seeds/devProject"
       }
     },
     "routers": {
       "JobLogsAPI": {
         "src": "src/routers/JobLogsAPI",
         "mount": "execution/logs"
+      },
+      "JobOriginAPI": {
+        "src": "src/routers/JobOriginAPI",
+        "mount": "job/origins"
       }
     }
   },