v0.22.0

* WIP #936 Added webgme-executor-worker

* WIP #936 Updated start-worker.js

* WIP #936 Fixed bug in class generation

.codeclimate.yml

@@ -31,6 +31,8 @@ exclude_paths:
 - test/
 - src/common/lua.js
 - src/common/js-yaml.min.js
+- src/visualizers/widgets/Sidebar/lib/
 - src/visualizers/widgets/TextEditor/lib/
 - src/visualizers/widgets/PipelineIndex/styles/PipelineIndex.css
 - src/visualizers/widgets/LineGraph/lib/
+- src/visualizers/widgets/PipelineEditor/klay.js

.npmignore

@@ -0,0 +1,40 @@
+src/worker/tmp
+.env
+*.swp
+*.swo
+**.sass-cache
+# Logs
+logs
+*.log
+
+# Runtime data
+pids
+*.pid
+*.seed
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (http://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directory
+# https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git
+node_modules
+tmp/
+test-tmp/
+blob-local-storage/
+src/seeds/nn/hash.txt
+src/seeds/pipeline/hash.txt
+
+# docs
+images/

README.md

@@ -1,19 +1,25 @@
 [![Release State](https://img.shields.io/badge/state-beta-yellow.svg)](https://img.shields.io/badge/state-beta-yellow.svg)
 [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](./LICENSE)
-[![Build Status](https://travis-ci.org/dfst/deepforge.svg?branch=master)](https://travis-ci.org/dfst/deepforge)
-[![Join the chat at https://gitter.im/dfst/deepforge](https://badges.gitter.im/dfst/deepforge.svg)](https://gitter.im/dfst/deepforge?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
-[![Stories in Ready](https://badge.waffle.io/dfst/deepforge.png?label=ready&title=Ready)](https://waffle.io/dfst/deepforge)
+[![Build Status](https://travis-ci.org/deepforge-dev/deepforge.svg?branch=master)](https://travis-ci.org/deepforge-dev/deepforge)
+[![Join the chat at https://gitter.im/deepforge-dev/deepforge](https://badges.gitter.im/deepforge-dev/deepforge.svg)](https://gitter.im/deepforge-dev/deepforge?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+[![Stories in Ready](https://badge.waffle.io/deepforge-dev/deepforge.png?label=ready&title=Ready)](https://waffle.io/deepforge-dev/deepforge)
+
+**Notice**: DeepForge is still a work in progress and in beta! That being said, any contributions and/or feedback is greatly appreciated. If you have any questions, check out the [wiki](https://github.com/dfst/deepforge/wiki/) or drop me a line on the gitter!
 
-**Notice**: DeepForge is still a work in progress and is also lacking significant documentation! That being said, any contributions and/or feedback is greatly appreciated (and feel free to always ask any questions on the gitter)!
 
 # DeepForge
-DeepForge is an open-source visual development environment for deep learning. Currently, it supports Convolutional Neural Networks, RNNs and LSTMs as well as the creation of custom layers. Additional features include:
+DeepForge is an open-source visual development environment for deep learning providing end-to-end support for creating deep learning models. This is achieved through providing the ability to design **architectures**, create training **pipelines**, and then execute these pipelines over a cluster. Using a notebook-esque api, users can get real-time feedback about the status of any of their **executions** including compare them side-by-side in real-time.
+
+![overview](images/overview.png "")
+
+Additional features include:
 - Graphical architecture editor
 - Training/testing pipeline creation
 - Distributed pipeline execution
 - Real-time pipeline feedback
 - Collaborative editing
 - Automatic version control.
+- Facilitates defining custom layers 
 
 ## Quick Start
 Simply run the following command to install deepforge with its dependencies:
@@ -28,12 +34,20 @@ Or, if you already have NodeJS (v6) installed, simply run
 npm install -g deepforge
 ```
 
-Next, start deepforge with `deepforge start`!
+Finally, start deepforge with `deepforge start`and navigate to [http://localhost:8888](http://localhost:8888) to start using DeepForge! For more, detailed instructions, check out the [wiki](https://github.com/dfst/deepforge/wiki/Installation-Guide).
 
-Finally, navigate to [http://localhost:8888](http://localhost:8888) to start using DeepForge! For more, detailed instructions, check out the [wiki](https://github.com/dfst/deepforge/wiki/Installation-Guide).
+**Note**: running deepforge w/ `deepforge start` will also require [MongoDB](https://www.mongodb.com/download-center?jmp=nav#community) to be installed locally.
 
 Also, be sure to check out the other available features of the `deepforge` cli; it can be used to update, manage your torch installation, uninstall deepforge and run individual components!
 
+## Additional Resources
+- [Intro to DeepForge Slides](https://docs.google.com/presentation/d/10_y5O3gHXSATfjHVLJg7dOdrz-tAXNWjlxhJ5SlA0ic/edit?usp=sharing)
+- [wiki](https://github.com/deepforge-dev/deepforge/wiki) containing overview, installation, configuration and developer information
+- [Starter Kit](https://github.com/deepforge-dev/examples/tree/master/starterkit) containing example pipelines demonstrating various deepforge features
+- [Examples](https://github.com/deepforge-dev/examples)
+
+- [Datamodel Developer Slides](https://docs.google.com/presentation/d/1hd3IyUlzW_TIPnzCnE-1pdz00Pw8WaIxYiOW_Hyog-M/edit#slide=id.p)
+
 ## Interested in contributing?
 Contributions are welcome! Either fork the project and submit some PR's or shoot me an email about getting more involved!
 

app.js

@@ -4,11 +4,17 @@
 var gmeConfig = require('./config'),
     webgme = require('webgme'),
     path = require('path'),
+    fs = require('fs'),
     rm_rf = require('rimraf'),
+    gracefulFs = require('graceful-fs'),
     myServer;
 
+process.chdir(__dirname);
 webgme.addToRequireJsPaths(gmeConfig);
 
+// Patch the 'fs' module to fix 'too many files open' error
+gracefulFs.gracefulify(fs);
+
 // Clear seed hash info
 ['nn', 'pipeline'].map(lib => path.join(__dirname, 'src', 'seeds', lib, 'hash.txt'))
     .forEach(file => rm_rf.sync(file));

bin/start-worker.js

@@ -6,8 +6,7 @@ var path = require('path'),
     spawn = childProcess.spawn,
     rm_rf = require('rimraf'),
     projectConfig = require(__dirname + '/../config'),
-    executorSrc = path.join(__dirname, '..', 'node_modules', 'webgme', 'src',
-        'server', 'middleware', 'executor', 'worker'),
+    executorSrc = path.join(__dirname, '..', 'node_modules', '.bin', 'webgme-executor-worker'),
     id = Date.now(),
     workerRootPath = process.env.DEEPFORGE_WORKER_DIR || path.join(__dirname, '..', 'src', 'worker'),
     workerPath = path.join(workerRootPath, `worker_${id}`),
@@ -60,7 +59,7 @@ var startExecutor = function() {
 
     // Start the executor
     var execJob = spawn('node', [
-        'node_worker.js',
+        executorSrc,
         workerConfigPath,
         workerTmp
     ]);
@@ -80,14 +79,4 @@ var createConfigJson = function() {
     fs.writeFile(workerConfigPath, JSON.stringify(config), startExecutor);
 };
 
-process.chdir(executorSrc);
-
-fs.mkdir(workerTmp, function() {
-    // npm install in this directory
-    var npmInstall = spawn('npm', ['install']);
-    npmInstall.stdout.pipe(process.stdout);
-    npmInstall.stderr.pipe(process.stderr);
-    npmInstall.on('close', function() {
-        createConfigJson();
-    });
-});
+fs.mkdir(workerTmp, createConfigJson);

config/components.json

@@ -2,13 +2,29 @@
     "AutoViz": {
         "preloadIds": [
             "ArchEditor",
+            "ArchIndex",
+            "PipelineIndex",
             "PipelineEditor",
             "OperationEditor",
             "ExecutionView"
-        ]
+        ],
+        "visualizerOverrides": {
+            "": "ForwardViz",
+            "MyArtifacts": "ArtifactIndex",
+            "MyArchitectures": "ArchIndex",
+            "MyExecutions": "ExecutionIndex",
+            "MyPipelines": "PipelineIndex"
+        }
+    },
+    "PipelineEditor": {
+        "itemName": "operation"
+    },
+    "ExecutionView": {
+        "itemName": "job"
     },
     "ArchEditor": {
         "hotkeys": "none",
+        "itemName": "layer",
         "LayerColors": {}
     },
     "BreadcrumbHeader": {
@@ -43,11 +59,11 @@
         "rootMenuClass": "deepforge-logo",
         "rootDisplayName": "DeepForge"
     },
-    "CHFLayout": {
+    "SidebarLayout": {
         "panels": [
             {
-                "id": "Header",
-                "panel": "BreadcrumbHeader/BreadcrumbHeaderPanel",
+                "id": "WorkerHeader",
+                "panel": "WorkerHeader/WorkerHeaderPanel",
                 "container": "header",
                 "DEBUG_ONLY": false
             },
@@ -63,6 +79,12 @@
                 "container": "center",
                 "DEBUG_ONLY": false
             },
+            {
+                "id": "Sidebar",
+                "panel": "Sidebar/SidebarPanel",
+                "container": "sidebar",
+                "DEBUG_ONLY": false
+            },
             {
                 "id": "ForgeActionButton",
                 "panel": "ForgeActionButton/ForgeActionButton",

config/config.webgme.js

@@ -9,6 +9,7 @@ var config = require('webgme/config/config.default'),
 // The paths can be loaded from the webgme-setup.json
 config.plugin.basePaths.push(__dirname + '/../src/plugins');
 config.plugin.basePaths.push(__dirname + '/../node_modules/webgme-simple-nodes/src/plugins');
+config.visualization.layout.basePaths.push(__dirname + '/../src/layouts');
 config.visualization.layout.basePaths.push(__dirname + '/../node_modules/webgme-chflayout/src/layouts');
 config.visualization.decoratorPaths.push(__dirname + '/../src/decorators');
 config.visualization.decoratorPaths.push(__dirname + '/../node_modules/webgme-easydag/src/decorators');
@@ -32,6 +33,8 @@ config.visualization.panelPaths.push(__dirname + '/../src/visualizers/panels');
 
 
 config.rest.components['execution/logs'] = __dirname + '/../src/routers/JobLogsAPI/JobLogsAPI.js';
+config.rest.components['job/origins'] = __dirname + '/../src/routers/JobOriginAPI/JobOriginAPI.js';
+config.rest.components['execution/pulse'] = __dirname + '/../src/routers/ExecPulse/ExecPulse.js';
 
 // Visualizer descriptors
 config.visualization.visualizerDescriptors.push(__dirname + '/../src/visualizers/Visualizers.json');
@@ -56,7 +59,7 @@ config.requirejsPaths = {
   'widgets/FloatingActionButton': './node_modules/webgme-fab/src/visualizers/widgets/FloatingActionButton'
 };
 
-config.visualization.layout.default = 'CHFLayout';
+config.visualization.layout.default = 'SidebarLayout';
 config.mongo.uri = 'mongodb://127.0.0.1:27017/deepforge';
 validateConfig(config);
 module.exports = config;

install.sh

@@ -42,10 +42,10 @@ detect_profile() {
 detect_profile
 
 set_node_version() {
-    # Install nodejs v6.2.0
-    echo "Installing NodeJS v6.2.0"
-    nvm install v6.2.0
-    nvm alias default v6.2.0
+    # Install nodejs v6.2.1
+    echo "Installing NodeJS v6.2.1"
+    nvm install v6.2.1
+    nvm alias default v6.2.1
 
     # Install npm@2
     npm install npm@2 -g

package.json

@@ -8,26 +8,29 @@
     "start-dev": "NODE_ENV=dev node app.js",
     "local": "node ./bin/start-local.js",
     "worker": "node ./bin/start-worker.js",
-    "test": "mkdir ./test-tmp; node ./node_modules/mocha/bin/mocha --recursive test",
-    "watch-test": "./node_modules/nodemon/bin/nodemon.js --exec 'node ./node_modules/mocha/bin/mocha --recursive test'",
+    "test": "mkdir ./test-tmp; mocha --recursive test",
+    "watch-test": "nodemon --exec 'mocha --recursive test'",
     "build-nn": "node ./utils/nn-parser.js"
   },
-  "version": "0.16.0",
+  "version": "0.22.0",
   "dependencies": {
     "commander": "^2.9.0",
     "dotenv": "^2.0.0",
     "exists-file": "^2.1.0",
     "express": "^4.14.0",
+    "graceful-fs": "^4.1.10",
     "lodash.difference": "^4.1.2",
     "lodash.merge": "^4.5.1",
+    "mongodb": "^2.2.10",
     "nodemon": "^1.9.2",
     "q": "1.4.1",
     "rimraf": "^2.4.0",
-    "webgme": "^2.0.0",
-    "webgme-autoviz": "dfst/webgme-autoviz",
+    "webgme": "^2.7.1",
+    "webgme-autoviz": "^2.2.0",
     "webgme-breadcrumbheader": "^2.1.1",
     "webgme-chflayout": "^2.0.0",
     "webgme-easydag": "dfst/webgme-easydag",
+    "webgme-executor-worker": "^1.0.1",
     "webgme-fab": "dfst/webgme-fab",
     "webgme-simple-nodes": "^2.1.0"
   },

src/common/Constants.js

@@ -1,31 +1,53 @@
 /* globals define */
-define({
-    LINE_OFFSET: 'lineOffset',
+(function(root, factory){
+    if(typeof define === 'function' && define.amd) {
+        define([], function(){
+            return factory();
+        });
+    } else if(typeof module === 'object' && module.exports) {
+        module.exports = factory();
+    } else {
+        root.CONSTANTS = factory();
+    }
+}(this, function() {
+    return {
+        CONTAINED_LAYER_SET: 'addLayers',
+        CONTAINED_LAYER_INDEX: 'index',
 
-    // DeepForge metadata creation in dist execution
-    START_CMD: 'deepforge-cmd',
+        LINE_OFFSET: 'lineOffset',
 
-    IMAGE: {  // all prefixed w/ 'IMG' for simple upload detection
-        PREFIX: 'IMG',
-        BASIC: 'IMG-B',
-        CREATE: 'IMG-C',
-        UPDATE: 'IMG-U',
-        NAME: 'IMAGE-N'  // No upload required
-    },
+        // DeepForge metadata creation in dist execution
+        START_CMD: 'deepforge-cmd',
 
-    GRAPH_CREATE: 'GRAPH',
-    GRAPH_PLOT: 'PLOT',
-    GRAPH_CREATE_LINE: 'LINE',
+        IMAGE: {  // all prefixed w/ 'IMG' for simple upload detection
+            PREFIX: 'IMG',
+            BASIC: 'IMG-B',
+            CREATE: 'IMG-C',
+            UPDATE: 'IMG-U',
+            NAME: 'IMAGE-N'  // No upload required
+        },
 
-    // Code Generation Constants
-    CTOR_ARGS_ATTR: 'ctor_arg_order',
+        GRAPH_CREATE: 'GRAPH',
+        GRAPH_PLOT: 'PLOT',
+        GRAPH_CREATE_LINE: 'LINE',
 
-    // Operation types
-    OP: {
-        INPUT: 'Input',
-        OUTPUT: 'Output'
-    },
+        // Code Generation Constants
+        CTOR_ARGS_ATTR: 'ctor_arg_order',
 
-    // Job stdout update
-    STDOUT_UPDATE: 'stdout_update'
-});
+        // Operation types
+        OP: {
+            INPUT: 'Input',
+            OUTPUT: 'Output'
+        },
+
+        // Heartbeat constants (ExecPulse router)
+        PULSE: {
+            DEAD: 0,
+            ALIVE: 1,
+            DOESNT_EXIST: 2
+        },
+
+        // Job stdout update
+        STDOUT_UPDATE: 'stdout_update'
+    };
+}));

src/common/LayerParser.js

@@ -265,6 +265,7 @@
     var findTorchClass = function(ast){ 
         var torchClassArgs,  // args for `torch.class(...)`
             name = '',
+            alias,
             baseType,
             params,
             setters = {},
@@ -283,6 +284,7 @@
                     name = torchClassArgs[0];
                     if(name !== ''){
                         name = name.replace('nn.', '');
+                        alias = func.names[0] || name;
                         if (torchClassArgs.length > 1) {
                             baseType = torchClassArgs[1].replace('nn.', '');
                         }
@@ -302,7 +304,7 @@
                 attrName;
 
             // Record the setter functions
-            if (isSetterMethod(curr, parent, name)) {
+            if (isSetterMethod(curr, parent, alias)) {
                 firstLine = curr.block.stats[0];
                 // just use the attribute attrName for now...
                 attrName = getSettingAttrName(firstLine);
@@ -316,7 +318,7 @@
                 } else {
                     setters[attrName] = schema;
                 }
-            } else if (isInitFn(curr, name)) {  // Record the defaults
+            } else if (isInitFn(curr, alias)) {  // Record the defaults
                 paramDefs = getAttrsAndVals(curr);
                 attrDefs = getClassAttrDefs(curr);
                 types = inferParamTypes(curr, paramDefs);

src/common/api/APIClient.js

@@ -0,0 +1,75 @@
+/*globals define*/
+define([
+    'q',
+    'superagent'
+], function(
+    Q,
+    superagent
+) {
+    'use strict';
+
+    // Wrap the ability to read, update, and delete logs using the JobLogsAPI
+    var APIClient = function(params) {
+        params = params || {};
+
+        this.logger = this.logger || params.logger.fork('APIClient');
+
+        // Get the server url
+        this.token = params.token;
+        this.origin = this._getServerUrl(params);
+        this.relativeUrl = this.relativeUrl || '';
+        this.url = this.origin + this.relativeUrl;
+
+        this.logger.debug(`Setting url to ${this.url}`);
+
+        this.branch = params.branchName;
+        this.project = params.projectId;
+        this._modifiedJobs = [];
+
+        this.logger.debug(`Using <project>:<branch>: "${this.project}"/"${this.branch}"`);
+        this.logger.info('ctor finished');
+    };
+
+    APIClient.prototype._getServerUrl = function(params) {
+        if (typeof window !== 'undefined') {
+            return window.location.origin;
+        }
+
+        // If not in browser, set using the params
+        var server = params.server || '127.0.0.1',
+            port = params.port || '80',
+            protocol = params.httpsecure ? 'https' : 'http';  // default is http
+
+        return params.origin || `${protocol}://${server}:${port}`;
+    };
+
+    APIClient.prototype.getUrl = function() {
+        return this.url;
+    };
+
+    APIClient.prototype._request = function(method, jobId, content) {
+        var deferred = Q.defer(),
+            req = superagent[method](this.getUrl(jobId));
+
+        this.logger.debug(`sending ${method} request to ${this.getUrl(jobId)}`);
+        if (this.token) {
+            req.set('Authorization', 'Bearer ' + this.token);
+        }
+
+        if (content) {
+            req = req.send(content);
+        }
+
+        req.end((err, res) => {
+            if (err || res.status > 399) {
+                return deferred.reject(res || err);
+            }
+
+            return deferred.resolve(res);
+        });
+
+        return deferred.promise;
+    };
+
+    return APIClient;
+});

src/common/api/ExecPulseClient.js

@@ -0,0 +1,44 @@
+/* globals define */
+define([
+    './APIClient'
+], function(
+    APIClient
+) {
+    'use strict';
+
+    var ExecPulseClient = function(params) {
+        this.relativeUrl = '/execution/pulse/';
+        this.logger = params.logger.fork('ExecPulseClient');
+        APIClient.call(this, params);
+    };
+
+    ExecPulseClient.prototype = Object.create(APIClient.prototype);
+
+    ExecPulseClient.prototype.getUrl = function(hash) {
+        return this.url + hash;
+    };
+
+    // - update the heartbeat
+    // - check the heartbeat
+    // - delete the heartbeat
+    ExecPulseClient.prototype.update = function(hash) {
+        return this._request('post', hash)
+            .catch(err => {
+                throw err.text || err;
+            });
+    };
+
+    ExecPulseClient.prototype.check = function(hash) {
+        return this._request('get', hash)
+            .then(res => JSON.parse(res.text))
+            .catch(err => {
+                throw err.text || err;
+            });
+    };
+
+    ExecPulseClient.prototype.clear = function(hash) {
+        return this._request('delete', hash);
+    };
+
+    return ExecPulseClient;
+});

src/common/api/JobLogsClient.js

@@ -1,51 +1,38 @@
 /* globals define */
 define([
+    './APIClient',
     'q',
     'superagent'
 ], function(
+    APIClient,
     Q,
     superagent
 ) {
     'use strict';
 
     // Wrap the ability to read, update, and delete logs using the JobLogsAPI
+    var METADATA_FIELDS = [
+        'lineCount'
+    ];
     var JobLogsClient = function(params) {
         params = params || {};
 
-        this.logger = params.logger.fork('JobLogsClient');
-
-        // Get the server url
-        this.token = params.token;
-        this.origin = this._getServerUrl(params);
         this.relativeUrl = '/execution/logs';
-        this.url = this.origin + this.relativeUrl;
-
-        this.logger.debug(`Setting url to ${this.url}`);
+        this.logger = params.logger.fork('JobLogsClient');
+        APIClient.call(this, params);
 
         // Get the project, branch name
         if (!(params.branchName && params.projectId)) {
             throw Error('"branchName" and "projectId" required');
         }
-        this.branch = params.branchName;
-        this.project = params.projectId;
+
         this._modifiedJobs = [];
 
         this.logger.debug(`Using <project>:<branch>: "${this.project}"/"${this.branch}"`);
         this.logger.info('ctor finished');
     };
 
-    JobLogsClient.prototype._getServerUrl = function(params) {
-        if (typeof window !== 'undefined') {
-            return window.location.origin;
-        }
-
-        // If not in browser, set using the params
-        var server = params.server || '127.0.0.1',
-            port = params.port || '80',
-            protocol = params.httpsecure ? 'https' : 'http';  // default is http
-
-        return params.origin || `${protocol}://${server}:${port}`;
-    };
+    JobLogsClient.prototype = Object.create(APIClient.prototype);
 
     // This method could be optimized - it could make a log of requests
     JobLogsClient.prototype.fork = function(forkName) {
@@ -79,53 +66,54 @@ define([
     };
 
     JobLogsClient.prototype.getUrl = function(jobId) {
+        var url = this.url;
+            
+        if (typeof jobId !== 'string') {
+            url = this.url + jobId.route;
+            jobId = jobId.jobId;
+        }
+
         return [
-            this.url,
+            url,
             encodeURIComponent(this.project),
             encodeURIComponent(this.branch),
             encodeURIComponent(jobId)
         ].join('/');
     };
 
-    JobLogsClient.prototype._logRequest = function(method, jobId, content) {
-        var deferred = Q.defer(),
-            req = superagent[method](this.getUrl(jobId));
-
-        this.logger.info(`sending ${method} request to ${this.getUrl(jobId)}`);
-        if (this.token) {
-            req.set('Authorization', 'Bearer ' + this.token);
-        }
-
-        if (content) {
-            req = req.send(content);
-        }
-
-        req.end((err, res) => {
-            if (err || res.status > 399) {
-                return deferred.reject(err || res.status);
-            }
-
-            return deferred.resolve(res);
-        });
-
-        return deferred.promise;
+    var hasRequiredFields = function(md) {
+        return METADATA_FIELDS.reduce((passing, nextField) => {
+            return passing && md.hasOwnProperty(nextField);
+        }, true);
     };
 
-    JobLogsClient.prototype.appendTo = function(jobId, text) {
+    JobLogsClient.prototype.appendTo = function(jobId, text, metadata) {
         this._modifiedJobs.push(jobId);
         this.logger.info(`Appending logs to ${jobId}`);
-        return this._logRequest('patch', jobId, {patch: text});
+
+        if (metadata && !hasRequiredFields(metadata)) {
+            throw Error(`Required metadata fields: ${METADATA_FIELDS.join(', ')}`);
+        }
+        metadata = metadata || {};
+        metadata.patch = text;
+        return this._request('patch', jobId, metadata);
     };
 
     JobLogsClient.prototype.getLog = function(jobId) {
         this.logger.info(`Getting logs for ${jobId}`);
-        return this._logRequest('get', jobId)
+        return this._request('get', jobId)
             .then(res => res.text);
     };
 
     JobLogsClient.prototype.deleteLog = function(jobId) {
         this.logger.info(`Deleting logs for ${jobId}`);
-        return this._logRequest('delete', jobId);
+        return this._request('delete', jobId);
+    };
+
+    JobLogsClient.prototype.getMetadata = function(jobId) {
+        this.logger.info(`Getting line count for ${jobId}`);
+        return this._request('get', {jobId: jobId, route: '/metadata'})
+            .then(res => JSON.parse(res.text));
     };
 
     return JobLogsClient;

src/common/api/JobOriginClient.js

@@ -0,0 +1,60 @@
+/* globals define */
+define([
+    './APIClient'
+], function(
+    APIClient
+) {
+    'use strict';
+
+    var JobOriginClient = function(params) {
+        this.relativeUrl = '/job/origins/';
+        this.logger = params.logger.fork('JobOriginClient');
+        APIClient.call(this, params);
+    };
+
+    JobOriginClient.prototype = Object.create(APIClient.prototype);
+
+    // - Record the origin
+    // - Look up the origin
+    // - Delete record
+    JobOriginClient.prototype.getUrl = function(hash) {
+        return this.url + hash;
+    };
+
+    JobOriginClient.prototype.record = function(hash, info) {
+        var jobInfo = {
+            hash: hash,
+            nodeId: info.nodeId,
+            job: info.job,
+            project: info.project || this.project,
+            branch: info.branch || this.branch,
+            execution: info.execution
+        };
+
+        return this._request('post', hash, jobInfo)
+            .catch(err => {
+                throw err.text || err;
+            });
+    };
+
+    JobOriginClient.prototype.getOrigin = function(hash) {
+        return this._request('get', hash)
+            .then(res => JSON.parse(res.text))
+            .catch(res => {
+                if (res.status && res.status === 404) {
+                    return null;
+                }
+                throw res;
+            });
+    };
+
+    JobOriginClient.prototype.fork = function(hash, forkName) {
+        return this._request('patch', hash, {branch: forkName});
+    };
+
+    JobOriginClient.prototype.deleteRecord = function(hash) {
+        return this._request('delete', hash);
+    };
+
+    return JobOriginClient;
+});

src/common/globals.js

@@ -57,7 +57,7 @@ define([
     };
 
     var createNamedNode = function(baseId, parentId, isMeta) {
-        var newId = client.createChild({parentId, baseId}),
+        var newId = client.createNode({parentId, baseId}),
             baseNode = client.getNode(baseId),
             basename = 'New' + baseNode.getAttribute('name'),
             newName = getUniqueName(parentId, basename);
@@ -72,7 +72,7 @@ define([
             client.setRegistry(newId, 'isAbstract', false);
         }
 
-        client.setAttributes(newId, 'name', newName);
+        client.setAttribute(newId, 'name', newName);
         return newId;
     };
 
@@ -83,7 +83,9 @@ define([
             exists = {},
             i = 2;
 
-        children.forEach(child => exists[child.getAttribute('name')] = true);
+        children
+            .filter(child => child !== null)
+            .forEach(child => exists[child.getAttribute('name')] = true);
 
         while (exists[name]) {
             name = basename + '_' + i;
@@ -191,7 +193,7 @@ define([
                 .getId();
 
         // Look up the parent container
-        DeepForge.places[placeName]().then(parentId => {
+        return DeepForge.places[placeName]().then(parentId => {
 
             client.startTransaction(msg);
             newId = createNamedNode(baseId, parentId, !!metasheetName);
@@ -265,7 +267,7 @@ define([
         }
 
         dataBaseId = dataBase.getId();
-        dataTypes = metanodes.filter(n => client.isTypeOf(n.getId(), dataBaseId))
+        dataTypes = metanodes.filter(n => n.isTypeOf(dataBaseId))
             .filter(n => !n.getRegistry('isAbstract'))
             .map(node => node.getAttribute('name'));
 
@@ -297,7 +299,8 @@ define([
     };
 
     DeepForge.last = {};
-    DeepForge.create  = {};
+    DeepForge.create = {};
+    DeepForge.register = {};
     instances.forEach(type => {
         DeepForge.create[type] = function() {
             return createNew.call(null, type);
@@ -308,6 +311,10 @@ define([
         DeepForge.create[type] = function() {
             return createNew.call(null, type, type);
         };
+        DeepForge.register[type] = function(id) {
+            // Add the given element to the metasheet!
+            return addToMetaSheet(id, type);
+        };
     });
 
     DeepForge.create.Layer = createCustomLayer;

src/common/lua.js

@@ -2565,7 +2565,7 @@ function LuaContext(){
             case 'number':
                 return c;
             case 'string':
-                return parseInt(s) || dummy0;
+                return parseInt(c) || dummy0;
             default:
                 if (c == dummy0){
                     return c;

src/common/plugin/LocalExecutor.js

@@ -148,11 +148,14 @@ define([
                 // get the input node
                 if (dataNodes.length !== 0) {
                     var newNodes = this.core.copyNodes(dataNodes, parentNode),
-                        newName = this.core.getOwnAttribute(node, 'saveName');
+                        newName = this.core.getOwnAttribute(node, 'saveName'),
+                        createdAt = Date.now();
+
                     if (newName) {
-                        newNodes.forEach(node =>
-                            this.setAttribute(node, 'name', newName)
-                        );
+                        newNodes.forEach(node => {
+                            this.setAttribute(node, 'name', newName);
+                            this.setAttribute(node, 'createdAt', createdAt);
+                        });
                     }
                     var hashes = dataNodes.map(n => this.getAttribute(n, 'data'));
                     this.logger.info(`saving hashes: ${hashes.map(h => `"${h}"`)}`);

src/common/viz/Buttons.js

@@ -1,8 +1,10 @@
 /*globals define, WebGMEGlobal*/
 define([
+    'deepforge/globals',
     'widgets/EasyDAG/Buttons',
     'widgets/EasyDAG/Icons'
 ], function(
+    DeepForge,
     EasyDAGButtons,
     Icons
 ) {
@@ -57,9 +59,79 @@ define([
         return n && n.getBaseId();
     };
 
+    var CloneAndEdit = function(params) {
+        GoToBase.call(this, params);
+    };
+
+    CloneAndEdit.prototype = Object.create(GoToBase.prototype);
+    CloneAndEdit.prototype.BTN_CLASS = 'clone-and-edit';
+
+    CloneAndEdit.prototype._render = function() {
+        var lineRadius = GoToBase.SIZE - GoToBase.BORDER,
+            btnColor = '#a5d6a7';
+
+        if (this.disabled) {
+            btnColor = '#e0e0e0';
+        }
+
+        this.$el
+            .append('circle')
+            .attr('r', GoToBase.SIZE)
+            .attr('fill', btnColor);
+
+        // Show the 'code' icon
+        Icons.addIcon('code', this.$el, {
+            radius: lineRadius
+        });
+    };
+
+    CloneAndEdit.prototype._onClick = function(item) {
+        var node = client.getNode(item.id),
+            baseId = node && node.getBaseId(),
+            base = baseId && client.getNode(baseId),
+            typeId = base && base.getBaseId(),
+            type = typeId && client.getNode(typeId),
+            ctrName,
+            typeName,
+            name,
+            newId;
+
+        // Clone the given node's base and change to it
+        if (type) {
+            typeName = type.getAttribute('name');
+            ctrName = `My${typeName}s`;
+            if (DeepForge.places[ctrName]) {
+                DeepForge.places[ctrName]().then(ctrId => {
+                    type = base.getAttribute('name');
+                    client.startTransaction(`Creating new ${typeName} from ${item.name}`);
+                    newId = client.copyNode(baseId, ctrId);
+                    name = node.getAttribute('name');
+                    client.setAttribute(newId, 'name', 'Copy of ' + name);
+                    DeepForge.register[typeName](newId);
+
+                    client.completeTransaction();
+                    WebGMEGlobal.State.registerActiveObject(newId);
+                });
+            }
+        } else {
+            this._logger.warn('Could not find the base node!');
+        }
+    };
+
+    var Insert = function(params) {
+        EasyDAGButtons.ButtonBase.call(this, params);
+    };
+
+    Insert.prototype = Object.create(EasyDAGButtons.Add.prototype);
+    Insert.prototype._onClick = function(item) {
+        this.onInsertButtonClicked(item);
+    };
+
     return {
         DeleteOne: EasyDAGButtons.DeleteOne,
-        GoToBase: GoToBase
+        GoToBase: GoToBase,
+        CloneAndEdit: CloneAndEdit,
+        Insert: Insert
     };
 });
 

src/common/viz/Execute.js

@@ -3,21 +3,31 @@
 define([
     'q',
     'executor/ExecutorClient',
+    'deepforge/api/ExecPulseClient',
+    'deepforge/api/JobOriginClient',
+    'deepforge/Constants',
     'panel/FloatingActionButton/styles/Materialize'
 ], function(
     Q,
     ExecutorClient,
+    ExecPulseClient,
+    JobOriginClient,
+    CONSTANTS,
     Materialize
 ) {
 
     var Execute = function(client, logger) {
         this.client = this.client || client;
         this.logger = this.logger || logger;
+        this.pulseClient = new ExecPulseClient({
+            logger: this.logger
+        });
         this._executor = new ExecutorClient({
             logger: this.logger.fork('ExecutorClient'),
             serverPort: WebGMEGlobal.gmeConfig.server.port,
             httpsecure: window.location.protocol === 'https:'
         });
+        this.originManager = new JobOriginClient({logger: this.logger});
     };
 
     Execute.prototype.executeJob = function(node) {
@@ -102,6 +112,21 @@ define([
             .fail(err => this.logger.error(`Job cancel failed: ${err}`));
     };
 
+    Execute.prototype._setJobStopped = function(jobId, silent) {
+        if (!silent) {
+            var name = this.client.getNode(jobId).getAttribute('name');
+            this.client.startTransaction(`Stopping "${name}" job`);
+        }
+
+        this.client.delAttribute(jobId, 'jobId');
+        this.client.delAttribute(jobId, 'secret');
+        this.client.setAttribute(jobId, 'status', 'canceled');
+
+        if (!silent) {
+            this.client.completeTransaction();
+        }
+    };
+
     Execute.prototype.stopJob = function(job, silent) {
         var jobId;
 
@@ -109,18 +134,7 @@ define([
         jobId = job.getId();
 
         this.silentStopJob(job);
-
-        if (!silent) {
-            this.client.startTransaction(`Stopping "${name}" job`);
-        }
-
-        this.client.delAttributes(jobId, 'jobId');
-        this.client.delAttributes(jobId, 'secret');
-        this.client.setAttributes(jobId, 'status', 'canceled');
-
-        if (!silent) {
-            this.client.completeTransaction();
-        }
+        this._setJobStopped(jobId, silent);
     };
 
 
@@ -165,14 +179,17 @@ define([
     };
 
     Execute.prototype._stopExecution = function(execNode, inTransaction) {
-        var msg = `Canceling ${execNode.getAttribute('name')} execution`;
+        var msg = `Canceling ${execNode.getAttribute('name')} execution`,
+            jobIds;
 
         if (!inTransaction) {
             this.client.startTransaction(msg);
         }
 
-        this._silentStopExecution(execNode);
-        this.client.setAttributes(execNode.getId(), 'status', 'canceled');
+        jobIds = this._silentStopExecution(execNode);
+
+        this.client.setAttribute(execNode.getId(), 'status', 'canceled');
+        jobIds.forEach(jobId => this._setJobStopped(jobId, true));
 
         if (!inTransaction) {
             this.client.completeTransaction();
@@ -180,11 +197,88 @@ define([
     };
 
     Execute.prototype._silentStopExecution = function(execNode) {
-        var jobIds = execNode.getChildrenIds();
+        var runningJobIds = execNode.getChildrenIds()
+            .map(id => this.client.getNode(id))
+            .filter(job => this.isRunning(job));  // get running jobs
 
-        jobIds.map(id => this.client.getNode(id))
-            .filter(job => this.isRunning(job))  // get running jobs
-            .forEach(job => this.silentStopJob(job));  // stop them
+        runningJobIds.forEach(job => this.silentStopJob(job));  // stop them
+
+        return runningJobIds;
+    };
+
+    // Resuming Executions
+    Execute.prototype.checkJobExecution= function (job) {
+        var pipelineId = job.getParentId(),
+            pipeline = this.client.getNode(pipelineId);
+
+        // First check the parent execution. If it doesn't exist, then check the job
+        return this.checkPipelineExecution(pipeline)
+            .then(tryToStartJob => {
+                if (tryToStartJob) {
+                    return this._checkJobExecution(job);
+                }
+            });
+    };
+
+    Execute.prototype._checkJobExecution = function (job) {
+        var jobId = job.getAttribute('jobId'),
+            status = job.getAttribute('status');
+
+        if (status === 'running' && jobId) {
+            return this.pulseClient.check(jobId)
+                .then(status => {
+                    if (status !== CONSTANTS.PULSE.DOESNT_EXIST) {
+                        return this._onOriginBranch(jobId).then(onBranch => {
+                            if (onBranch) {
+                                this.runExecutionPlugin('ExecuteJob', {
+                                    node: job
+                                });
+                            }
+                        });
+                    } else {
+                        this.logger.warn(`Could not restart job: ${job.getId()}`);
+                    }
+                });
+        }
+        return Q();
+    };
+
+    Execute.prototype._onOriginBranch = function (hash) {
+        return this.originManager.getOrigin(hash)
+            .then(origin => {
+                var currentBranch = this.client.getActiveBranchName();
+                if (origin && origin.branch) {
+                    return origin.branch === currentBranch;
+                }
+                return false;
+            });
+    };
+
+    Execute.prototype.checkPipelineExecution = function (pipeline) {
+        var runId = pipeline.getAttribute('runId'),
+            status = pipeline.getAttribute('status'),
+            tryToStartJob = true;
+
+        if (status === 'running' && runId) {
+            return this.pulseClient.check(runId)
+                .then(status => {
+                    if (status === CONSTANTS.PULSE.DEAD) {
+                        // Check the origin branch
+                        return this._onOriginBranch(runId).then(onBranch => {
+                            if (onBranch) {
+                                this.runExecutionPlugin('ExecutePipeline', {
+                                    node: pipeline
+                                });
+                            }
+                        });
+                    }
+                    // only try to start if the pulse info doesn't exist
+                    tryToStartJob = status === CONSTANTS.PULSE.DOESNT_EXIST;
+                    return tryToStartJob;
+                });
+        } else {
+            return Q().then(() => tryToStartJob);
+        }
     };
 
     return Execute;

src/common/viz/PipelineControl.js

@@ -65,7 +65,7 @@ define([
 
     PipelineControl.prototype.createNode = function(baseId) {
         var parentId = this._currentNodeId,
-            newNodeId = this._client.createChild({parentId, baseId});
+            newNodeId = this._client.createNode({parentId, baseId});
 
         return newNodeId;
     };

src/common/viz/RenameablePanel.js

@@ -49,7 +49,7 @@ define([
 
                 if (!/^\s*$/.test(newValue)) {
                     this._client.startTransaction(msg);
-                    this._client.setAttributes(nodeId, 'name', newValue);
+                    this._client.setAttribute(nodeId, 'name', newValue);
                     this._client.completeTransaction();
                 }
             }

src/common/viz/panels/ThumbnailControl.js

@@ -0,0 +1,41 @@
+/* globals define */
+define([
+    'panels/EasyDAG/EasyDAGControl'
+], function(
+    EasyDAGControl
+) {
+    var ThumbnailControl = function() {
+        EasyDAGControl.apply(this, arguments);
+    };
+
+    ThumbnailControl.prototype = Object.create(EasyDAGControl.prototype);
+
+    ThumbnailControl.prototype._initWidgetEventHandlers = function () {
+        EasyDAGControl.prototype._initWidgetEventHandlers.call(this);
+        this._widget.updateThumbnail = this.updateThumbnail.bind(this);
+    };
+
+    ThumbnailControl.prototype.updateThumbnail = function (svg) {
+        var node = this._client.getNode(this._currentNodeId),
+            name,
+            attrs,
+            currentThumbnail,
+            attrName = 'thumbnail',
+            msg;
+
+        if (node) {  // may have been deleted
+            name = node.getAttribute('name');
+            attrs = node.getValidAttributeNames();
+            currentThumbnail = node.getAttribute(attrName);
+            msg = `Updating pipeline thumbnail for "${name}"`;
+
+            if (attrs.indexOf(attrName) > -1 && currentThumbnail !== svg) {
+                this._client.startTransaction(msg);
+                this._client.setAttribute(this._currentNodeId, attrName, svg);
+                this._client.completeTransaction();
+            }
+        }
+    };
+
+    return ThumbnailControl;
+});

src/common/viz/widgets/Thumbnail.js

@@ -0,0 +1,84 @@
+/* globals define, $, _ */
+define([
+    'widgets/EasyDAG/EasyDAGWidget'
+], function(
+    EasyDAGWidget
+) {
+
+    var ThumbnailWidget = function() {
+        EasyDAGWidget.apply(this, arguments);
+    };
+
+    ThumbnailWidget.prototype = Object.create(EasyDAGWidget.prototype);
+
+    ThumbnailWidget.prototype.addNode = function() {
+        var result = EasyDAGWidget.prototype.addNode.apply(this, arguments);
+
+        this.refreshThumbnail();
+        return result;
+    };
+
+    ThumbnailWidget.prototype.removeNode = function() {
+        var result = EasyDAGWidget.prototype.removeNode.apply(this, arguments);
+
+        this.refreshThumbnail();
+        return result;
+    };
+
+    ThumbnailWidget.prototype._removeConnection = function() {
+        var result = EasyDAGWidget.prototype._removeConnection.apply(this, arguments);
+
+        this.refreshThumbnail();
+        return result;
+    };
+
+    ThumbnailWidget.prototype.addConnection = function() {
+        var result = EasyDAGWidget.prototype.addConnection.apply(this, arguments);
+
+        this.refreshThumbnail();
+        return result;
+    };
+
+    ////////////////////////// Thumbnail updates //////////////////////////
+    ThumbnailWidget.prototype.getSvgDistanceDim = function(dim) {
+        var maxValue = this._getMaxAlongAxis(dim),
+            nodes,
+            minValue;
+
+        nodes = this.graph.nodes().map(id => this.graph.node(id));
+        minValue = nodes.length ? Math.min.apply(null, nodes.map(node => node[dim] || 0)) : 0;
+        return maxValue-minValue;
+    };
+
+    ThumbnailWidget.prototype.getSvgWidth = function() {
+        return this.getSvgDistanceDim('x');
+    };
+
+    ThumbnailWidget.prototype.getSvgHeight = function() {
+        return this.height - 25;
+    };
+
+    ThumbnailWidget.prototype.getViewBox = function() {
+        var maxX = this.getSvgWidth('x'),
+            maxY = this.getSvgHeight('y');
+
+        return `0 0 ${maxX} ${maxY}`;
+    };
+
+    ThumbnailWidget.prototype.refreshThumbnail = _.debounce(function() {
+        // Get the svg...
+        var svg = document.createElement('svg'),
+            group = this.$svg.node(),
+            child;
+
+        svg.setAttribute('viewBox', this.getViewBox());
+        for (var i = 0; i < group.children.length; i++) {
+            child = $(group.children[i]);
+            svg.appendChild(child.clone()[0]);
+        }
+
+        this.updateThumbnail(svg.outerHTML);
+    }, 1000);
+
+    return ThumbnailWidget;
+});

src/decorators/ArtifactOpDecorator/EasyDAG/ArtifactOpDecorator.EasyDAGWidget.js

@@ -57,13 +57,13 @@ define([
             this.client.startTransaction(`Removing output of ${this.name}`);
             this.client.delPointer(this._node.id, name);
             if (outputId) {
-                this.client.delAttributes(outputId, 'data');
+                this.client.delAttribute(outputId, 'data');
             }
             this.client.completeTransaction();
         } else if (name === this.castOpts.ptr) {  // set the casted value
             this.client.startTransaction(`Setting output of ${this.name} to ${to}`);
             this.castOutputType(to);
-            this.client.makePointer(this._node.id, name, to);
+            this.client.setPointer(this._node.id, name, to);
             this.client.completeTransaction();
         } else {
             DecoratorBase.prototype.savePointer.call(this, name, to);

src/decorators/ContainerLayerDecorator/ContainerLayerDecorator.js

@@ -0,0 +1,39 @@
+/*globals define, _*/
+/*jshint browser: true, camelcase: false*/
+
+define([
+    'js/Decorators/DecoratorBase',
+    './EasyDAG/ContainerLayerDecorator.EasyDAGWidget'
+], function (
+    DecoratorBase,
+    ContainerLayerDecoratorEasyDAGWidget
+) {
+
+    'use strict';
+
+    var ContainerLayerDecorator,
+        __parent__ = DecoratorBase,
+        __parent_proto__ = DecoratorBase.prototype,
+        DECORATOR_ID = 'ContainerLayerDecorator';
+
+    ContainerLayerDecorator = function (params) {
+        var opts = _.extend({loggerName: this.DECORATORID}, params);
+
+        __parent__.apply(this, [opts]);
+
+        this.logger.debug('ContainerLayerDecorator ctor');
+    };
+
+    _.extend(ContainerLayerDecorator.prototype, __parent_proto__);
+    ContainerLayerDecorator.prototype.DECORATORID = DECORATOR_ID;
+
+    /*********************** OVERRIDE DecoratorBase MEMBERS **************************/
+
+    ContainerLayerDecorator.prototype.initializeSupportedWidgetMap = function () {
+        this.supportedWidgetMap = {
+            EasyDAG: ContainerLayerDecoratorEasyDAGWidget
+        };
+    };
+
+    return ContainerLayerDecorator;
+});

src/decorators/ContainerLayerDecorator/EasyDAG/ContainerLayerDecorator.EasyDAGWidget.css

@@ -0,0 +1,25 @@
+.condense .nested-layers {
+  display: none;
+}
+
+.nested-layers .hover-box {
+  stroke: black;
+  stroke-dasharray: 4, 4;
+  stroke-opacity: 0;
+}
+
+.nested-layers .unhovered .button {
+  opacity: 0;
+}
+
+.nested-layers .hovered .button {
+  opacity: 1;
+}
+
+.nested-layers .unhovered .hover-box {
+  stroke-opacity: 0;
+}
+
+.nested-layers .hovered .hover-box {
+  stroke-opacity: 0;
+}

src/decorators/ContainerLayerDecorator/EasyDAG/ContainerLayerDecorator.EasyDAGWidget.js

@@ -0,0 +1,428 @@
+/*globals define, _, */
+/*jshint browser: true, camelcase: false*/
+
+define([
+    'decorators/LayerDecorator/EasyDAG/LayerDecorator.EasyDAGWidget',
+    'js/Constants',
+    'deepforge/Constants',
+    './NestedLayer',
+    'widgets/EasyDAG/Buttons',
+    'css!./ContainerLayerDecorator.EasyDAGWidget.css'
+], function (
+    LayerDecorator,
+    GME_CONSTANTS,
+    CONSTANTS,
+    NestedLayer,
+    Buttons
+) {
+
+    'use strict';
+
+    var ContainerLayerDecorator,
+        ZOOM = 0.8,
+        DECORATOR_ID = 'ContainerLayerDecorator';
+
+    // Container layer nodes need to be able to nest the containedLayers
+    // in order inside of themselves when expanded
+    ContainerLayerDecorator = function (options) {
+        this.nestedLayers = {};
+        LayerDecorator.call(this, options);
+        this.$nested = this.$el.append('g')
+            .attr('class', 'nested-layers');
+
+        // If clicked, deselect the given nested layer
+        this.$el.on('click', () => {
+            if (this.expanded) {
+                Object.keys(this.nestedLayers).forEach(id => {
+                    this.nestedLayers[id].widget.onBackgroundClick();
+                });
+            }
+        });
+        this.onNestedRefresh = _.debounce(this.updateExpand.bind(this), 50);
+
+        // Add event handlers
+        NestedLayer.prototype.addLayerBefore = function(layerId) {
+            var decorator = this._parent,
+                index = decorator._node.containedLayers.indexOf(this.id);
+            return decorator.addLayerAt(layerId, index - 1);
+        };
+
+        NestedLayer.prototype.addLayerAfter = function(layerId) {
+            var decorator = this._parent,
+                index = decorator._node.containedLayers.indexOf(this.id);
+            return decorator.addLayerAt(layerId, index + 1);
+        };
+
+        NestedLayer.prototype.isLast = function() {
+            var index = this._parent._node.containedLayers.length - 1;
+            return this._parent._node.containedLayers[index] === this.id;
+        };
+
+        NestedLayer.prototype.isFirst = function() {
+            return this._parent._node.containedLayers[0] === this.id;
+        };
+
+        NestedLayer.prototype.moveLayerForward = function() {
+            return this.moveLayer(true);
+        };
+
+        NestedLayer.prototype.moveLayerBackward = function() {
+            return this.moveLayer();
+        };
+
+        NestedLayer.prototype.moveLayer = function(forward) {
+            var decorator = this._parent,
+                index = decorator._node.containedLayers.indexOf(this.id),
+                client = decorator.client,
+                msg;
+
+            decorator._node.containedLayers.splice(index, 1);
+            if (forward) {
+                index = Math.max(0, index - 1);
+            } else {
+                index++;
+            }
+
+            decorator._node.containedLayers.splice(index, 0, this.id);
+
+            msg = `Swapping nested layers at ${index} and ${forward ? index-1 : index+1}`;
+            client.startTransaction(msg);
+            decorator._updateNestedIndices();
+            client.completeTransaction();
+        };
+
+        NestedLayer.prototype.onLastNodeRemoved = function() {
+            var decorator = this._parent,
+                index = decorator._node.containedLayers.indexOf(this.id),
+                msg = `Removing nested layer of ${decorator._node.name} at position ${index}`;
+
+            decorator.client.startTransaction(msg);
+            decorator.client.deleteNode(this.id);
+            decorator.client.completeTransaction();
+        };
+        this.updateNestedTerritory();
+    };
+
+    _.extend(ContainerLayerDecorator.prototype, LayerDecorator.prototype);
+
+    ContainerLayerDecorator.prototype.DECORATOR_ID = DECORATOR_ID;
+
+    ContainerLayerDecorator.prototype._updateNestedIndices = function() {
+        this._node.containedLayers.forEach((layerId, index) => {
+            // Set the layer's member registry to it's index
+            this.client.setMemberRegistry(
+                this._node.id,
+                layerId,
+                CONSTANTS.CONTAINED_LAYER_SET,
+                CONSTANTS.CONTAINED_LAYER_INDEX,
+                index
+            );
+        });
+    };
+
+    ContainerLayerDecorator.prototype.addLayerAt = function(baseId, index) {
+        var client = this.client,
+            parentId = this._node.id,
+            archNode,
+            newId,
+            msg;
+
+        // Get the index of the given layer
+        index = Math.max(index, 0);
+
+        archNode = client.getAllMetaNodes()
+            .find(node => node.getAttribute('name') === 'Architecture');
+
+        // Create a new Architecture node in the given node
+        msg = `Adding layer to ${this._node.name} at position ${index}`;
+        client.startTransaction(msg);
+
+        newId = client.createNode({
+            parentId: parentId,
+            baseId: archNode.getId()
+        });
+        // Create the selected layer
+        client.createNode({
+            parentId: newId,
+            baseId: baseId
+        });
+        client.addMember(parentId, newId, CONSTANTS.CONTAINED_LAYER_SET);
+        this._node.containedLayers.splice(index, 0, newId);
+        this._updateNestedIndices();
+
+        client.completeTransaction();
+    };
+
+    ContainerLayerDecorator.prototype.condense = function() {
+        // hide the nested layers
+        this.$el.attr('class', 'centering-offset condense');
+        this.removeCreateNestedBtn();
+        return LayerDecorator.prototype.condense.apply(this, arguments);
+    };
+
+    ContainerLayerDecorator.prototype.updateNestedTerritory = function() {
+        // Add the nested layers and update
+        if (!this._nestedTerritoryUI) {
+            this._nestedTerritoryUI = this.client.addUI(this, this._containedEvents.bind(this));
+        }
+        this._territory = {};
+        this._node.containedLayers.forEach(id => this._territory[id] = {children: 0});
+        this.client.updateTerritory(this._nestedTerritoryUI, this._territory);
+    };
+
+    ContainerLayerDecorator.prototype._containedEvents = function(events) {
+        for (var i = events.length; i--;) {
+            switch (events[i].etype) {
+            case GME_CONSTANTS.TERRITORY_EVENT_LOAD:
+                if (!this.nestedLayers[events[i].eid]) {
+                    this.createNestedWidget(events[i].eid);
+                }
+                break;
+
+            case GME_CONSTANTS.TERRITORY_EVENT_UNLOAD:
+                this.removeNestedWidget(events[i].eid);
+                break;
+            }
+        }
+        if (events.length > 1) {  // if more than just 'complete' event
+            this.updateExpand();
+        }
+    };
+
+    ContainerLayerDecorator.prototype.update = function(node) {
+        var attrsUpdated = false,
+            attrs = this._attributes;
+
+        this._node = node;
+        // Update the attributes
+        this.setAttributes();
+        attrsUpdated = !_.isEqual(attrs, this._attributes);
+
+        // Check for a new nested layer
+        var hasNewLayers = this._node.containedLayers
+            .filter(id => !this.nestedLayers[id])
+            .length > 0;
+
+        if (hasNewLayers) {
+            this.updateNestedTerritory();
+        } else {
+            // Update the order of the nested layers
+            if (this._selected) {
+                this.expand();
+            } else {
+                this.condense();
+            }
+        }
+        // Only reset fieldsWidth if the attribute has gotten larger
+        if (attrsUpdated) {
+            this.fieldsWidth = null;
+        }
+    };
+
+    ContainerLayerDecorator.prototype.updateExpand = function() {
+        if (this.expanded) {
+            this.expand();
+        }
+    };
+
+    ContainerLayerDecorator.prototype.createNestedWidget = function(id) {
+        if (!this.$nested) {
+            this.$nested = this.$el.append('g')
+                .attr('class', 'nested-layers');
+        }
+
+        this.nestedLayers[id] = new NestedLayer({
+            $container: this.$nested,
+            parent: this,
+            client: this.client,
+            logger: this.logger,
+            onRefresh: this.onNestedRefresh,
+            id: id
+        });
+        return this.nestedLayers[id];
+    };
+
+    ContainerLayerDecorator.prototype.removeNestedWidget = function(id) {
+        this.nestedLayers[id].destroy();
+        delete this.nestedLayers[id];
+        this.updateExpand();
+    };
+
+    ContainerLayerDecorator.prototype._renderInfo = function(top, width) {
+        var isAnUpdate = this.expanded,
+            y = top;
+
+        // Add the attribute fields
+        this.clearFields();
+        this.$attributes = this.$el.append('g')
+            .attr('fill', '#222222');
+
+        if (!isAnUpdate) {
+            this.$attributes.attr('opacity', 0);
+        }
+
+        y = this.createAttributeFields(y, width);
+        y = this.createPointerFields(y, width);
+
+        if (y !== top) {
+            y += this.ROW_HEIGHT/2;
+        }
+        return y;
+    };
+
+    ContainerLayerDecorator.prototype.expand = function() {
+        // This should be rendered with the attributes
+        var height,
+            width,
+
+            // Attributes
+            initialY = 25,
+            isAnUpdate = this.expanded,
+            NAME_MARGIN = 15,
+            nestedMargin = 15,  // minimum
+            margin = 5,
+            y = margin + initialY,
+            x = margin,
+            i;
+
+        // Shift name down
+        this.$name.attr('y', 20);
+
+        // Add the nested children
+        var ids = this._node.containedLayers.filter(id => this.nestedLayers[id]),
+            totalNestedWidth = 0,
+            maxNestedHeight = 0,
+            fieldWidth,
+            widget;
+
+        if (ids.length === 0) {
+            maxNestedHeight = CreateNestedBtn.SIZE * 2;
+        } else {
+            for (i = 0; i < ids.length; i++) {
+                widget = this.nestedLayers[ids[i]].widget;
+                totalNestedWidth += widget.getSvgWidth() * ZOOM;
+                maxNestedHeight = Math.max(widget.getSvgHeight() * ZOOM, maxNestedHeight);
+
+                // Update the buttons (in case of reorder)
+                this.nestedLayers[ids[i]].refreshButtons();
+            }
+        }
+
+        fieldWidth = this.fieldsWidth + 3 * NAME_MARGIN;
+        width = Math.max(
+            this.nameWidth + 2 * NAME_MARGIN,
+            this.size.width,
+            fieldWidth,
+            totalNestedWidth + (ids.length + 1) * nestedMargin
+        );
+
+        // Render attributes
+        y = this._renderInfo(y, fieldWidth);
+        y += nestedMargin;
+
+        // Update width, height
+        height = y + maxNestedHeight + nestedMargin;
+
+        // Equally space the nested widgets
+        nestedMargin = (width - totalNestedWidth)/(ids.length + 1);
+        x = nestedMargin - width/2;
+        for (i = 0; i < ids.length; i++) {
+            this.nestedLayers[ids[i]].$el
+                .attr('transform', `translate(${x}, ${y}) scale(${ZOOM})`);
+            x += this.nestedLayers[ids[i]].widget.getSvgWidth() * ZOOM + nestedMargin;
+        }
+
+        this.removeCreateNestedBtn();
+
+        if (ids.length === 0) {
+            // Add the 'create nested layer' button if no nested layers
+            this.$createNestedBtn = new CreateNestedBtn({
+                context: this,
+                $pEl: this.$el,
+                y: y + CreateNestedBtn.SIZE
+            });
+        }
+
+        this.$body
+            .transition()
+            .attr('x', -width/2)
+            .attr('y', 0)
+            .attr('rx', 0)
+            .attr('ry', 0)
+            .attr('width', width)
+            .attr('height', height)
+            .each('end', () => {
+                if (!isAnUpdate) {
+                    this.$attributes.attr('opacity', 1);
+                    this.$el.attr('class', 'centering-offset expand');
+                }
+            });
+
+        if (this.height !== height || this.width !== width) {
+            this.height = height;
+            this.width = width;
+            this.expanded = true;
+            this.$el
+                .attr('transform', `translate(${this.width/2}, 0)`);
+
+            this.onResize();
+        }
+    };
+
+    ContainerLayerDecorator.prototype.removeCreateNestedBtn = function() {
+        if (this.$createNestedBtn) {
+            this.$createNestedBtn.remove();
+            this.$createNestedBtn = null;
+        }
+    };
+
+    ContainerLayerDecorator.prototype.destroyNested = function() {
+        Object.keys(this.nestedLayers).forEach(id => this.nestedLayers[id].destroy());
+        this.nestedLayers = {};
+
+        if (this.$nested) {
+            this.$nested.remove();
+            this.$nested = this.$el.append('g')
+                .attr('class', 'nested-layers');
+        }
+    };
+
+    ContainerLayerDecorator.prototype.destroy = function() {
+        LayerDecorator.prototype.destroy.call(this);
+        if (this._nestedTerritoryUI) {
+            this.client.removeUI(this._nestedTerritoryUI);
+            this._nestedTerritoryUI = null;
+        }
+        this.destroyNested();
+    };
+
+    var CreateNestedBtn = function(params) {
+        params.title = 'Add nested layer';
+        Buttons.Add.call(this, params);
+    };
+
+    CreateNestedBtn.SIZE = Buttons.Add.SIZE;
+    CreateNestedBtn.prototype = Object.create(Buttons.Add.prototype);
+
+    CreateNestedBtn.prototype._onClick = function() {
+        // Call addLayerAfter and prompt for a layer
+        this.promptLayer()
+            .then(layerId => this.addLayerAt(layerId, 0));
+    };
+
+    ContainerLayerDecorator.prototype.expandAll = function() {
+        this.expand();
+        // For each of the nested layers, expand all their nodes
+        Object.keys(this.nestedLayers)
+            .forEach(id => this.nestedLayers[id].widget.expandAllNodes());
+    };
+
+    ContainerLayerDecorator.prototype.condenseAll = function() {
+        this.condense();
+        // For each of the nested layers, expand all their nodes
+        Object.keys(this.nestedLayers)
+            .forEach(id => this.nestedLayers[id].widget.expandAllNodes(true));
+    };
+
+    return ContainerLayerDecorator;
+});

src/decorators/ContainerLayerDecorator/EasyDAG/NestedLayer.js

@@ -0,0 +1,175 @@
+/*globals define, _ */
+define([
+    'panels/ArchEditor/ArchEditorControl',
+    'widgets/ArchEditor/ArchEditorWidget',
+    'widgets/EasyDAG/Buttons'
+], function(
+    ArchEditor,
+    ArchEditorWidget,
+    Buttons
+) {
+    var nop = () => {};
+    var NestedLayer = function(opts) {
+        this.$el = opts.$container.append('g')
+            .attr('class', 'nested-layer');
+
+        this.id = opts.id;
+        this._parent = opts.parent;
+        this.logger = opts.logger;
+
+        this.refreshButtons = _.debounce(this.updateButtons.bind(this), 100);
+        this.$outline = this.$el.append('rect')  // for hover detection
+            .attr('fill-opacity', 0)
+            .attr('x', 0)
+            .attr('y', 0);
+
+        this.$content = this.$el.append('g');
+        this.initHover();
+
+        this.widget = new ArchEditorWidget({
+            logger: this.logger.fork('ArchWidget'),
+            autoCenter: false,
+            svg: this.$content
+        });
+        this.widget.setTitle =
+        this.widget.updateEmptyMsg = nop;
+        this.onRefresh = opts.onRefresh;
+        this.widget.refreshExtras = this.onWidgetRefresh.bind(this);
+
+        this.control = new ArchEditor({
+            logger: this.logger.fork('ArchControl'),
+            client: opts.client,
+            embedded: true,
+            widget: this.widget
+        });
+        this.control._onUnload = () => {
+            ArchEditor.prototype._onUnload.apply(this.control, arguments);
+            // If it was the last node, remove it
+            var node = this.control._client.getNode(this.id);
+            if (node.getChildrenIds().length === 0) {
+                this.onLastNodeRemoved();
+            }
+        };
+
+        // hack :(
+        this.control.$btnModelHierarchyUp = {
+            show: nop,
+            hide: nop
+        };
+        this.widget.active = true;
+        this.control.selectedObjectChanged(this.id);
+    };
+
+    NestedLayer.prototype.initHover = function() {
+        this.$hover = this.$el.append('g')
+            .attr('class', 'hover-items');
+
+
+
+        this.$el.on('mouseenter', this.onHover.bind(this));
+        this.$el.on('mouseleave', this.onUnhover.bind(this));
+
+        // Buttons
+        this.$leftBtn = new Buttons.Add({
+            hide: true,
+            icon: this.isFirst() ? 'plus' : 'chevron-left',
+            $pEl: this.$hover
+        });
+
+        this.$rightBtn = new Buttons.Add({
+            hide: true,
+            icon: this.isLast() ? 'plus' : 'chevron-right',
+            $pEl: this.$hover
+        });
+
+        this.$deleteBtn = new Buttons.DeleteOne({
+            hide: true,
+            title: 'Delete',
+            $pEl: this.$hover
+        });
+
+        this.$leftBtn._onClick = this.clickLeft.bind(this);
+        this.$rightBtn._onClick = this.clickRight.bind(this);
+        this.$deleteBtn._onClick = () => this.onLastNodeRemoved();
+
+        this.$leftHint = this.$leftBtn.$el.append('title');
+        this.$rightHint = this.$rightBtn.$el.append('title');
+        this.refreshButtons();
+    };
+
+    NestedLayer.prototype.updateButtons = function() {
+        this.$leftBtn.icon = this.isFirst() ? 'plus' : 'chevron-left';
+        this.$rightBtn.icon = this.isLast() ? 'plus' : 'chevron-right';
+
+        this.$leftHint.text(this.isFirst() ?
+            'Add nested layer' :
+            'Move nested layer left'
+        );
+        this.$rightHint.text(this.isLast() ?
+            'Add nested layer' :
+            'Move nested layer right'
+        );
+
+        this.$leftBtn.render();
+        this.$rightBtn.render();
+    };
+
+    NestedLayer.prototype.clickLeft = function() {
+        if (this.isFirst()) {
+            this.promptLayer()
+                .then(layerId => this.addLayerBefore(layerId));
+        } else {
+            this.moveLayerForward();
+        }
+        this.onUnhover();
+    };
+
+    NestedLayer.prototype.promptLayer = function() {
+        var nodes = this.widget.getValidInitialNodes();
+
+        return this.widget.promptLayer(nodes)
+            .then(selected => selected.node.id);
+    };
+
+    NestedLayer.prototype.clickRight = function() {
+        if (this.isLast()) {
+            this.promptLayer()
+                .then(layerId => this.addLayerAfter(layerId));
+        } else {
+            this.moveLayerBackward();
+        }
+        this.onUnhover();
+    };
+
+    NestedLayer.prototype.onHover = function() {
+        this.refreshButtons();
+        this.$hover.attr('class', 'hover-items hovered');
+    };
+
+    NestedLayer.prototype.onUnhover = function() {
+        this.$hover.attr('class', 'hover-items unhovered');
+    };
+
+    NestedLayer.prototype.onWidgetRefresh = function() {
+        var width = this.widget.getSvgWidth(),
+            height = this.widget.getSvgHeight();
+
+        this.$outline
+            .attr('width', width)
+            .attr('height', height);
+
+        this.$leftBtn.$el.attr('transform', `translate(0, ${height/2})`);
+        this.$rightBtn.$el
+            .attr('transform', `translate(${width}, ${height/2})`);
+
+        this.onRefresh();
+    };
+
+    NestedLayer.prototype.destroy = function() {
+        this.control.destroy();
+        this.widget.destroy();
+        this.$el.remove();
+    };
+
+    return NestedLayer;
+});

src/decorators/DcOpDecorator/EasyDAG/DcOpDecorator.EasyDAGWidget.js

@@ -47,11 +47,11 @@ define([
             // create the outputId node
             outputId = this._createOutputNode(baseId);
         } else {
-            this.client.makePointer(outputId, CONSTANTS.POINTER_BASE, baseId);
+            this.client.setPointer(outputId, CONSTANTS.POINTER_BASE, baseId);
         }
         // Copy the data content to the output node
         hash = target.getAttribute('data');
-        this.client.setAttributes(outputId, 'data', hash);
+        this.client.setAttribute(outputId, 'data', hash);
     };
 
     DcOpDecorator.prototype._createOutputNode = function(baseId) {
@@ -69,7 +69,7 @@ define([
             return metaType.getAttribute('name') === 'Outputs';
         });
 
-        return this.client.createChild({
+        return this.client.createNode({
             baseId: baseId,
             parentId: outputCntrId
         });

src/decorators/LayerDecorator/EasyDAG/LayerDecorator.EasyDAGWidget.js

@@ -77,11 +77,11 @@ define([
         // Create a nested "architecture" node and set the ptr target to it
         baseId = base.getId();
         this.client.startTransaction(msg);
-        tgtId = this.client.createChild({
+        tgtId = this.client.createNode({
             parentId: this._node.id,
             baseId: baseId
         });
-        this.client.setAttributes(tgtId, 'name', `${ptr} (${this._node.name})`);
+        this.client.setAttribute(tgtId, 'name', `${ptr} (${this._node.name})`);
         this.savePointer(ptr, tgtId);
         this.client.completeTransaction();
         WebGMEGlobal.State.registerActiveObject(tgtId);
@@ -95,7 +95,7 @@ define([
             // If the target is contained in the current node, delete it!
             if (currentId.indexOf(this._node.id) === 0) {
                 this.client.startTransaction(`Removing layer for ${ptr} of ${name}`);
-                this.client.delMoreNodes([currentId]);
+                this.client.deleteNode(currentId);
                 this.client.completeTransaction();
                 this.logger.info(`Removed ${ptr} and deleted target (${currentId})`);
             } else {

src/decorators/OpIntDecorator/EasyDAG/OpIntDecorator.EasyDAGWidget.js

@@ -102,8 +102,8 @@ define([
             msg = `Deleting "${name}" attribute from "${opName}" operation`;
 
         this.client.startTransaction(msg);
-        this.client.removeAttributeSchema(this._node.id, name);
-        this.client.delAttributes(this._node.id, name);
+        this.client.delAttributeMeta(this._node.id, name);
+        this.client.delAttribute(this._node.id, name);
         this.client.completeTransaction();
     };
 
@@ -129,14 +129,14 @@ define([
 
         if (name !== desc.name) {  // Renaming attribute
             if (name) {
-                this.client.removeAttributeSchema(this._node.id, name);
-                this.client.delAttributes(this._node.id, name);
+                this.client.delAttributeMeta(this._node.id, name);
+                this.client.delAttribute(this._node.id, name);
             }
             name = desc.name;
         }
 
-        this.client.setAttributeSchema(this._node.id, name, schema);
-        this.client.setAttributes(this._node.id, name, desc.defaultValue);
+        this.client.setAttributeMeta(this._node.id, name, schema);
+        this.client.setAttribute(this._node.id, name, desc.defaultValue);
         this.client.completeTransaction();
     };
 

src/decorators/OperationDecorator/EasyDAG/OperationDecorator.EasyDAGWidget.js

@@ -73,14 +73,15 @@ define([
 
     OperationDecorator.prototype.showPorts = function(ids, areInputs) {
         var allPorts = areInputs ? this._node.inputs : this._node.outputs,
-            ports = ids ? allPorts.filter(port => ids.indexOf(port.id) > -1) : allPorts,
             x = -this.width/2,
-            dx = this.width/(ports.length+1),
+            dx = this.width/(allPorts.length+1),
             y = areInputs ? 0 : this.height;  // (this.height/2);
 
-        ports.forEach(port => {
+        allPorts.forEach(port => {
             x += dx;
-            this.renderPort(port, x, y, areInputs);
+            if (!ids || ids.indexOf(port.id) > -1) {
+                this.renderPort(port, x, y, areInputs);
+            }
         });
     };
 
@@ -117,7 +118,7 @@ define([
         tooltip = new Opentip(portIcon[0][0], PORT_TOOLTIP_OPTS);
         tooltip.setContent(port.name);
         portIcon.on('mouseenter', () => tooltip.show());
-        portIcon.on('mouseout', () => tooltip.hide());
+        portIcon.on('mouseleave', () => tooltip.hide());
         this.$portTooltips[port.id] = tooltip;
     };
 

src/layouts/SidebarLayout/SidebarLayout.css

@@ -0,0 +1,12 @@
+.ui-layout-center .layout-center {
+    left: 40px;
+    height: 100%;
+    width: 100%;
+    position: absolute;
+}
+
+.ui-layout-sidebar {
+    top: 64px;
+    width: 40px;
+    bottom: 27px;
+}

src/layouts/SidebarLayout/SidebarLayout.js

@@ -0,0 +1,57 @@
+/*globals define, */
+define([
+    'layout/CHFLayout/CHFLayout/CHFLayout',
+    'text!./templates/SidebarLayout.html',
+    'css!./SidebarLayout.css'
+], function(
+    CHFLayout,
+    SidebarTemplate
+) {
+    'use strict';
+    
+    var SidebarLayout = function(params) {
+        params = params || {};
+        params.template = SidebarTemplate;
+        CHFLayout.call(this, params);
+    };
+
+    SidebarLayout.prototype = Object.create(CHFLayout.prototype);
+
+    SidebarLayout.prototype.getComponentId = function () {
+        return 'SidebarLayout';
+    };
+
+    /**
+     * Initialize the html page. This example is using the jQuery Layout plugin.
+     *
+     * @return {undefined}
+     */
+    SidebarLayout.prototype.init = function() {
+        CHFLayout.prototype.init.apply(this, arguments);
+        this._sidebarPanel = this._body.find('div.ui-layout-sidebar');
+        this._centerPanel = this._body.find('div.layout-center');
+    };
+
+    /**
+     * Add a panel to a given container. This is defined in the corresponding
+     * layout config JSON file.
+     *
+     * @param {Panel} panel
+     * @param {String} container
+     * @return {undefined}
+     */
+    SidebarLayout.prototype.addToContainer = function(panel, container) {
+        if (container === 'sidebar') {
+            this._sidebarPanel.append(panel.$pEl);
+        } else {
+            CHFLayout.prototype.addToContainer.apply(this, arguments);
+        }
+    };
+
+    SidebarLayout.prototype._onCenterResize = function() {
+        var width = this._centerPanel.width() - this._sidebarPanel.width();
+        this._canvas.setSize(width, this._centerPanel.height());
+    };
+
+    return SidebarLayout;
+});

src/layouts/SidebarLayout/templates/SidebarLayout.html

@@ -0,0 +1,7 @@
+<div class="ui-layout-center" style="position: relative;">
+    <div class="layout-center"></div>
+    <div class="float"></div>
+</div>
+<div class="ui-layout-sidebar"></div>
+<div class="ui-layout-north"></div>
+<div class="ui-layout-south"></div>

src/plugins/CheckLibraries/CheckLibraries.js

@@ -92,7 +92,7 @@ define([
 
                     this.logger.info(`${tuple[0]} version info:\n${projVersion} ` +
                         `(project)\n${latest} (latest)`);
-                    return latest !== projVersion;
+                    return projVersion < latest;
                 });
 
                 return Q.all(tuples.map(tuple => this.uploadSeed.apply(this, tuple)));

src/plugins/CreateTorchMeta/schemas/nn.json

@@ -65,7 +65,14 @@
       "affine"
     ],
     "setters": {},
-    "defaults": {},
+    "types": {
+      "eps": "number",
+      "momentum": "number"
+    },
+    "defaults": {
+      "momentum": 0.1,
+      "eps": 0.00001
+    },
     "type": "Misc"
   },
   {
@@ -82,6 +89,35 @@
     "defaults": {},
     "type": "Misc"
   },
+  {
+    "name": "Bottle",
+    "baseType": "Container",
+    "params": [
+      "module",
+      "nInputDim",
+      "nOutputDim"
+    ],
+    "setters": {},
+    "types": {
+      "nInputDim": "number",
+      "module": "nn.Module"
+    },
+    "defaults": {
+      "nInputDim": 2
+    },
+    "type": "Container"
+  },
+  {
+    "name": "CAdd",
+    "baseType": "Module",
+    "params": [
+      "params"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Misc"
+  },
   {
     "name": "CAddTable",
     "baseType": "Module",
@@ -102,6 +138,24 @@
     "defaults": {},
     "type": "Misc"
   },
+  {
+    "name": "CMaxTable",
+    "baseType": "Module",
+    "params": [],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Misc"
+  },
+  {
+    "name": "CMinTable",
+    "baseType": "Module",
+    "params": [],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Misc"
+  },
   {
     "name": "CMul",
     "baseType": "Module",
@@ -175,7 +229,7 @@
     "setters": {},
     "types": {},
     "defaults": {},
-    "type": "Containers"
+    "type": "Simple"
   },
   {
     "name": "ConcatTable",
@@ -184,7 +238,7 @@
     "setters": {},
     "types": {},
     "defaults": {},
-    "type": "Misc"
+    "type": "Container"
   },
   {
     "name": "Contiguous",
@@ -296,13 +350,16 @@
     "params": [
       "p",
       "v1",
-      "inplace"
+      "inplace",
+      "stochasticInference"
     ],
     "setters": {},
     "types": {
-      "p": "number"
+      "p": "number",
+      "stochasticInference": "boolean"
     },
     "defaults": {
+      "stochasticInference": false,
       "p": 0.5
     },
     "type": "Simple"
@@ -357,8 +414,11 @@
   {
     "name": "GradientReversal",
     "baseType": "Module",
-    "params": [],
+    "params": [
+      "lambda"
+    ],
     "setters": {},
+    "types": {},
     "defaults": {},
     "type": "Misc"
   },
@@ -382,7 +442,8 @@
     "baseType": "Module",
     "params": [
       "min_value",
-      "max_value"
+      "max_value",
+      "inplace"
     ],
     "setters": {},
     "types": {
@@ -508,9 +569,7 @@
   {
     "name": "Log",
     "baseType": "Module",
-    "params": [
-      "inputSize"
-    ],
+    "params": [],
     "setters": {},
     "types": {},
     "defaults": {},
@@ -865,7 +924,7 @@
     "setters": {},
     "types": {},
     "defaults": {},
-    "type": "Misc"
+    "type": "Container"
   },
   {
     "name": "ParallelCriterion",
@@ -885,7 +944,7 @@
     "setters": {},
     "types": {},
     "defaults": {},
-    "type": "Misc"
+    "type": "Container"
   },
   {
     "name": "PartialLinear",
@@ -900,6 +959,17 @@
     "defaults": {},
     "type": "Misc"
   },
+  {
+    "name": "PixelShuffle",
+    "baseType": "Module",
+    "params": [
+      "upscaleFactor"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Misc"
+  },
   {
     "name": "Power",
     "baseType": "Module",
@@ -939,6 +1009,17 @@
     "defaults": {},
     "type": "Transfer"
   },
+  {
+    "name": "ReLU6",
+    "baseType": "Module",
+    "params": [
+      "inplace"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Transfer"
+  },
   {
     "name": "Replicate",
     "baseType": "Module",
@@ -1156,6 +1237,18 @@
     "defaults": {},
     "type": "Convolution"
   },
+  {
+    "name": "SpatialClassNLLCriterion",
+    "baseType": "Criterion",
+    "params": [
+      "weights",
+      "sizeAverage"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Criterion"
+  },
   {
     "name": "SpatialContrastiveNormalization",
     "baseType": "Module",
@@ -1279,6 +1372,56 @@
     },
     "type": "Convolution"
   },
+  {
+    "name": "SpatialDilatedConvolution",
+    "baseType": "SpatialConvolution",
+    "params": [
+      "nInputPlane",
+      "nOutputPlane",
+      "kW",
+      "kH",
+      "dW",
+      "dH",
+      "padW",
+      "padH",
+      "dilationW",
+      "dilationH"
+    ],
+    "setters": {},
+    "types": {
+      "dilationW": "number",
+      "dilationH": "number"
+    },
+    "defaults": {
+      "dilationH": 1,
+      "dilationW": 1
+    },
+    "type": "Convolution"
+  },
+  {
+    "name": "SpatialDilatedMaxPooling",
+    "baseType": "SpatialMaxPooling",
+    "params": [
+      "kW",
+      "kH",
+      "dW",
+      "dH",
+      "padW",
+      "padH",
+      "dilationW",
+      "dilationH"
+    ],
+    "setters": {},
+    "types": {
+      "dilationW": "number",
+      "dilationH": "number"
+    },
+    "defaults": {
+      "dilationH": 1,
+      "dilationW": 1
+    },
+    "type": "Misc"
+  },
   {
     "name": "SpatialDivisiveNormalization",
     "baseType": "Module",
@@ -1301,13 +1444,16 @@
     "name": "SpatialDropout",
     "baseType": "Module",
     "params": [
-      "p"
+      "p",
+      "stochasticInference"
     ],
     "setters": {},
     "types": {
-      "p": "number"
+      "p": "number",
+      "stochasticInference": "boolean"
     },
     "defaults": {
+      "stochasticInference": false,
       "p": 0.5
     },
     "type": "Convolution"
@@ -1387,6 +1533,14 @@
     "defaults": {},
     "type": "Convolution"
   },
+  {
+    "name": "SpatialLogSoftMax",
+    "baseType": "Module",
+    "params": [],
+    "setters": {},
+    "defaults": {},
+    "type": "Misc"
+  },
   {
     "name": "SpatialMaxPooling",
     "baseType": "Module",
@@ -1497,6 +1651,17 @@
     },
     "type": "Convolution"
   },
+  {
+    "name": "SpatialUpSamplingBilinear",
+    "baseType": "Module",
+    "params": [
+      "params"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Misc"
+  },
   {
     "name": "SpatialUpSamplingNearest",
     "baseType": "Module",
@@ -1627,6 +1792,22 @@
     "defaults": {},
     "type": "Convolution"
   },
+  {
+    "name": "TemporalDynamicKMaxPooling",
+    "baseType": "Module",
+    "params": [
+      "minK",
+      "factor"
+    ],
+    "setters": {},
+    "types": {
+      "factor": "number"
+    },
+    "defaults": {
+      "factor": 0
+    },
+    "type": "Misc"
+  },
   {
     "name": "TemporalMaxPooling",
     "baseType": "Module",
@@ -1764,16 +1945,81 @@
     "type": "Convolution"
   },
   {
-    "name": "VolumetricDropout",
-    "baseType": "Module",
+    "name": "VolumetricDilatedConvolution",
+    "baseType": "VolumetricConvolution",
     "params": [
-      "p"
+      "nInputPlane",
+      "nOutputPlane",
+      "kT",
+      "kW",
+      "kH",
+      "dT",
+      "dW",
+      "dH",
+      "padT",
+      "padW",
+      "padH",
+      "dilationT",
+      "dilationW",
+      "dilationH"
     ],
     "setters": {},
     "types": {
-      "p": "number"
+      "dilationT": "number",
+      "dilationW": "number",
+      "dilationH": "number"
     },
     "defaults": {
+      "dilationH": 1,
+      "dilationW": 1,
+      "dilationT": 1
+    },
+    "type": "Misc"
+  },
+  {
+    "name": "VolumetricDilatedMaxPooling",
+    "baseType": "VolumetricMaxPooling",
+    "params": [
+      "kT",
+      "kW",
+      "kH",
+      "dT",
+      "dW",
+      "dH",
+      "padT",
+      "padW",
+      "padH",
+      "dilationT",
+      "dilationW",
+      "dilationH"
+    ],
+    "setters": {},
+    "types": {
+      "dilationT": "number",
+      "dilationW": "number",
+      "dilationH": "number"
+    },
+    "defaults": {
+      "dilationH": 1,
+      "dilationW": 1,
+      "dilationT": 1
+    },
+    "type": "Misc"
+  },
+  {
+    "name": "VolumetricDropout",
+    "baseType": "Module",
+    "params": [
+      "p",
+      "stochasticInference"
+    ],
+    "setters": {},
+    "types": {
+      "p": "number",
+      "stochasticInference": "boolean"
+    },
+    "defaults": {
+      "stochasticInference": false,
       "p": 0.5
     },
     "type": "Convolution"
@@ -1864,6 +2110,22 @@
     "defaults": {},
     "type": "Convolution"
   },
+  {
+    "name": "VolumetricReplicationPadding",
+    "baseType": "Module",
+    "params": [
+      "pleft",
+      "pright",
+      "ptop",
+      "pbottom",
+      "pfront",
+      "pback"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Misc"
+  },
   {
     "name": "WeightedEuclidean",
     "baseType": "Module",
@@ -1887,4 +2149,4 @@
     "defaults": {},
     "type": "Criterion"
   }
-]
+]

src/plugins/ExecuteJob/ExecuteJob.Files.js

@@ -0,0 +1,338 @@
+/*globals define*/
+define([
+    './templates/index',
+    'q',
+    'underscore',
+    'deepforge/Constants'
+], function(
+    Templates,
+    Q,
+    _,
+    CONSTANTS
+) {
+
+    var ExecuteJob = function() {
+    };
+
+    ExecuteJob.prototype.createOperationFiles = function (node) {
+        var files = {};
+        // For each operation, generate the output files:
+        //   inputs/<arg-name>/init.lua  (respective data deserializer)
+        //   pointers/<name>/init.lua  (result of running the main plugin on pointer target - may need a rename)
+        //   outputs/<name>/  (make dirs for each of the outputs)
+        //   outputs/init.lua  (serializers for data outputs)
+        //
+        //   attributes.lua (returns lua table of operation attributes)
+        //   init.lua (main file -> calls main and serializes outputs)
+        //   <name>.lua (entry point -> calls main operation code)
+
+        // add the given files
+        this.logger.info('About to create dist execution files');
+        files['start.js'] = _.template(Templates.START)(CONSTANTS);
+        return this.createEntryFile(node, files)
+            .then(() => this.createClasses(node, files))
+            .then(() => this.createCustomLayers(node, files))
+            .then(() => this.createInputs(node, files))
+            .then(() => this.createOutputs(node, files))
+            .then(() => this.createMainFile(node, files))
+            .then(() => {
+                this.createAttributeFile(node, files);
+                return Q.ninvoke(this, 'createPointers', node, files);
+            });
+    };
+
+    ExecuteJob.prototype.createEntryFile = function (node, files) {
+        this.logger.info('Creating entry files...');
+        return this.getOutputs(node)
+            .then(outputs => {
+                var name = this.getAttribute(node, 'name'),
+                    content = {};
+
+                // inputs and outputs
+                content.name = name;
+                content.outputs = outputs;
+
+                files['init.lua'] = _.template(Templates.ENTRY)(content);
+
+                // Create the deepforge file
+                files['deepforge.lua'] = _.template(Templates.DEEPFORGE)(CONSTANTS);
+            });
+    };
+
+    ExecuteJob.prototype.createClasses = function (node, files) {
+        var metaDict = this.core.getAllMetaNodes(this.rootNode),
+            isClass,
+            metanodes,
+            classNodes,
+            inheritanceLvl = {},
+            code;
+
+        this.logger.info('Creating custom layer file...');
+        metanodes = Object.keys(metaDict).map(id => metaDict[id]);
+        isClass = this.getTypeDictFor('Complex', metanodes);
+
+        classNodes = metanodes.filter(node => {
+            var base = this.core.getBase(node),
+                baseId,
+                count = 1;
+
+            // Count the sets back to a class node
+            while (base) {
+                baseId = this.core.getPath(base);
+                if (isClass[baseId]) {
+                    inheritanceLvl[this.core.getPath(node)] = count;
+                    return true;
+                }
+                base = this.core.getBase(base);
+                count++;
+            }
+
+            return false;
+        });
+
+        // Get the code definitions for each
+        // Sort by levels of inheritance...
+        code = classNodes.sort((a, b) => {
+            var aId = this.core.getPath(a),
+                bId = this.core.getPath(b);
+
+            return inheritanceLvl[aId] > inheritanceLvl[bId];
+        }).map(node =>
+            `require './${this.getAttribute(node, 'name')}.lua'`
+        ).join('\n');
+
+        // Create the class files
+        classNodes.forEach(node => {
+            var name = this.getAttribute(node, 'name');
+            files[`classes/${name}.lua`] = this.getAttribute(node, 'code');
+        });
+
+        // Create the custom layers file
+        files['classes/init.lua'] = code;
+    };
+
+    ExecuteJob.prototype.getTypeDictFor = function (name, metanodes) {
+        var isType = {};
+        // Get all the custom layers
+        for (var i = metanodes.length; i--;) {
+            if (this.getAttribute(metanodes[i], 'name') === name) {
+                isType[this.core.getPath(metanodes[i])] = true;
+            }
+        }
+        return isType;
+    };
+
+    ExecuteJob.prototype.createCustomLayers = function (node, files) {
+        var metaDict = this.core.getAllMetaNodes(this.rootNode),
+            isCustomLayer,
+            metanodes,
+            customLayers,
+            code;
+
+        this.logger.info('Creating custom layer file...');
+        metanodes = Object.keys(metaDict).map(id => metaDict[id]);
+        isCustomLayer = this.getTypeDictFor('CustomLayer', metanodes);
+
+        customLayers = metanodes.filter(node =>
+            this.core.getMixinPaths(node).some(id => isCustomLayer[id]));
+
+        // Get the code definitions for each
+        code = 'require \'nn\'\n\n' + customLayers
+            .map(node => this.getAttribute(node, 'code')).join('\n');
+
+        // Create the custom layers file
+        files['custom-layers.lua'] = code;
+    };
+
+    ExecuteJob.prototype.createInputs = function (node, files) {
+        var tplContents,
+            inputs;
+
+        this.logger.info('Retrieving inputs and deserialize fns...');
+        return this.getInputs(node)
+            .then(allInputs => {
+                // For each input, match the connection with the input name
+                //   [ name, type ] => [ name, type, node ]
+                //
+                // For each input,
+                //  - create the deserializer
+                //  - put it in inputs/<name>/init.lua
+                //  - copy the data asset to /inputs/<name>/init.lua
+                inputs = allInputs
+                    .filter(pair => !!this.getAttribute(pair[2], 'data'));  // remove empty inputs
+
+                files.inputAssets = {};  // data assets
+                return Q.all(inputs.map(pair => {
+                    var name = pair[0],
+                        node = pair[2],
+                        nodeId = this.core.getPath(node),
+                        fromNodeId;
+
+                    // Get the deserialize function. First, try to get it from
+                    // the source method (this guarantees that the correct
+                    // deserialize method is used despite any auto-upcasting
+                    fromNodeId = this.inputPortsFor[nodeId][0] || nodeId;
+
+                    return this.core.loadByPath(this.rootNode, fromNodeId)
+                        .then(fromNode => {
+                            var deserFn,
+                                base,
+                                className;
+
+                            deserFn = this.getAttribute(fromNode, 'deserialize');
+
+                            if (this.isMetaTypeOf(node, this.META.Complex)) {
+                                // Complex objects are expected to define their own
+                                // (static) deserialize factory method
+                                base = this.core.getMetaType(node);
+                                className = this.getAttribute(base, 'name');
+                                deserFn = `return ${className}.deserialize(path)`;
+                            }
+
+                            return {
+                                name: name,
+                                code: deserFn
+                            };
+                        });
+                }));
+            })
+            .then(_tplContents => {
+                tplContents = _tplContents;
+                var hashes = inputs.map(pair => {
+                    var hash = this.getAttribute(pair[2], 'data');
+                    files.inputAssets[pair[0]] = hash;
+                    return {
+                        hash: hash,
+                        name: pair[0]
+                    };
+                });
+
+                return Q.all(hashes.map(pair => 
+                    this.blobClient.getMetadata(pair.hash)
+                        .fail(err => this.onBlobRetrievalFail(node, pair.name, err))));
+            })
+            .then(metadatas => {
+                // Create the deserializer
+                tplContents.forEach((ctnt, i) => {
+                    // Get the name of the given asset
+                    ctnt.filename = metadatas[i].name;
+                    files['inputs/' + ctnt.name + '/init.lua'] = _.template(Templates.DESERIALIZE)(ctnt);
+                });
+                return files;
+            });
+    };
+
+    ExecuteJob.prototype.createOutputs = function (node, files) {
+        // For each of the output types, grab their serialization functions and
+        // create the `outputs/init.lua` file
+        this.logger.info('Creating outputs/init.lua...');
+        return this.getOutputs(node)
+            .then(outputs => {
+                var outputTypes = outputs
+                // Get the serialize functions for each
+                    .map(tuple => {
+                        var node = tuple[2],
+                            serFn = this.getAttribute(node, 'serialize');
+
+                        if (this.isMetaTypeOf(node, this.META.Complex)) {
+                            // Complex objects are expected to define their own
+                            // serialize methods
+                            serFn = 'if data ~= nil then data:serialize(path) end';
+                        }
+
+                        return [tuple[1], serFn];
+                    });
+
+                files['outputs/init.lua'] = _.template(Templates.SERIALIZE)({types: outputTypes});
+            });
+    };
+
+    ExecuteJob.prototype.createMainFile = function (node, files) {
+        this.logger.info('Creating main file...');
+        return this.getInputs(node)
+            .then(inputs => {
+                var name = this.getAttribute(node, 'name'),
+                    code = this.getAttribute(node, 'code'),
+                    pointers = this.core.getPointerNames(node).filter(ptr => ptr !== 'base'),
+                    content = {
+                        name: name
+                    };
+
+                // Get input data arguments
+                content.inputs = inputs
+                    .map(pair => [pair[0], !this.getAttribute(pair[2], 'data')]);  // remove empty inputs
+
+                // Defined variables for each pointers
+                content.pointers = pointers
+                    .map(id => [id, this.core.getPointerPath(node, id) === null]);
+
+                // Add remaining code
+                content.code = code;
+
+                files['main.lua'] = _.template(Templates.MAIN)(content);
+
+                // Set the line offset
+                var lineOffset = this.getLineOffset(files['main.lua'], code);
+                this.setAttribute(node, CONSTANTS.LINE_OFFSET, lineOffset);
+            });
+    };
+
+    ExecuteJob.prototype.getLineOffset = function (main, snippet) {
+        var i = main.indexOf(snippet),
+            lines = main.substring(0, i).match(/\n/g);
+
+        return lines ? lines.length : 0;
+    };
+
+    ExecuteJob.prototype.createAttributeFile = function (node, files) {
+        var skip = ['code', 'stdout', 'execFiles', 'jobId', 'secret', CONSTANTS.LINE_OFFSET],
+            numOrBool = /^(-?\d+\.?\d*((e|e-)\d+)?|(true|false))$/,
+            table;
+
+        this.logger.info('Creating attributes file...');
+        table = '{\n\t' + this.core.getAttributeNames(node)
+            .filter(attr => skip.indexOf(attr) === -1)
+            .map(name => {
+                var value = this.getAttribute(node, name);
+                if (!numOrBool.test(value)) {
+                    value = `"${value}"`;
+                }
+                return [name, value];
+            })
+            .map(pair => pair.join(' = '))
+            .join(',\n\t') + '\n}';
+
+        files['attributes.lua'] = `-- attributes of ${this.getAttribute(node, 'name')}\nreturn ${table}`;
+    };
+
+    ExecuteJob.prototype.createPointers = function (node, files, cb) {
+        var pointers,
+            nIds;
+
+        this.logger.info('Creating pointers file...');
+        pointers = this.core.getPointerNames(node)
+            .filter(name => name !== 'base')
+            .filter(id => this.core.getPointerPath(node, id) !== null);
+
+        nIds = pointers.map(p => this.core.getPointerPath(node, p));
+        files.ptrAssets = {};
+        Q.all(
+            nIds.map(nId => this.getPtrCodeHash(nId))
+        )
+        .then(resultHashes => {
+            var name = this.getAttribute(node, 'name');
+            this.logger.info(`Pointer generation for ${name} FINISHED!`);
+            resultHashes.forEach((hash, index) => {
+                files.ptrAssets[`pointers/${pointers[index]}/init.lua`] = hash;
+            });
+            return cb(null, files);
+        })
+        .fail(e => {
+            this.logger.error(`Could not generate pointer files for ${this.getAttribute(node, 'name')}: ${e.toString()}`);
+            return cb(e);
+        });
+    };
+
+    return ExecuteJob;
+});
+

src/plugins/ExecuteJob/ExecuteJob.Metadata.js

@@ -0,0 +1,122 @@
+/*globals define*/
+define([
+    'deepforge/Constants'
+], function(
+    CONSTANTS
+) {
+
+    var ExecuteJob = function() {
+    };
+
+    ExecuteJob.prototype[CONSTANTS.GRAPH_CREATE] = function (job, id) {
+        var graph,
+            name = Array.prototype.slice.call(arguments, 2).join(' '),
+            jobId = this.core.getPath(job);
+
+        id = jobId + '/' + id;
+        this.logger.info(`Creating graph ${id} named ${name}`);
+
+        // Check if the graph already exists
+        graph = this._getExistingMetadata(jobId, 'Graph', name);
+        if (!graph) {
+            graph = this.createNode('Graph', job);
+
+            if (name) {
+                this.setAttribute(graph, 'name', name);
+            }
+            this.createIdToMetadataId[graph] = id;
+        }
+
+        this._metadata[id] = graph;
+    };
+
+    ExecuteJob.prototype[CONSTANTS.GRAPH_PLOT] = function (job, id, x, y) {
+        var jobId = this.core.getPath(job),
+            nonNum = /[^\d-\.]*/g,
+            line,
+            points;
+            
+
+        id = jobId + '/' + id;
+        this.logger.info(`Adding point ${x}, ${y} to ${id}`);
+        line = this._metadata[id];
+        if (!line) {
+            this.logger.warn(`Can't add point to non-existent line: ${id}`);
+            return;
+        }
+
+        // Clean the points by removing and special characters
+        x = x.replace(nonNum, '');
+        y = y.replace(nonNum, '');
+        points = this.getAttribute(line, 'points');
+        points += `${x},${y};`;
+        this.setAttribute(line, 'points', points);
+    };
+
+    ExecuteJob.prototype[CONSTANTS.GRAPH_CREATE_LINE] = function (job, graphId, id) {
+        var jobId = this.core.getPath(job),
+            graph = this._metadata[jobId + '/' + graphId],
+            name = Array.prototype.slice.call(arguments, 3).join(' '),
+            line;
+
+        // Create a 'line' node in the given Graph metadata node
+        name = name.replace(/\s+$/, '');
+        line = this.createNode('Line', graph);
+        this.setAttribute(line, 'name', name);
+        this._metadata[jobId + '/' + id] = line;
+        this.createIdToMetadataId[line] = jobId + '/' + id;
+    };
+
+    ExecuteJob.prototype[CONSTANTS.IMAGE.BASIC] =
+    ExecuteJob.prototype[CONSTANTS.IMAGE.UPDATE] =
+    ExecuteJob.prototype[CONSTANTS.IMAGE.CREATE] = function (job, hash, imgId) {
+        var name = Array.prototype.slice.call(arguments, 3).join(' '),
+            imageNode = this._getImageNode(job, imgId, name);
+
+        this.setAttribute(imageNode, 'data', hash);
+    };
+
+    ExecuteJob.prototype[CONSTANTS.IMAGE.NAME] = function (job, imgId) {
+        var name = Array.prototype.slice.call(arguments, 2).join(' '),
+            imageNode = this._getImageNode(job, imgId, name);
+
+        this.setAttribute(imageNode, 'name', name);
+    };
+
+    ExecuteJob.prototype._getImageNode = function (job, imgId, name) {
+        var jobId = this.core.getPath(job),
+            id = jobId + '/IMAGE/' + imgId,
+            imageNode = this._metadata[id];  // Look for the metadata imageNode
+
+        if (!imageNode) {
+
+            // Check if the imageNode already exists
+            imageNode = this._getExistingMetadata(jobId, 'Image', name);
+            if (!imageNode) {
+                this.logger.info(`Creating image ${id} named ${name}`);
+                imageNode = this.createNode('Image', job);
+                this.setAttribute(imageNode, 'name', name);
+                this.createIdToMetadataId[imageNode] = id;
+            }
+            this._metadata[id] = imageNode;
+        }
+        return imageNode;
+    };
+
+    ExecuteJob.prototype._getExistingMetadata = function (jobId, type, name) {
+        var oldMetadata = this._oldMetadataByName[jobId] && this._oldMetadataByName[jobId][type],
+            node,
+            id;
+
+        if (oldMetadata && oldMetadata[name]) {
+            id = oldMetadata[name];
+            node = this._markForDeletion[jobId][id];
+            delete this._markForDeletion[jobId][id];
+            this.createdMetadataIds[jobId].push(id);  // used for resuming jobs
+        }
+
+        return node || null;
+    };
+
+    return ExecuteJob;
+});

src/plugins/ExecuteJob/ExecuteJob.SafeSave.js

@@ -0,0 +1,377 @@
+/*globals define*/
+define([
+    'plugin/PluginBase',
+    'common/storage/constants',
+    'q',
+    'common/util/assert'
+], function(
+    PluginBase,
+    STORAGE_CONSTANTS,
+    Q,
+    assert
+) {
+
+    var CREATE_PREFIX = 'created_node_',
+        INDEX = 1;
+
+    var ExecuteJob = function() {
+        this.forkNameBase = null;
+        this.runningJobHashes = [];
+        this._currentSave = Q();
+    };
+
+    ExecuteJob.prototype.getCreateId = function () {
+        return CREATE_PREFIX + (++INDEX);
+    };
+
+    ExecuteJob.prototype.isCreateId = function (id) {
+        return (typeof id === 'string') && (id.indexOf(CREATE_PREFIX) === 0);
+    };
+
+    ExecuteJob.prototype.createNode = function (baseType, parent) {
+        var id = this.getCreateId(),
+            parentId = this.isCreateId(parent) ? parent : this.core.getPath(parent);
+
+        this.logger.info(`Creating ${id} of type ${baseType} in ${parentId}`);
+        assert(this.META[baseType], `Cannot create node w/ unrecognized type: ${baseType}`);
+        this.creations[id] = {
+            base: baseType,
+            parent: parentId
+        };
+        return id;
+    };
+
+    ExecuteJob.prototype.deleteNode = function (nodeId) {
+        this.deletions.push(nodeId);
+    };
+
+    ExecuteJob.prototype.delAttribute = function (node, attr) {
+        return this.setAttribute(node, attr, null);
+    };
+
+    ExecuteJob.prototype.setAttribute = function (node, attr, value) {
+        var nodeId;
+
+        if (this.isCreateId(node)) {
+            nodeId = node;
+        } else {
+            nodeId = this.core.getPath(node);
+            assert(typeof nodeId === 'string', `Cannot set attribute of ${nodeId}`);
+        }
+
+        if (value !== null) {
+            this.logger.info(`Setting ${attr} of ${nodeId} to ${value}`);
+        } else {
+            this.logger.info(`Deleting ${attr} of ${nodeId}`);
+        }
+
+        if (!this.changes[nodeId]) {
+            this.changes[nodeId] = {};
+        }
+        this.changes[nodeId][attr] = value;
+    };
+
+    ExecuteJob.prototype.getAttribute = function (node, attr) {
+        var nodeId;
+
+        assert(this.deletions.indexOf(nodeId) === -1,
+            `Cannot get ${attr} from deleted node ${nodeId}`);
+
+        // Check if it was newly created
+        if (this.isCreateId(node)) {
+            nodeId = node;
+            assert(this.creations[nodeId], `Creation node not updated: ${nodeId}`);
+            node = this.META[this.creations[nodeId].base];
+        } else {
+            nodeId = this.core.getPath(node);
+        }
+
+        // Check the most recent changes, then the currentChanges, then the model
+        var value = this._getValueFrom(nodeId, attr, node, this.changes) ||
+            this._getValueFrom(nodeId, attr, node, this.currentChanges);
+
+        if (value) {
+            return value;
+        }
+
+        return this.core.getAttribute(node, attr);
+    };
+
+    ExecuteJob.prototype._getValueFrom = function (nodeId, attr, node, changes) {
+        var base;
+        if (changes[nodeId] && changes[nodeId][attr] !== undefined) {
+            // If deleted the attribute, get the default (inherited) value
+            if (changes[nodeId][attr] === null) {
+                base = this.isCreateId(nodeId) ? node : this.core.getBase(node);
+                return this.getAttribute(base, attr);
+            }
+            return changes[nodeId][attr];
+        }
+    };
+
+    ExecuteJob.prototype._applyNodeChanges = function (node, changes) {
+        var attr,
+            value;
+
+        this.logger.info(`About to apply changes for ${this.core.getPath(node)}`);
+        for (var i = changes.length; i--;) {
+            attr = changes[i][0];
+            value = changes[i][1];
+            if (value !== null) {
+                this.logger.info(`Setting ${attr} to ${value} (${this.core.getPath(node)})`);
+                this.core.setAttribute(node, attr, value);
+            } else {
+                this.core.delAttribute(node, attr);
+            }
+        }
+        return node;
+    };
+
+    ExecuteJob.prototype.applyModelChanges = function () {
+        return this.applyCreations()
+            .then(() => this.applyChanges())
+            .then(() => this.applyDeletions());
+    };
+
+    ExecuteJob.prototype.applyChanges = function () {
+        var nodeIds = Object.keys(this.changes),
+            attrs,
+            value,
+            changes,
+            promises = [],
+            changesFor = {},
+            id,
+            promise;
+
+        this.logger.info('Collecting changes to apply in commit');
+
+        for (var i = nodeIds.length; i--;) {
+            changes = [];
+            attrs = Object.keys(this.changes[nodeIds[i]]);
+            for (var a = attrs.length; a--;) {
+                value = this.changes[nodeIds[i]][attrs[a]];
+                changes.push([attrs[a], value]);
+            }
+            changesFor[nodeIds[i]] = changes;
+
+            assert(changes, `changes are invalid for ${nodeIds[i]}: ${changes}`);
+            assert(!this.isCreateId(nodeIds[i]),
+                `Creation id not resolved to actual id: ${nodeIds[i]}`);
+            promise = this.core.loadByPath(this.rootNode, nodeIds[i]);
+            promises.push(promise);
+        }
+
+        this.currentChanges = this.changes;
+        this.changes = {};
+        // Need to differentiate between read/write changes.
+        this.logger.info(`About to apply changes for ${promises.length} nodes`);
+        return Q.all(promises)
+            .then(nodes => {
+                for (var i = nodes.length; i--;) {
+                    id = this.core.getPath(nodes[i]);
+                    assert(nodes[i], `node is ${nodes[i]} (${nodeIds[i]})`);
+                    this._applyNodeChanges(nodes[i], changesFor[id]);
+                }
+
+                // Local model is now up-to-date. No longer need currentChanges
+                this.currentChanges = {};
+            });
+    };
+
+    ExecuteJob.prototype.applyCreations = function () {
+        var nodeIds = Object.keys(this.creations),
+            tiers = this.createCreationTiers(nodeIds),
+            creations = this.creations,
+            newIds = {},
+            promise = Q(),
+            tier;
+
+        this.logger.info('Applying node creations');
+        for (var i = 0; i < tiers.length; i++) {
+            tier = tiers[i];
+            // Chain the promises, loading each tier sequentially
+            promise = promise.then(this.applyCreationTier.bind(this, creations, newIds, tier));
+        }
+
+        this.creations = {};
+        return promise;
+    };
+
+    ExecuteJob.prototype.applyCreationTier = function (creations, newIds, tier) {
+        var promises = [],
+            parentId,
+            node;
+
+        for (var j = tier.length; j--;) {
+            node = creations[tier[j]];
+            assert(node, `Could not find create info for ${tier[j]}`);
+            parentId = newIds[node.parent] || node.parent;
+            promises.push(this.applyCreation(tier[j], node.base, parentId));
+        }
+        return Q.all(promises).then(nodes => {
+            // Record the newIds so they can be used to resolve creation ids
+            // in subsequent tiers
+            for (var i = tier.length; i--;) {
+                newIds[tier[i]] = this.core.getPath(nodes[i]);
+            }
+        });
+    };
+
+    // Figure out the dependencies between nodes to create.
+    // eg, if newId1 is to be created in newId2, then newId2 will
+    // be in an earlier tier than newId1. Essentially a topo-sort
+    // on a tree structure
+    ExecuteJob.prototype.createCreationTiers = function (nodeIds) {
+        var tiers = [],
+            prevTier = {},
+            tier = {},
+            id,
+            prevLen,
+            i;
+
+        // Create first tier (created inside existing nodes)
+        for (i = nodeIds.length; i--;) {
+            id = nodeIds[i];
+            if (!this.isCreateId(this.creations[id].parent)) {
+                tier[id] = true;
+                nodeIds.splice(i, 1);
+            }
+        }
+        prevTier = tier;
+        tiers.push(Object.keys(tier));
+
+        // Now, each tier consists of the nodes to be created inside a
+        // node from the previous tier
+        while (nodeIds.length) {
+            prevLen = nodeIds.length;
+            tier = {};
+            for (i = nodeIds.length; i--;) {
+                id = nodeIds[i];
+                if (prevTier[this.creations[id].parent]) {
+                    tier[id] = true;
+                    nodeIds.splice(i, 1);
+                }
+            }
+            prevTier = tier;
+            tiers.push(Object.keys(tier));
+            // Every iteration should find at least one node
+            assert(prevLen > nodeIds.length,
+                `Created empty create tier! Remaining: ${nodeIds.join(', ')}`);
+        }
+
+        return tiers;
+    };
+
+    ExecuteJob.prototype.applyCreation = function (tmpId, baseType, parentId) {
+        var base = this.META[baseType],
+            nodeId,
+            id;
+
+        this.logger.info(`Applying creation of ${tmpId} (${baseType}) in ${parentId}`);
+
+        assert(!this.isCreateId(parentId),
+            `Did not resolve parent id: ${parentId} for ${tmpId}`);
+        assert(base, `Invalid base type: ${baseType}`);
+        return this.core.loadByPath(this.rootNode, parentId)
+            .then(parent => this.core.createNode({base, parent}))
+            .then(node => {  // Update the _metadata records
+                id = this.createIdToMetadataId[tmpId];
+                delete this.createIdToMetadataId[tmpId];
+                this._metadata[id] = node;
+
+                // Update creations
+                nodeId = this.core.getPath(node);
+                if (this.changes[tmpId]) {
+                    assert(!this.changes[nodeId],
+                        `Newly created node cannot already have changes! (${nodeId})`);
+                    this.changes[nodeId] = this.changes[tmpId];
+                    delete this.changes[tmpId];
+                }
+                return node;
+            });
+    };
+
+    ExecuteJob.prototype.applyDeletions = function () {
+        var deletions = this.deletions;
+
+        this.deletions = [];
+        return Q.all(deletions.map(id => this.core.loadByPath(this.rootNode, id)))
+            .then(nodes => {
+                for (var i = nodes.length; i--;) {
+                    this.core.deleteNode(nodes[i]);
+                }
+            });
+    };
+
+    // Override 'save' to notify the user on fork
+    ExecuteJob.prototype.save = function (msg) {
+        this._currentSave = this._currentSave
+            .then(() => this.updateForkName(this.forkNameBase))
+            .then(() => this.applyModelChanges())
+            .then(() => PluginBase.prototype.save.call(this, msg))
+            .then(result => {
+                this.logger.info(`Save finished w/ status: ${result.status}`);
+                if (result.status === STORAGE_CONSTANTS.FORKED) {
+                    return this.onSaveForked(result.forkName);
+                } else if (result.status === STORAGE_CONSTANTS.MERGED) {
+                    this.logger.debug('Merged changes. About to update plugin nodes');
+                    return this.updateNodes();
+                }
+            });
+
+        return this._currentSave;
+    };
+
+    ExecuteJob.prototype.onSaveForked = function (forkName) {
+        var name = this.getAttribute(this.activeNode, 'name'),
+            msg = `"${name}" execution has forked to "${forkName}"`;
+        this.currentForkName = forkName;
+
+        this.logManager.fork(forkName);
+        this.runningJobHashes.forEach(jobId => this.originManager.fork(jobId, forkName));
+        this.sendNotification(msg);
+    };
+
+    ExecuteJob.prototype.updateNodes = function (hash) {
+        var activeId = this.core.getPath(this.activeNode);
+
+        hash = hash || this.currentHash;
+        return Q.ninvoke(this.project, 'loadObject', hash)
+            .then(commitObject => {
+                return this.core.loadRoot(commitObject.root);
+            })
+            .then(rootObject => {
+                this.rootNode = rootObject;
+                return this.core.loadByPath(rootObject,activeId);
+            })
+            .then(activeObject => this.activeNode = activeObject)
+            .then(() => {
+                var metaNames = Object.keys(this.META);
+
+                return Q.all(metaNames.map(name => this.updateMetaNode(name)));
+            })
+            .then(() => {
+                var mdNodes,
+                    mdIds;
+
+                mdIds = Object.keys(this._metadata)
+                    .filter(id => !this.isCreateId(this._metadata[id]));
+
+                mdNodes = mdIds.map(id => this.core.getPath(this._metadata[id]))
+                    .map(nodeId => this.core.loadByPath(this.rootNode, nodeId));
+
+                return Q.all(mdNodes).then(nodes => {
+                    for (var i = nodes.length; i--;) {
+                        this._metadata[mdIds[i]] = nodes[i];
+                    }
+                });
+            });
+    };
+
+    ExecuteJob.prototype.updateMetaNode = function (name) {
+        var id = this.core.getPath(this.META[name]);
+        return this.core.loadByPath(this.rootNode, id).then(node => this.META[name] = node);
+    };
+
+    return ExecuteJob;
+});

src/plugins/ExecuteJob/templates/start.ejs

@@ -74,7 +74,7 @@ requirejs([
     var checkFinished = () => {
         if (exitCode !== null && remainingImageCount === 0) {
             log('finished!');
-            process.exit(exitCode);
+            cleanup();
         }
     };
 
@@ -201,14 +201,45 @@ requirejs([
 
     // Download the large files
     var inputData = JSON.parse(fs.readFileSync('./input-data.json')),
-        inputPaths = Object.keys(inputData);
+        inputPaths = Object.keys(inputData),
+        pid,
+        job,
+        cleanup;
+
+    // Make sure to kill the spawned process group on exit
+    cleanup = function() {
+        if (job) {
+            pid = job.pid;
+            job = null;
+            log(`killing process group: ${pid}`);
+            process.kill(-pid, 'SIGTERM');
+            if (exitCode !== null) {
+                log(`exiting w/ code ${exitCode}`);
+                process.exit(exitCode);
+            }
+        }
+    };
+
+    process.on('exit', () => {
+        log('received "exit" event')
+        cleanup();
+    });
+    process.on('SIGINT', function() {
+        log('received "SIGINT" event')
+        cleanup();
+        process.exit(130);
+    });
+    process.on('uncaughtException', () => {
+        log('received "uncaughtException" event')
+        cleanup();
+    });
 
     // Request the data from the blob
     prepareCache()
         .then(() => Q.all(inputPaths.map(ipath => getData(ipath, inputData[ipath]))))
         .then(() => {
             // Run 'th init.lua' and merge the stdout, stderr
-            var job = spawn('th', ['init.lua']);
+            job = spawn('th', ['init.lua'], {detached: true});
             job.stdout.on('data', onStdout);
             job.stderr.on('data', onStderr);
             job.on('close', code => {

src/plugins/ExecutePipeline/ExecutePipeline.js

@@ -4,17 +4,17 @@
 define([
     'plugin/CreateExecution/CreateExecution/CreateExecution',
     'plugin/ExecuteJob/ExecuteJob/ExecuteJob',
-    'deepforge/JobLogsClient',
     'common/storage/constants',
     'common/core/constants',
+    'deepforge/Constants',
     'q',
     'text!./metadata.json',
     'underscore'
 ], function (
     CreateExecution,
     ExecuteJob,
-    JobLogsClient,
     STORAGE_CONSTANTS,
+    GME_CONSTANTS,
     CONSTANTS,
     Q,
     pluginMetadata,
@@ -33,9 +33,9 @@ define([
     var ExecutePipeline = function () {
         // Call base class' constructor.
         CreateExecution.call(this);
+        ExecuteJob.call(this);
         this.pluginMetadata = pluginMetadata;
 
-        this._currentSave = Q();
         this.changes = {};
         this.currentChanges = {};  // read-only changes being applied
         this.creations = {};
@@ -101,7 +101,8 @@ define([
      * @param {function(string, plugin.PluginResult)} callback - the result callback
      */
     ExecutePipeline.prototype.main = function (callback) {
-        var startPromise;
+        var startPromise,
+            runId;
 
         this.initRun();
         if (this.core.isTypeOf(this.activeNode, this.META.Pipeline)) {
@@ -118,58 +119,133 @@ define([
             return callback('Current node is not a Pipeline or Execution!', this.result);
         }
 
-        // Get the gmeConfig...
-        this.logManager = new JobLogsClient({
-            logger: this.logger,
-            port: this.gmeConfig.server.port,
-            branchName: this.branchName,
-            projectId: this.projectId
-        });
         this._callback = callback;
         this.currentForkName = null;
 
         startPromise
-        .then(() => this.core.loadSubTree(this.activeNode))
-        .then(subtree => {
-            var children = subtree
-                .filter(n => this.core.getParent(n) === this.activeNode);
+            .then(() => this.core.loadSubTree(this.activeNode))
+            .then(subtree => {
+                var children;
 
-            this.pipelineName = this.getAttribute(this.activeNode, 'name');
-            this.logger.debug(`Loaded subtree of ${this.pipelineName}. About to build cache`);
-            this.buildCache(subtree);
-            this.logger.debug('Parsing execution for job inter-dependencies');
-            this.parsePipeline(children);  // record deps, etc
+                children = subtree
+                    .filter(n => this.core.getParent(n) === this.activeNode);
 
-            this.logger.debug('Clearing old results');
-            return this.clearResults();
-        })
-        .then(() => this.executePipeline())
-        .fail(e => this.logger.error(e));
+                this.pipelineName = this.getAttribute(this.activeNode, 'name');
+                this.forkNameBase = this.pipelineName;
+                this.logger.debug(`Loaded subtree of ${this.pipelineName}. About to build cache`);
+                this.buildCache(subtree);
+                this.logger.debug('Parsing execution for job inter-dependencies');
+                this.parsePipeline(children);  // record deps, etc
+
+                // Detect if resuming execution
+                runId = this.getAttribute(this.activeNode, 'runId');
+                return this.isResuming().then(resuming => {
+                    if (resuming) {
+                        this.currentRunId = runId;
+                        this.startExecHeartBeat();
+                        return this.resumePipeline();
+                    }
+                    return this.startPipeline();
+                });
+
+            })
+            .fail(e => this.logger.error(e));
     };
 
-    // Override 'save' to prevent race conditions while saving
-    ExecutePipeline.prototype.save = function (msg) {
-        // When 'save'  is called, it should still finish any current save op
-        // before continuing
-        this._currentSave = this._currentSave
-            .then(() => this.updateForkName(this.pipelineName))
-            .then(() => this.applyModelChanges())
-            .then(() => CreateExecution.prototype.save.call(this, msg))
-            .then(result => {
-                var msg;
-                if (result.status === STORAGE_CONSTANTS.FORKED) {
-                    this.currentForkName = result.forkName;
-                    this.logManager.fork(result.forkName);
-                    msg = `"${this.pipelineName}" execution has forked to "${result.forkName}"`;
-                    this.sendNotification(msg);
-                } else if (result.status === STORAGE_CONSTANTS.MERGED) {
-                    this.logger.debug('Merged changes. About to update plugin nodes');
-                    return this.updateNodes();
+    ExecutePipeline.prototype.isResuming = function () {
+        var currentlyRunning = this.getAttribute(this.activeNode, 'status') === 'running',
+            runId = this.getAttribute(this.activeNode, 'runId');
+
+        if (runId && currentlyRunning) {
+            // Verify that it is on the correct branch
+            return this.originManager.getOrigin(runId)
+                .then(origin => {
+                    if (origin && origin.branch === this.branchName) {
+                        return this.pulseClient.check(runId)
+                            // If it is dead (not unknown!), then resume
+                            .then(status => status === CONSTANTS.PULSE.DEAD);
+                    } else {
+                        return false;
+                    }
+                });
+        }
+        return Q().then(() => false);
+    };
+
+    ExecutePipeline.prototype.resumePipeline = function () {
+        var nodes = Object.keys(this.nodes).map(id => this.nodes[id]),
+            allJobs = nodes.filter(node => this.core.isTypeOf(node, this.META.Job)),
+            status,
+            jobs = {
+                success: [],
+                failed: [],
+                running: [],
+                pending: []
+            };
+
+        this.logger.info(`Resuming pipeline execution: ${this.currentRunId}`);
+
+        // Get all completed jobs' operations and update records for these
+        for (var i = allJobs.length; i--;) {
+            status = this.getAttribute(allJobs[i], 'status');
+            if (!jobs[status]) {
+                jobs[status] = [];
+            }
+
+            // If any running jobs are missing jobIds, set them to pending
+            if (status === 'running' && !this.canResumeJob(allJobs[i])) {
+                jobs.pending.push(allJobs[i]);
+            } else {
+                jobs[status].push(allJobs[i]);
+            }
+        }
+
+        // Remove finished jobs from incomingCounts
+        jobs.success.concat(jobs.failed, jobs.running)
+            .map(job => this.core.getPath(job))
+            .forEach(id => delete this.incomingCounts[id]);
+
+        return Q.all(allJobs.map(job => this.recordOldMetadata(job, true)))
+            .then(() => Q.all(jobs.success.map(job => this.getOperation(job))))
+            .then(ops => ops.forEach(op => this.updateJobCompletionRecords(op)))
+            .then(() => {
+                
+                if (jobs.running.length) {  // Resume all running jobs
+                    return Q.all(jobs.running.map(job => this.resumeJob(job)));
+                } else if (this.completedCount === this.totalCount) {
+                    return this.onPipelineComplete();
+                } else {
+                    // If none are running, try to start the next ones
+                    return this.executeReadyOperations();
                 }
+            })
+            .fail(err => this._callback(err));
+    };
 
-            });
+    ExecutePipeline.prototype.startPipeline = function () {
+        var rand = Math.floor(Math.random()*10000),
+            commit = this.commitHash.replace('#', '');
 
-        return this._currentSave;
+        this.logger.debug('Clearing old results');
+        this.currentRunId = `Pipeline_${commit}_${Date.now()}_${rand}`;
+
+        // Record the execution origin
+        this.originManager.record(this.currentRunId, {
+            nodeId: this.core.getPath(this.activeNode),
+            job: 'N/A',
+            execution: this.getAttribute(this.activeNode, 'name')
+        });
+
+        this.startExecHeartBeat();
+        return this.clearResults()
+            .then(() => this.executePipeline())
+            .fail(e => this.logger.error(e));
+    };
+
+    ExecutePipeline.prototype.onSaveForked = function (forkName) {
+        // Update the origin on fork
+        this.originManager.fork(this.currentRunId, forkName);
+        return ExecuteJob.prototype.onSaveForked.call(this, forkName);
     };
 
     ExecutePipeline.prototype.updateNodes = function (hash) {
@@ -220,6 +296,7 @@ define([
         this.logger.info('Setting all jobs status to "pending"');
         this.logger.debug(`Making a commit from ${this.currentHash}`);
         this.setAttribute(this.activeNode, 'startTime', Date.now());
+        this.setAttribute(this.activeNode, 'runId', this.currentRunId);
         this.core.delAttribute(this.activeNode, 'endTime');
         return this.save(`Initializing ${this.pipelineName} for execution`);
     };
@@ -297,10 +374,10 @@ define([
     };
 
     ExecutePipeline.prototype.getSiblingIdContaining = function (nodeId) {
-        var parentId = this.core.getPath(this.activeNode) + CONSTANTS.PATH_SEP,
+        var parentId = this.core.getPath(this.activeNode) + GME_CONSTANTS.PATH_SEP,
             relid = nodeId.replace(parentId, '');
 
-        return parentId + relid.split(CONSTANTS.PATH_SEP).shift();
+        return parentId + relid.split(GME_CONSTANTS.PATH_SEP).shift();
     };
 
     ExecutePipeline.prototype.executePipeline = function() {
@@ -358,7 +435,8 @@ define([
             msg += 'finished!';
         }
 
-        this.isDeleted().then(isDeleted => {
+        return this.isDeleted().then(isDeleted => {
+            this.stopExecHeartBeat();
             if (!isDeleted) {
 
                 this.logger.debug(`Pipeline "${name}" complete!`);
@@ -441,10 +519,9 @@ define([
 
     ExecutePipeline.prototype.onOperationComplete = function (opNode) {
         var name = this.getAttribute(opNode, 'name'),
-            nextPortIds = this.getOperationOutputIds(opNode),
             jNode = this.core.getParent(opNode),
-            resultPorts,
             jobId = this.core.getPath(jNode),
+            counts,
             hasReadyOps;
 
         // Set the operation to 'success'!
@@ -457,35 +534,9 @@ define([
         this.save(`Operation "${name}" in ${this.pipelineName} completed successfully`)
             .then(() => {
 
-                // Transport the data from the outputs to any connected inputs
-                //   - Get all the connections from each outputId
-                //   - Get the corresponding dst outputs
-                //   - Use these new ids for checking 'hasReadyOps'
-                resultPorts = nextPortIds.map(id => this.inputPortsFor[id])
-                    .reduce((l1, l2) => l1.concat(l2), []);
+                counts = this.updateJobCompletionRecords(opNode);
+                hasReadyOps = counts.indexOf(0) > -1;
 
-                resultPorts
-                    .map((id, i) => [this.nodes[id], this.nodes[nextPortIds[i]]])
-                    .forEach(pair => {  // [ resultPort, nextPort ]
-                        var result = pair[0],
-                            next = pair[1],
-                            hash = this.getAttribute(result, 'data');
-                        
-                        this.logger.info(`forwarding data (${hash}) from ${this.core.getPath(result)} ` +
-                            `to ${this.core.getPath(next)}`);
-                        this.setAttribute(next, 'data', hash);
-                        this.logger.info(`Setting ${jobId} data to ${hash}`);
-                    });
-
-                // For all the nextPortIds, decrement the corresponding operation's incoming counts
-                hasReadyOps = nextPortIds.map(id => this.getSiblingIdContaining(id))
-                    .reduce((l1, l2) => l1.concat(l2), [])
-
-                    // decrement the incoming counts for each operation id
-                    .map(opId => --this.incomingCounts[opId])
-                    .indexOf(0) > -1;
-
-                this.completedCount++;
                 this.logger.debug(`Operation "${name}" completed. ` + 
                     `${this.totalCount - this.completedCount} remaining.`);
                 if (hasReadyOps) {
@@ -496,9 +547,47 @@ define([
             });
     };
 
+    ExecutePipeline.prototype.updateJobCompletionRecords = function (opNode) {
+        var nextPortIds = this.getOperationOutputIds(opNode),
+            resultPorts,
+            counts;
+
+        // Transport the data from the outputs to any connected inputs
+        //   - Get all the connections from each outputId
+        //   - Get the corresponding dst outputs
+        //   - Use these new ids for checking 'hasReadyOps'
+
+        resultPorts = nextPortIds.map(id => this.inputPortsFor[id])  // dst -> src port
+            .reduce((l1, l2) => l1.concat(l2), []);
+
+        resultPorts
+            .map((id, i) => [this.nodes[id], this.nodes[nextPortIds[i]]])
+            .forEach(pair => {  // [ resultPort, nextPort ]
+                var result = pair[0],
+                    next = pair[1],
+                    hash = this.getAttribute(result, 'data');
+                
+                this.logger.info(`forwarding data (${hash}) from ${this.core.getPath(result)} ` +
+                    `to ${this.core.getPath(next)}`);
+                this.setAttribute(next, 'data', hash);
+                //this.logger.info(`Setting ${jobId} data to ${hash}`);
+            });
+
+        // For all the nextPortIds, decrement the corresponding operation's incoming counts
+        counts = nextPortIds.map(id => this.getSiblingIdContaining(id))
+            .reduce((l1, l2) => l1.concat(l2), [])
+
+            // decrement the incoming counts for each operation id
+            .map(opId => --this.incomingCounts[opId]);
+
+        this.completedCount++;
+        return counts;
+    };
+
     ExecutePipeline.prototype.getOperationOutputIds = function(node) {
         var jobId = this.getSiblingIdContaining(this.core.getPath(node));
 
+        // Map the job to it's output ports
         return this.outputsOf[jobId] || [];
     };
 

src/plugins/GenerateArchitecture/GenerateArchitecture.js

@@ -6,13 +6,15 @@ define([
     'SimpleNodes/Constants',
     'deepforge/layer-args',
     'deepforge/utils',
+    'deepforge/Constants',
     'underscore',
     'text!./metadata.json'
 ], function (
     PluginBase,
-    Constants,
+    SimpleNodeConstants,
     createLayerDict,
     utils,
+    Constants,
     _,
     metadata
 ) {
@@ -47,6 +49,11 @@ define([
         this.LayerDict = createLayerDict(this.core, this.META);
         this.uniqueId = 2;
         this.varnames = {net: true};
+        this.definitions = [
+            'require \'nn\'',
+            'require \'rnn\''
+        ];
+
         return PluginBase.prototype.main.apply(this, arguments);
     };
 
@@ -65,15 +72,10 @@ define([
     };
 
     GenerateArchitecture.prototype.createOutputFiles = function (tree) {
-        var layers = tree[Constants.CHILDREN],
+        var layers = tree[SimpleNodeConstants.CHILDREN],
             result = {},
             code = '';
 
-        this.definitions = [
-            'require \'nn\'',
-            'require \'rnn\''
-        ];
-
         // Add an index to each layer
         layers.forEach((l, index) => l[INDEX] = index);
 
@@ -125,12 +127,57 @@ define([
     };
 
     GenerateArchitecture.prototype.createLayer = function (layer) {
-        var args = this.createArgString(layer);
-        return `nn.${layer.name}${args}`;
+        var args = this.createArgString(layer),
+            def = `nn.${layer.name}${args}`,
+            type = layer.base.base.name,
+            addedIds,
+            node,
+            name,
+            children,
+            id;
+
+        // Check if it is a container and has the 'addLayers' set
+        // If so, it should sort them by their registry 'index' and add
+        // each nested architecture's code to the given container
+        if (type === 'Container') {
+            // Get the members of the 'addLayers' set
+            addedIds = {};
+            id = layer[SimpleNodeConstants.NODE_PATH];
+            node = this._nodeCache[id];
+            this.core.getMemberPaths(node, Constants.CONTAINED_LAYER_SET)
+                .forEach(id => addedIds[id] = true);
+
+            // Get the (sorted) children
+            children = layer[SimpleNodeConstants.CHILDREN]
+                .map(child => {  // get (child, index) tuples
+                    var index;
+
+                    id = child[SimpleNodeConstants.NODE_PATH];
+                    index = this.core.getMemberRegistry(node, Constants.CONTAINED_LAYER_SET, id, Constants.CONTAINED_LAYER_INDEX);
+                    return [child, index];
+                })
+                .filter(pair => pair[1] !== undefined)  // remove non-members
+                .sort((a, b) => a[1] < b[1] ? -1 : 1)  // sort by 'index'
+                .map(pair => pair[0]);
+
+
+            var addedLayerDefs = '',
+                firstLayer;
+            for (var i = 0; i < children.length; i++) {
+                id = children[i][SimpleNodeConstants.NODE_PATH];
+                // Get the children!
+                firstLayer = children[i][SimpleNodeConstants.CHILDREN][0];
+                name = this.getVarName(utils.abbr(layer.name + '_' + i));
+                addedLayerDefs += this.createSequential(firstLayer, name).code;
+                def += `:add(${name})`;
+            }
+            this.hoist(addedLayerDefs);
+        }
+        return def;
     };
 
     GenerateArchitecture.prototype.createSequential = function (layer, name) {
-        var next = layer[Constants.NEXT][0],
+        var next = layer[SimpleNodeConstants.NEXT][0],
             args,
             snippet,
             snippets,
@@ -142,7 +189,7 @@ define([
 
         while (layer) {
             // if there is only one successor, just add the given layer
-            if (layer[Constants.PREV].length > 1) {  // sequential layers are over
+            if (layer[SimpleNodeConstants.PREV].length > 1) {  // sequential layers are over
                 next = layer;  // the given layer will be added by the caller
                 break;
             } else {  // add the given layer
@@ -151,11 +198,11 @@ define([
 
             }
 
-            while (layer && layer[Constants.NEXT].length > 1) {  // concat/parallel
+            while (layer && layer[SimpleNodeConstants.NEXT].length > 1) {  // concat/parallel
                 // if there is a fork, recurse and add a concat layer
 
-                this.logger.debug(`detected fork of size ${layer[Constants.NEXT].length}`);
-                snippets = layer[Constants.NEXT].map(nlayer =>
+                this.logger.debug(`detected fork of size ${layer[SimpleNodeConstants.NEXT].length}`);
+                snippets = layer[SimpleNodeConstants.NEXT].map(nlayer =>
                     this.createSequential(nlayer, this.getVarName('net')));
                 code += '\n' + snippets.map(snippet => snippet.code).join('\n');
 
@@ -183,7 +230,7 @@ define([
                                 `concat_${layer[INDEX]}:add(${snippet.name})`)
                             .join('\n') + `\n\n${name}:add(concat_${layer[INDEX]})`;
                         
-                        next = layer[Constants.NEXT][0];
+                        next = layer[SimpleNodeConstants.NEXT][0];
                     } else {
                         next = null;  // no next layers
                     }
@@ -203,7 +250,7 @@ define([
             }
 
             layer = next;
-            next = layer && layer[Constants.NEXT][0];
+            next = layer && layer[SimpleNodeConstants.NEXT][0];
         }
 
         return {
@@ -218,14 +265,14 @@ define([
         var content = layer[arg];
 
         if (typeof content === 'object') {  // layer as arg
-            if (content[Constants.CHILDREN].length) {
+            if (content[SimpleNodeConstants.CHILDREN].length) {
                 // Generate the code for the children of layer[arg]
                 var name = this.getVarName(utils.abbr(arg)),
                     layers;
 
                 this.logger.debug(`Adding layer arg for ${arg} (${layer.name})`);
                 try {
-                    layers = this.genRawArchCode(layer[arg][Constants.CHILDREN], name);
+                    layers = this.genRawArchCode(layer[arg][SimpleNodeConstants.CHILDREN], name);
                 } catch (e) {
                     this.logger.error(`Layer arg creation failed: ${e}`);
                     return null;
@@ -244,7 +291,7 @@ define([
     GenerateArchitecture.prototype.createArgString = function (layer) {
         var setters = this.LayerDict[layer.name].setters,
             setterNames = Object.keys(this.LayerDict[layer.name].setters),
-            base = layer[Constants.BASE],
+            base = layer[SimpleNodeConstants.BASE],
             desc,
             fn,
             layerCode,

src/plugins/GenerateExecFile/GenerateExecFile.js

@@ -1,27 +1,40 @@
-/*globals define, _*/
+/*globals define */
 /*jshint node:true, browser:true*/
 
-/**
- * Generated by PluginGenerator 1.7.0 from webgme on Sat Jun 04 2016 18:01:54 GMT-0500 (CDT).
- * A plugin that inherits from the PluginBase. To see source code documentation about available
- * properties and methods visit %host%/docs/source/PluginBase.html.
- */
-
 define([
     'text!./metadata.json',
+    'text!./deepforge.ejs',
+    'text!./toboolean.lua',
     'plugin/PluginBase',
     'deepforge/plugin/PtrCodeGen',
+    'deepforge/Constants',
+    'underscore',
     'q'
 ], function (
     pluginMetadata,
+    deepForgeTxt,
+    TOBOOLEAN,
     PluginBase,
     PtrCodeGen,
+    CONSTANTS,
+    _,
     Q
 ) {
     'use strict';
 
     pluginMetadata = JSON.parse(pluginMetadata);
-    var HEADER_LENGTH = 60;
+    var HEADER_LENGTH = 60,
+        SKIP_ATTRS = {
+            lineOffset: true,
+            code: true
+        },
+        RESERVED = /^(and|break|do|else|elseifend|false|for|function|if|in|local|nil|not|orrepeat|return|then|true|until|while|print)$/,
+        INDENT = '   ',
+        INIT_CLASSES_FN = '__initClasses',
+        INIT_LAYERS_FN = '__initLayers',
+        DEEPFORGE_CODE = _.template(deepForgeTxt)({
+            initCode: `${INIT_CLASSES_FN}()\n${INDENT}${INIT_LAYERS_FN}()`
+        });
 
     /**
      * Initializes a new instance of GenerateExecFile.
@@ -33,21 +46,7 @@ define([
     var GenerateExecFile = function () {
         // Call base class' constructor.
         PluginBase.call(this);
-        this.pluginMetadata = pluginMetadata;
-
-        this._srcIdFor = {};  // input path -> output data node path
-
-        this._nameFor = {};  // input path -> opname
-        this._dataNameFor = {};  
-        this._opNames = {};
-
-        // topo sort stuff
-        this._nextOps = {};
-        this._incomingCnts = {};
-
-        this._operations = {};
-        this.activeNodeId = null;
-        this.activeNodeDepth = null;
+        this.initRecords();
     };
 
     /**
@@ -61,6 +60,35 @@ define([
     GenerateExecFile.prototype = Object.create(PluginBase.prototype);
     GenerateExecFile.prototype.constructor = GenerateExecFile;
 
+    GenerateExecFile.prototype.initRecords = function() {
+        this.pluginMetadata = pluginMetadata;
+
+        this._srcIdFor = {};  // input path -> output data node path
+
+        this._nameFor = {};  // input path -> opname
+        this._outputNames = {};
+        this._baseNameFor = {};
+        this._dataNameFor = {};  
+        this._instanceNames = {};
+        this._opBaseNames = {};
+        this._fnNameFor = {};
+        this._functions = {};  // function definitions for the operations
+
+        // topo sort stuff
+        this._nextOps = {};
+        this._incomingCnts = {};
+
+        this._operations = {};
+        this.activeNodeId = null;
+        this.activeNodeDepth = null;
+
+        this.isInputOp = {};
+        this._portCache = {};
+        this.inputNode = {};
+        this.outputDataToOpId = {};
+        this.isOutputOp = {};
+    };
+
     /**
      * Main function for the plugin to execute. This will perform the execution.
      * Notes:
@@ -71,6 +99,10 @@ define([
      * @param {function(string, plugin.PluginResult)} callback - the result callback
      */
     GenerateExecFile.prototype.main = function (callback) {
+        var name = this.core.getAttribute(this.activeNode, 'name');
+
+        this.initRecords();
+
         // Get all the children and call generate exec file
         this.activeNodeId = this.core.getPath(this.activeNode);
         this.activeNodeDepth = this.activeNodeId.split('/').length + 1;
@@ -81,7 +113,7 @@ define([
 
         return this.core.loadChildren(this.activeNode)
             .then(nodes => this.createExecFile(nodes))
-            .then(code => this.blobClient.putFile('init.lua', code))
+            .then(code => this.blobClient.putFile(`${name}.lua`, code))
             .then(hash => {
                 this.result.addArtifact(hash);
                 this.result.setSuccess(true);
@@ -91,6 +123,61 @@ define([
     };
 
     GenerateExecFile.prototype.createExecFile = function (children) {
+        return this.createCodeSections(children)
+            .then(sections => {
+                var classes,
+                    initClassFn,
+                    initLayerFn,
+                    code = [];
+
+                // concat all the sections into a single file
+
+                // wrap the class/layer initialization in a fn
+                // Add the classes ordered wrt their deps
+                classes = Object.keys(sections.classes)
+                    .sort((a, b) => {
+                        // if a depends on b, switch them (return 1)
+                        if (sections.classDependencies[a].includes(b)) {
+                            return 1;
+                        }
+                        return -1;
+                    })
+                    // Create fns from the classes
+                    .map(name => [
+                        `local function init${name}()`,
+                        indent(sections.classes[name]),
+                        'end',
+                        `init${name}()`
+                    ].join('\n'));
+
+                initClassFn = [
+                    `local function ${INIT_CLASSES_FN}()`,
+                    indent(classes.join('\n\n')),
+                    'end'
+                ].join('\n');
+
+                code = code.concat(initClassFn);
+
+                // wrap the layers in a function
+                initLayerFn = [
+                    `local function ${INIT_LAYERS_FN}()`,
+                    indent(_.values(sections.layers).join('\n\n')),
+                    'end'
+                ].join('\n');
+                code = code.concat(initLayerFn);
+                code = code.concat(_.values(sections.operations));
+
+                code = code.concat(_.values(sections.pipelines));
+
+                code.push(DEEPFORGE_CODE);
+                code.push('deepforge.initialize()');
+                code.push(sections.main);
+
+                return code.join('\n\n');
+            });
+    };
+
+    GenerateExecFile.prototype.createCodeSections = function (children) {
         // Convert opNodes' jobs to the nested operations
         var opNodes,
             nodes;
@@ -100,18 +187,37 @@ define([
                 nodes = _nodes;
                 opNodes = nodes
                     .filter(node => this.isMetaTypeOf(node, this.META.Operation));
-                return Q.all(nodes.map(node => this.registerNameAndData(node)));
+
+                // Sort the connections to come first
+                nodes
+                    .map(node => [
+                        node,
+                        this.isMetaTypeOf(node, this.META.Transporter) ? -1 : 1
+                    ])
+                    .sort((a, b) => a[1] < b[1] ? -1 : 1);
+
+                return Q.all(nodes.map(node => this.registerNode(node)));
             })
-            .then(() => Q.all(opNodes.map(node => this.createOperation(node))))
+            .then(() => Q.all(opNodes
+                .filter(n => {
+                    var id = this.core.getPath(n);
+                    return !this.isInputOp[id];
+                })
+                .map(node => this.createOperation(node)))
+            )
             .then(operations => {
-                var nextIds = opNodes.map(n => this.core.getPath(n))
-                        .filter(id => !this._incomingCnts[id]);
+                var opDict = {},
+                    firstOpIds;
 
-                operations.forEach(op => this._operations[op.id] = op);
+                firstOpIds = opNodes.map(n => this.core.getPath(n))
+                    .filter(id => !this._incomingCnts[id]);
 
-                // Toposort and concat!
-                return this.combineOpNodes(nextIds);
+                operations.forEach(op => opDict[op.id] = op);
+
+                // Toposort!
+                return this.sortOperations(opDict, firstOpIds);
             })
+            .then(operations => this.generateCodeSections(operations))
             .fail(err => this.logger.error(err));
     };
 
@@ -129,18 +235,20 @@ define([
         );
     };
 
-    GenerateExecFile.prototype.combineOpNodes = function (opIds) {
+    GenerateExecFile.prototype.sortOperations = function (operationDict, opIds) {
         var nextIds = [],
+            sorted = opIds,
             dstIds,
-            code,
             id;
 
-        // Combine all nodes with incoming cnts of 0
-        code = opIds.map(id => this._operations[id].code).join('\n');
+        if (!opIds.length) {
+            return [];
+        }
 
         // Decrement all next ops
         dstIds = opIds.map(id => this._nextOps[id])
             .reduce((l1, l2) => l1.concat(l2), []);
+
         for (var i = dstIds.length; i--;) {
             id = dstIds[i];
             if (--this._incomingCnts[id] === 0) {
@@ -149,64 +257,447 @@ define([
         }
 
         // append
+        return sorted
+            .map(id => operationDict[id])
+            .filter(op => !!op)
+            .concat(this.sortOperations(operationDict, nextIds));
+    };
+
+    GenerateExecFile.prototype.generateCodeSections = function(sortedOps) {
+        // Create the code sections:
+        //  - operation definitions
+        //  - pipeline definition
+        //  - main
+        var code = {},
+            baseIds = [],
+            outputOps = [],
+            mainOps = [];
+
+        // Define the operation functions...
+        code.operations = {};
+        for (var i = 0; i < sortedOps.length; i++) {
+            if (this.isInputOp[sortedOps[i].id]) {
+                continue;
+            }
+            if (!this.isOutputOp[sortedOps[i].id]) {
+                if (!baseIds.includes(sortedOps[i].baseId)) {  // new definition
+                    code.operations[sortedOps[i].basename] = this.defineOperationFn(sortedOps[i]);
+                    baseIds.push(sortedOps[i].baseId);
+                }
+                mainOps.push(sortedOps[i]);
+            } else {
+                outputOps.push(sortedOps[i]);
+            }
+        }
+
+        // Define the pipeline function
+        code.pipelines = this.definePipelineFn(mainOps, outputOps);
+
+        // Define the main body
+        this.addCodeMain(code);
+
+        // Add custom class definitions
+        this.addCustomClasses(code);
+
+        // Add custom layer definitions
+        this.addCustomLayers(code);
+
+        return code;
+    };
+
+    var indent = function(text) {
+        return text.replace(/^/mg, INDENT);
+    };
+
+    GenerateExecFile.prototype.defineOperationFn = function(operation) {
+        var lines = [],
+            args = operation.inputNames || [];
+
+        // Create the function definition
+        args.unshift('attributes');
+        // Add the refs to the end
+        args = args.concat(operation.refNames);
+
+        args = args.join(', ');
+
+        lines.push(`local function ${operation.basename}(${args})`);
+        lines.push(indent(operation.code));
+        lines.push('end');
+
+        return lines.join('\n');
+    };
+
+    GenerateExecFile.prototype.definePipelineFn = function(sortedOps, outputOps) {
+        var inputArgs = Object.keys(this.isInputOp).map(id => this._nameFor[id]),
+            name = this.core.getAttribute(this.activeNode, 'name'),
+            safename = getUniqueName(name, this._opBaseNames),
+            results = [],
+            result = {},
+            returnStat,
+            fnbody;
+
+        // Call each function in order, with the respective attributes, etc
+        fnbody = sortedOps.map(op => this.getOpInvocation(op)).join('\n');
+
+        // Create the return statement
+        results.push('\n\nresults = {}');
+        outputOps.map(op => this.getOutputPair(op))
+            .forEach(pair => results.push(`results['${pair[0]}'] = ${pair[1]}`));
+        results.push('return results');
+        returnStat = results.join('\n');
+
+        // Merge the fnbody, return statement and the function def
+        result[safename] = `local function ${safename} (${inputArgs.join(', ')})\n` +
+            `${indent(fnbody + returnStat)}\nend`;
+
+        return result;
+    };
+
+    GenerateExecFile.prototype.getOutputPair = function(operation) {
+        var input = operation.inputValues[0].slice(),
+            value;
+
+        // Get the src operation name and data value name
+        input[0] += '_results';
+        value = input.join('.');
+        return [this._nameFor[operation.id], value];
+    };
+
+    GenerateExecFile.prototype.addCodeMain = function(sections) {
+        var pipelineName = Object.keys(sections.pipelines)[0],
+            hasBool = false,
+            code = [],
+            loadNodes = {},
+            args;
+
+        args = Object.keys(this.isInputOp).map((id, index) => {
+            var node = this.inputNode[id],
+                base = this.core.getBase(node),
+                type = this.core.getAttribute(base, 'name'),
+                arg = `arg[${index+1}]`;
+
+            if (type === 'boolean') {
+                hasBool = true;
+                return `toboolean(${arg})`;
+            } else if (type === 'number') {
+                return `tonumber(${arg})`;
+            } else if (type === 'string') {
+                return arg;
+            } else {
+                loadNodes[id] = node;
+                return `load['${this._nameFor[id]}'](${arg})`;
+            }
+        });
+
+        // Handle the arg types
+        if (hasBool) {
+            // add toboolean def
+            code.push(TOBOOLEAN);
+        }
+        // Define the 'saveOutputs' method
+        var saveNodes = {};
+        Object.keys(this.outputDataToOpId).forEach(dataId => {
+            var opId = this.outputDataToOpId[dataId];
+            // The key is used for the output name resolution. The
+            // value is used for the serialization fn look-up. So,
+            // the key is the output operation id and the value is
+            // the data port connected to the output operation
+            saveNodes[opId] = this._portCache[this._srcIdFor[dataId]];
+        });
+
+        // Add dictionary of serializers/deserializers
+        code.push(
+            this.createTorchFnDict('load', loadNodes, 'deserialize', 'path'),
+            this.createTorchFnDict('save', saveNodes, 'serialize', 'path, data')
+        );
+
+        // Add a saveOutputs method for convenience
+        code.push([
+            'local function saveOutputs(data)',
+            indent(Object.keys(this.isOutputOp).map(id => {
+                var name = this._nameFor[id];
+                return `print('saving ${name}...')\nsave['${name}']('${name}', data['${name}'])`;
+            }).join('\n')),
+            'end'
+        ].join('\n'));
+
+        code.push(
+            `local outputs = ${pipelineName}(${args.join(', ')})\n` +
+            'saveOutputs(outputs)',
+            'return outputs'
+        );
+
+        sections.main = code.join('\n\n');
+    };
+
+    GenerateExecFile.prototype.createTorchFnDict = function(name, nodeDict, attr, args) {
         return [
-            code,
-            nextIds.length ? this.combineOpNodes(nextIds) : ''
+            `local ${name} = {}`,
+            Object.keys(nodeDict).map(id => {
+                var node = nodeDict[id];
+                return [
+                    `${name}['${this._nameFor[id]}'] = function(${args})`,
+                    indent(this.core.getAttribute(node, attr)),
+                    'end'
+                ].join('\n');
+            }).join('\n')
         ].join('\n');
     };
 
-    GenerateExecFile.prototype.registerNameAndData = function (node) {
+    GenerateExecFile.prototype.addCustomClasses = function(sections) {
+        var metaDict = this.core.getAllMetaNodes(this.rootNode),
+            isClass,
+            metanodes,
+            classNodes,
+            inheritanceLvl = {};
+
+        this.logger.info('Creating custom layer file...');
+        metanodes = Object.keys(metaDict).map(id => metaDict[id]);
+        isClass = this.getTypeDictFor('Complex', metanodes);
+
+        // Store the dependencies for each class
+        sections.classDependencies = {};
+
+        classNodes = metanodes.filter(node => {
+            var base = this.core.getBase(node),
+                baseId,
+                deps = [],
+                name,
+                count = 1;
+
+            // Count the sets back to a class node
+            while (base) {
+                deps.push(this.core.getAttribute(base, 'name'));
+                baseId = this.core.getPath(base);
+                if (isClass[baseId]) {
+                    inheritanceLvl[this.core.getPath(node)] = count;
+                    name = this.core.getAttribute(node, 'name');
+                    sections.classDependencies[name] = deps;
+                    return true;
+                }
+                base = this.core.getBase(base);
+                count++;
+            }
+
+            return false;
+        });
+
+        // Get the code definitions for each
+        sections.classes = {};
+        classNodes
+            .sort((a, b) => {
+                var aId = this.core.getPath(a),
+                    bId = this.core.getPath(b);
+
+                return inheritanceLvl[aId] > inheritanceLvl[bId];
+            })
+            .forEach(node => {
+                var name = this.core.getAttribute(node, 'name'),
+                    code = this.core.getAttribute(node, 'code');
+
+                sections.classes[name] = code;
+            });
+    };
+
+    GenerateExecFile.prototype.addCustomLayers = function(sections) {
+        var metaDict = this.core.getAllMetaNodes(this.rootNode),
+            isCustomLayer,
+            metanodes,
+            customLayers;
+
+        this.logger.info('Creating custom layer file...');
+        metanodes = Object.keys(metaDict).map(id => metaDict[id]);
+        isCustomLayer = this.getTypeDictFor('CustomLayer', metanodes);
+
+        customLayers = metanodes.filter(node =>
+            this.core.getMixinPaths(node).some(id => isCustomLayer[id]));
+
+        // Get the code definitions for each
+        sections.layers = {};
+        customLayers
+            .map(layer => [
+                this.core.getAttribute(layer, 'name'),
+                this.core.getAttribute(layer, 'code')
+            ])
+            .forEach(pair => sections.layers[pair[0]] = pair[1]);
+    };
+
+
+    GenerateExecFile.prototype.getTypeDictFor = function (name, metanodes) {
+        var isType = {};
+        // Get all the custom layers
+        for (var i = metanodes.length; i--;) {
+            if (this.core.getAttribute(metanodes[i], 'name') === name) {
+                isType[this.core.getPath(metanodes[i])] = true;
+            }
+        }
+        return isType;
+    };
+
+    var toAttrString = function(attr) {
+        if (/^\d+\.?\d*$/.test(attr) || /^(true|false|nil)$/.test(attr)) {
+            return attr;
+        }
+        return `"${attr}"`;
+    };
+
+    GenerateExecFile.prototype.getOpInvocation = function(op) {
+        var lines = [],
+            attrs,
+            refInits = [],
+            args;
+
+        attrs = '{' +
+            Object.keys(op.attributes).map(key => `${key}=${toAttrString(op.attributes[key])}`)
+            .join(',') +
+        '}';
+
+        lines.push(`local ${op.name}_attrs = ${attrs}`);
+        args = (op.inputValues || [])
+            .map(val => val instanceof Array ? `${val[0]}_results.${val[1]}` : val);
+
+        args.unshift(op.name + '_attrs');
+            
+        // Create the ref init functions
+        refInits = op.refs.map((code, index) => {
+            return [
+                `local function create_${op.refNames[index]}()`,
+                indent(code),
+                'end'
+            ].join('\n');
+        });
+        lines = lines.concat(refInits);
+        args = args.concat(op.refNames.map(name => `create_${name}()`));
+        args = args.join(', ');
+        lines.push(`local ${op.name}_results = ${op.basename}(${args})`);
+
+        return lines.join('\n');
+    };
+
+    GenerateExecFile.prototype.getOutputName = function(node) {
+        var basename = this.core.getAttribute(node, 'saveName');
+
+        return getUniqueName(basename, this._outputNames, true);
+    };
+
+    GenerateExecFile.prototype.getVariableName = function (/*node*/) {
+        var c = Object.keys(this.isInputOp).length;
+
+        if (c !== 1) {
+            return `input${c}`;
+        }
+
+        return 'input';
+    };
+
+    GenerateExecFile.prototype.registerNode = function (node) {
+        if (this.isMetaTypeOf(node, this.META.Operation)) {
+            return this.registerOperation(node);
+        } else if (this.isMetaTypeOf(node, this.META.Transporter)) {
+            return this.registerTransporter(node);
+        }
+    };
+
+    var getUniqueName = function(namebase, takenDict, unsafeAllowed) {
+        var name,
+            i = 2,
+            isUnsafe = function(name) {
+                return !unsafeAllowed && RESERVED.test(name);
+            };
+
+        if (!unsafeAllowed) {
+            namebase = namebase.replace(/[^A-Za-z\d]/g, '_');
+        }
+        name = namebase;
+        // Get a unique operation name
+        while (takenDict[name] || isUnsafe(name)) {
+            name = namebase + '_' + i;
+            i++;
+        }
+        takenDict[name] = true;
+
+        return name;
+    };
+
+    GenerateExecFile.prototype.registerOperation = function (node) {
         var name = this.core.getAttribute(node, 'name'),
             id = this.core.getPath(node),
-            basename = name,
-            i = 2;
+            base = this.core.getBase(node),
+            baseId = this.core.getPath(base),
+            baseName = this.core.getAttribute(base, 'name');
 
-        if (this.isMetaTypeOf(node, this.META.Operation)) {
-
-            // Get a unique operation name
-            while (this._opNames[name]) {
-                name = basename + '_' + i;
-                i++;
-            }
-
-            // register the unique name
-            this._opNames[name] = true;
-            this._nameFor[id] = name;
-
-            // For operations, register all output data node names by path
-            return this.core.loadChildren(node)
-                .then(cntrs => {
-                    var cntr = cntrs.find(n => this.isMetaTypeOf(n, this.META.Outputs));
-                    return this.core.loadChildren(cntr);
-                })
-                .then(outputs => {
-                    outputs.forEach(output => {
-                        var dataId = this.core.getPath(output);
-
-                        name = this.core.getAttribute(output, 'name');
-                        this._dataNameFor[dataId] = name;
-                    });
-                });
-
-        // For each input data node, register the associated output id
-        } else if (this.isMetaTypeOf(node, this.META.Transporter)) {
-            var outputData = this.core.getPointerPath(node, 'src'),
-                inputData = this.core.getPointerPath(node, 'dst'),
-                srcOpId = this.getOpIdFor(outputData),
-                dstOpId = this.getOpIdFor(inputData);
-
-            this._srcIdFor[inputData] = outputData;
-
-            // Store the next operation ids for the op id
-            if (!this._nextOps[srcOpId]) {
-                this._nextOps[srcOpId] = [];
-            }
-            this._nextOps[srcOpId].push(dstOpId);
-
-            // Increment the incoming counts for each dst op
-            this._incomingCnts[dstOpId] = this._incomingCnts[dstOpId] || 0;
-            this._incomingCnts[dstOpId]++;
+        // If it is an Input/Output operation, assign it a variable name
+        if (baseName === CONSTANTS.OP.INPUT) {
+            this.isInputOp[id] = node;
+            name = this.getVariableName(node);
+        } else if (baseName === CONSTANTS.OP.OUTPUT) {
+            this.isOutputOp[id] = node;
+            name = this.getOutputName(node);
+        } else {
+            // get a unique operation instance name
+            name = getUniqueName(name, this._instanceNames);
         }
+
+        this._nameFor[id] = name;
+
+        // get a unique operation base name
+        if (!this._fnNameFor[baseId]) {
+            name = this.core.getAttribute(base, 'name');
+            name = getUniqueName(name, this._opBaseNames);
+            this._fnNameFor[baseId] = name;
+        }
+
+        // For operations, register all output data node names by path
+        return this.core.loadChildren(node)
+            .then(cntrs => {
+                var outputs = cntrs.find(n => this.isMetaTypeOf(n, this.META.Outputs)),
+                    inputs = cntrs.find(n => this.isMetaTypeOf(n, this.META.Inputs));
+
+                return Q.all([inputs, outputs].map(cntr => this.core.loadChildren(cntr)));
+            })
+            .then(data => {
+                var inputs = data[0],
+                    outputs = data[1];
+
+                // Get the input type
+                outputs.forEach(output => {
+                    var dataId = this.core.getPath(output);
+
+                    name = this.core.getAttribute(output, 'name');
+                    this._dataNameFor[dataId] = name;
+
+                    this._portCache[dataId] = output;
+                });
+                inputs.forEach(input => 
+                    this._portCache[this.core.getPath(input)] = input
+                );
+
+                // Extra recording for input/output nodes in the pipeline
+                if (this.isInputOp[id]) {
+                    this.inputNode[id] = outputs[0];
+                } else if (this.isOutputOp[id]) {
+                    this.outputDataToOpId[this.core.getPath(inputs[0])] = id;
+                }
+            });
+    };
+
+    GenerateExecFile.prototype.registerTransporter = function (node) {
+        var outputData = this.core.getPointerPath(node, 'src'),
+            inputData = this.core.getPointerPath(node, 'dst'),
+            srcOpId = this.getOpIdFor(outputData),
+            dstOpId = this.getOpIdFor(inputData);
+
+        this._srcIdFor[inputData] = outputData;
+
+        // Store the next operation ids for the op id
+        if (!this._nextOps[srcOpId]) {
+            this._nextOps[srcOpId] = [];
+        }
+        this._nextOps[srcOpId].push(dstOpId);
+
+        // Increment the incoming counts for each dst op
+        this._incomingCnts[dstOpId] = this._incomingCnts[dstOpId] || 0;
+        this._incomingCnts[dstOpId]++;
     };
 
     GenerateExecFile.prototype.getOpIdFor = function (dataId) {
@@ -226,16 +717,21 @@ define([
     //     - replace the `return <thing>` w/ `<ref-name> = <thing>`
     GenerateExecFile.prototype.createOperation = function (node) {
         var id = this.core.getPath(node),
+            baseId = this.core.getPath(this.core.getBase(node)),
+            attrNames = this.core.getValidAttributeNames(node),
             operation = {};
 
         operation.name = this._nameFor[id];
+        operation.basename = this._fnNameFor[baseId];
+        operation.baseId = baseId;
         operation.id = id;
         operation.code = this.core.getAttribute(node, 'code');
-
-        // Update the 'code' attribute
-        // Change the last return statement to assign the results to a table
-        operation.code = this.assignResultToVar(operation.code,
-            `${operation.name}_results`);
+        operation.attributes = {};
+        for (var i = attrNames.length; i--;) {
+            if (!SKIP_ATTRS[attrNames[i]]) {
+                operation.attributes[attrNames[i]] = this.core.getAttribute(node, attrNames[i]);
+            }
+        }
 
         // Get all the input names (and sources)
         return this.core.loadChildren(node)
@@ -245,7 +741,7 @@ define([
                 inputs = containers
                     .find(cntr => this.isMetaTypeOf(cntr, this.META.Inputs));
 
-                this.logger.info(`${name} has ${containers.length} cntrs`);
+                this.logger.info(`${operation.name} has ${containers.length} cntrs`);
                 return this.core.loadChildren(inputs);
             })
             .then(data => {
@@ -254,13 +750,18 @@ define([
                     ids = data.map(d => this.core.getPath(d)),
                     srcIds = ids.map(id => this._srcIdFor[id]);
 
-                operation.inputs = inputNames.map((name, i) => {
+                operation.inputNames = inputNames || [];
+                operation.inputValues = inputNames.map((name, i) => {
                     var id = srcIds[i],
                         srcDataName = this._dataNameFor[id],
                         srcOpId = this.getOpIdFor(id),
                         srcOpName = this._nameFor[srcOpId];
 
-                    return `local ${name} = ${srcOpName}_results.${srcDataName}`;
+                    if (this.isInputOp[srcOpId]) {
+                        return this._nameFor[srcOpId];
+                    } else {
+                        return [srcOpName, srcDataName];
+                    }
                 });
 
                 return operation;
@@ -269,8 +770,12 @@ define([
             .then(operation => {
 
                 // For each reference, run the plugin and retrieve the generated code
-                operation.refNames = this.core.getPointerNames(node)
-                    .filter(name => name !== 'base');
+                operation.refNames = [];
+
+                if (!this.isInputOp[operation.id]) {
+                    operation.refNames = this.core.getPointerNames(node)
+                        .filter(name => name !== 'base');
+                }
 
                 var refs = operation.refNames
                     .map(ref => [ref, this.core.getPointerPath(node, ref)]);
@@ -281,16 +786,13 @@ define([
             })
             .then(codeFiles => {
                 operation.refs = codeFiles;
-                this.genOperationCode(operation);
                 return operation;
             });
     };
 
     GenerateExecFile.prototype.genPtrSnippet = function (ptrName, pId) {
         return this.getPtrCodeHash(pId)
-            .then(hash => this.blobClient.getObjectAsString(hash))
-            .then(code => this.createHeader(`creating ${ptrName}`, 40) + '\n' +
-                this.assignResultToVar(code, ptrName));
+            .then(hash => this.blobClient.getObjectAsString(hash));
     };
 
     GenerateExecFile.prototype.createHeader = function (title, length) {
@@ -313,29 +815,37 @@ define([
 
     GenerateExecFile.prototype.genOperationCode = function (operation) {
         var header = this.createHeader(`"${operation.name}" Operation`),
-            codeParts = [];
+            codeParts = [],
+            body = [];
 
         codeParts.push(header);
+        codeParts.push(`local ${operation.name}_results`);
+        codeParts.push('do');
 
         if (operation.inputs.length) {
-            codeParts.push(operation.inputs.join('\n'));
+            body.push(operation.inputs.join('\n'));
         }
 
         if (operation.refs.length) {
-            codeParts.push(operation.refs.join('\n'));
+            body.push(operation.refs.join('\n'));
         }
 
-        codeParts.push(operation.code);
+        body.push(operation.code);
+
+        codeParts.push(indent(body.join('\n')));
+        codeParts.push('end');
         codeParts.push('');
+
         operation.code = codeParts.join('\n');
         return operation;
     };
 
     GenerateExecFile.prototype.assignResultToVar = function (code, name) {
         var i = code.lastIndexOf('return');
+
         return code.substring(0, i) +
             code.substring(i)
-                .replace('return', `local ${name} = `);
+                .replace('return', `${name} = `);
     };
 
     _.extend(GenerateExecFile.prototype, PtrCodeGen.prototype);

src/plugins/GenerateExecFile/deepforge.ejs

@@ -0,0 +1,47 @@
+-- Instantiate the deepforge object
+deepforge = {}
+
+function deepforge.initialize()
+    require 'nn'
+    require 'rnn'
+    <%= initCode %>
+end
+
+-- Graph support
+torch.class('deepforge.Graph')
+
+function deepforge.Graph:__init(name)
+    -- nop
+end
+
+torch.class('deepforge._Line')
+
+function deepforge._Line:__init(graphId, name, opts)
+   -- nop
+end
+
+function deepforge._Line:add(x, y)
+   -- nop
+end
+
+function deepforge.Graph:line(name, opts)
+    return deepforge._Line(self.id, name, opts)
+end
+
+-- Image support
+function deepforge.image(name, tensor)
+   -- nop
+end
+
+torch.class('deepforge.Image')
+function deepforge.Image:__init(name, tensor)
+   -- nop
+end
+
+function deepforge.Image:update(tensor)
+   -- nop
+end
+
+function deepforge.Image:title(name)
+   -- nop
+end

src/plugins/GenerateExecFile/toboolean.lua

@@ -0,0 +1,7 @@
+local function toboolean(str)
+    if str == 'true' then
+        return true
+    elseif str == 'false' then
+        return false
+    end
+end

src/plugins/ImportArtifact/ImportArtifact.js

@@ -86,6 +86,7 @@ define([
                 });
 
                 this.core.setAttribute(dataNode, 'data', hash);
+                this.core.setAttribute(dataNode, 'createdAt', Date.now());
                 baseName = this.core.getAttribute(baseType, 'name');
 
                 var getName;

src/plugins/ImportTorch/nn.js

@@ -3,10 +3,12 @@
 define([
     'deepforge/layer-args',
     'common/util/assert',
+    'deepforge/Constants',
     'deepforge/lua'
 ], function(
     createLayerDict,
     assert,
+    Constants,
     lua
 ) {
     'use strict';
@@ -41,6 +43,20 @@ define([
             return '{' + strings.join(', ') + '}';
         };
 
+        var getAttributeString = function(value, layerType) {
+            if (value instanceof lua.types.LuaTable) {
+                if (value.get('_node')) {
+                    throw Error(`Detected unsupported varargs (composed of layers) for ${layerType}`);
+                }
+                return stringify(value);
+            } else if ((typeof value) === 'object') {
+                // special lua.js object
+                value = value.valueOf();
+            }
+
+            return value;
+        };
+
         var allConnectedTo = function(current) {
             var connectedIds = {},
                 node,
@@ -79,6 +95,7 @@ define([
                 connsFrom[id] = [];
             }
             connsFrom[id].push(conn, dst);
+            return conn;
         };
 
         // nn drawing library
@@ -104,7 +121,8 @@ define([
                 cntr,
                 layer,
                 cntrName,
-                value;
+                value,
+                i;
 
             if (this._cachedNode) {
                 // only generate a single node for each layer
@@ -117,7 +135,17 @@ define([
                 parent: parent
             });
 
-            for (var i = this._attrs.length; i--;) {
+            // merge all the last arguments into a single one (ie, assume the last
+            // attribute is varargs
+            if (this._attrs.length < this._values.length) {
+                i = this._attrs.length;
+                value = this._values.splice(i-1)
+                    .map(val => getAttributeString(val, this._base)).join(', ');
+                this._values.push(value);
+            }
+
+            // Add the attributes to the layer
+            for (i = this._attrs.length; i--;) {
                 name = this._attrs[i].name;
                 value = this._values[i];
 
@@ -169,56 +197,131 @@ define([
             return self;
         };
 
-        // Each container will have `inputs` and `outputs`
+        Layer.prototype._getAllNodes = function() {
+            return [this._node()];
+        };
+
         var Container = function() {
+            Layer.apply(this, arguments);
+            this._nestedIndex = 0;
+        };
+
+        Container.prototype = Object.create(Layer.prototype);
+
+        Container.prototype.add = function(self, tlayer) {
+            var layer = tlayer.get('_node'),
+                container = this._node(),
+                children,
+                arch;
+
+            // Add a nested 'Architecture' node
+            arch = core.createNode({
+                parent: container,
+                base: META.Architecture
+            });
+
+            // Add this node to the 'addLayers' set
+            core.addMember(container, Constants.CONTAINED_LAYER_SET, arch);
+            // Assign it an appropriate 'index' value
+            core.setMemberRegistry(
+                container,
+                Constants.CONTAINED_LAYER_SET,
+                core.getPath(arch),
+                Constants.CONTAINED_LAYER_INDEX,
+                this._nestedIndex++
+            );
+
+            // Move the added node(s)/conns to this architecture node
+            children = layer._getAllNodes();
+            for (var i = children.length; i--;) {
+                core.moveNode(children[i], arch);
+            }
+            layer._parent = arch;
+            return self;
+        };
+
+        // Implicit Containers are sequential and concat containers;
+        // these containers are visually implied in deepforge (although
+        // they are explicitly defined in torch)
+        var ImplicitContainer = function() {
             // inputs and outputs are webgme nodes
             this._inputs = [];
             this._outputs = [];
+            this._children = [];
+            this._connections = [];
         };
 
-        Container.prototype.add = function() {
+        // Implicit containers will have to record their 'children'.
+        // When an implicit container is added to an actual container,
+        // the container will set it's '_parent' value. If any additional
+        // layers are added to the implicit container after, they will
+        // need to be moved to the parent of the implicit container
+        ImplicitContainer.prototype.add = function() {
             logger.error('Add is not overridden!');
         };
 
-        var Sequential = function(/*attrs, args*/) {
-            Container.call(this);
+        ImplicitContainer.prototype._getAllNodes = function() {
+            var nodes = this._children.map(layer => layer._getAllNodes())
+                .reduce((l1, l2) => l1.concat(l2), []);
+
+            return this._connections.concat(nodes);
         };
 
-        Sequential.prototype = new Container();
+        var Sequential = function(/*attrs, args*/) {
+            ImplicitContainer.call(this);
+        };
+
+        Sequential.prototype = new ImplicitContainer();
 
         Sequential.prototype.add = function(self, tlayer) {
             var layer = tlayer.get('_node'),
-                nodes = layer._inputs;
+                nodes = layer._inputs,
+                connections = [];
 
             // If this._inputs is empty, add the layer to the inputs list
             if (this._inputs.length === 0) {  // first node
                 this._inputs = this._inputs.concat(nodes);
             } else {
                 // connect all inputs of the added node to the current outputs
+                // add the connection to the list of allNodes
                 this._outputs.forEach(src =>
-                    nodes.forEach(dst => connect(src, dst))
+                    nodes.forEach(dst => connections.push(connect(src, dst)))
                 );
             }
             this._outputs = layer._outputs;
+            this._children.push(layer);
+            this._connections = this._connections.concat(connections);
+
+            // If _parent is set, move the nodes and connection to the _parent node
+            if (this._parent) {
+                nodes = layer._getAllNodes().concat(connections);
+                for (var i = nodes.length; i--;) {
+                    core.moveNode(nodes[i], this._parent);
+                }
+            }
             return self;
         };
 
         var Concat = function(attrs, args) {
-            Container.call(this);
+            ImplicitContainer.call(this);
 
             // Create a concat node and add it to this._outputs
             var concat = new Layer('Concat', attrs, args);
             this._outputs.push(concat._node());
+            this._children.push(concat);
         };
 
-        Concat.prototype = new Container();
+        Concat.prototype = new ImplicitContainer();
 
         Concat.prototype.add = function(self, tlayer) {
             // Connect the tlayer outputs to this._outputs
             var layer = tlayer.get('_node'),
-                concatLayer = this._outputs[0];
+                concatLayer = this._outputs[0],
+                connections = [],
+                nodes;
 
-            layer._outputs.forEach(output => connect(output, concatLayer));
+            layer._outputs.forEach(output =>
+                connections.push(connect(output, concatLayer)));
 
             // Connect the incomingly connected node to tlayer
             // TODO: This might not work if adding layers after this container is
@@ -226,14 +329,23 @@ define([
 
             // Add the layer's inputs to the inputs
             this._inputs = this._inputs.concat(layer._inputs);
+            this._children.push(layer);
+            this._connections = this._connections.concat(connections);
+            if (this._parent) {
+                nodes = layer._getAllNodes().concat(connections);
+                for (var i = nodes.length; i--;) {
+                    core.moveNode(nodes[i], this._parent);
+                }
+            }
             return self;
         };
 
         // Special layers (with special functions - like 'add')
-        var LAYERS = {
-            Concat: Concat,
-            Sequential: Sequential
-        };
+        var CONTAINERS,
+            LAYERS = {
+                Concat: Concat,
+                Sequential: Sequential
+            };
 
         var getValue = function(txt) {
             if (txt === 'true') {
@@ -286,6 +398,9 @@ define([
 
             if (LAYERS[type]) {
                 node = new LAYERS[type](args, attrs);
+            } else if (CONTAINERS[type]) {
+                node = new Container(type, args, attrs);
+                res.set('add', node.add.bind(node)); // add the 'add' method
             } else {  // Call generic Layer with type name
                 node = new Layer(type, args, attrs);
             }
@@ -327,11 +442,23 @@ define([
                 return;
             }
 
-            // TODO: Create the nn object
+            // Mocking the nn layers (as defined in the metamodel)
             var nn = lua.newContext()._G,
-                names = Object.keys(LayerDict);
+                names = Object.keys(LayerDict),
+                base,
+                baseName;
+
+            // For each layer, check the name of the base type. If it is 'Container',
+            // then it should be added to the CONTAINERS dictionary. This will change how
+            // it is handled in 'CreateLayer'
+            CONTAINERS = {};
 
             for (var i = names.length; i--;) {
+                base = core.getBase(META[names[i]]);
+                baseName = core.getAttribute(base, 'name');
+                if (baseName === 'Container') {
+                    CONTAINERS[names[i]] = true;
+                }
                 nn.set(names[i], CreateLayer.bind(null, names[i]));
             }
 

src/plugins/UpdateLibrarySeed/UpdateLibrarySeed.js

@@ -84,19 +84,27 @@ define([
             .fail(err => callback(err, this.result));
     };
 
+    UpdateLibrarySeed.prototype.bumpVersion = function (rawVersion, releaseType) {
+        var vnames = ['major', 'minor', 'patch'],
+            bumpIndex = vnames.indexOf(releaseType),
+            version = rawVersion.split('.').map(num => parseInt(num));
+
+        version[bumpIndex]++;
+        // zero out the smaller numbers
+        while (++bumpIndex < version.length) {
+            version[bumpIndex] = 0;
+        }
+        return version.join('.');
+    };
+
     UpdateLibrarySeed.prototype.getLibraryVersion = function () {
         var version,
             config = this.getCurrentConfig(),
-            vnames = ['major', 'minor', 'patch'],
-            bumpIndex,
             newVersion;
 
-        version = (this.core.getAttribute(this.rootNode, 'version') || '0.0.0')
-            .split('.').map(num => parseInt(num));
-        bumpIndex = vnames.indexOf(config.releaseType);
-        version[bumpIndex]++;
+        version = (this.core.getAttribute(this.rootNode, 'version') || '0.0.0');
+        newVersion = this.bumpVersion(version, config.releaseType);
 
-        newVersion = version.join('.');
         this.core.setAttribute(this.rootNode, 'version', newVersion);
         return this.save(`Bumped version to ${newVersion}`).then(() => newVersion);
     };
@@ -116,6 +124,10 @@ define([
         this.logger.info(`Updating ${seedName} seed`);
         job.on('error', _err => {
             err = _err;
+            if (err.code === 'ENOENT') {
+                return deferred.reject('"webgme" command not found. Is webgme-cli installed on the server?');
+            }
+            return deferred.reject(err);
         });
 
         job.on('exit', code => {

src/plugins/ValidateArchitecture/ValidateArchitecture.js

@@ -0,0 +1,194 @@
+/*globals define*/
+/*jshint node:true, browser:true*/
+
+define([
+    'plugin/GenerateArchitecture/GenerateArchitecture/GenerateArchitecture',
+    'SimpleNodes/Constants',
+    'text!./metadata.json',
+    'q',
+    'fs',
+    'path',
+    'child_process',
+    'rimraf'
+], function (
+    PluginBase,
+    SimpleNodeConstants,
+    pluginMetadata,
+    Q,
+    fs,
+    path,
+    childProcess,
+    rm_rf
+) {
+    'use strict';
+
+    pluginMetadata = JSON.parse(pluginMetadata);
+
+    /**
+     * Initializes a new instance of ValidateArchitecture.
+     * @class
+     * @augments {PluginBase}
+     * @classdesc This class represents the plugin ValidateArchitecture.
+     * @constructor
+     */
+    var TMP_DIR = '/tmp',
+        spawn = childProcess.spawn,
+        GET_ARG_INDEX = /argument #([0-9]+) to/,
+        TORCH_INSTALLED = true;
+
+    var ValidateArchitecture = function () {
+        // Call base class' constructor.
+        PluginBase.call(this);
+        this.pluginMetadata = pluginMetadata;
+    };
+
+    /**
+     * Metadata associated with the plugin. Contains id, name, version, description, icon, configStructue etc.
+     * This is also available at the instance at this.pluginMetadata.
+     * @type {object}
+     */
+    ValidateArchitecture.metadata = pluginMetadata;
+
+    // Prototypical inheritance from PluginBase.
+    ValidateArchitecture.prototype = Object.create(PluginBase.prototype);
+    ValidateArchitecture.prototype.constructor = ValidateArchitecture;
+
+    ValidateArchitecture.prototype.main = function (callback) {
+        var name = this.core.getAttribute(this.activeNode, 'name');
+
+        this._callback = callback;
+        // make the tmp dir
+        this._tmpFileId = path.join(TMP_DIR, `${name}_${Date.now()}`);
+        fs.mkdir(this._tmpFileId, err => {
+            if (err) throw err;
+            return PluginBase.prototype.main.call(this, callback);
+        });
+    };
+
+    ValidateArchitecture.prototype.createOutputFiles = function (tree) {
+        var layers = tree[SimpleNodeConstants.CHILDREN],
+            tests = [],
+            id;
+
+        if (!TORCH_INSTALLED) {
+            return this.validationFinished();
+        }
+
+        // Generate code for each layer
+        this.layerName = {};
+        for (var i = layers.length; i--;) {
+            id = layers[i][SimpleNodeConstants.NODE_PATH];
+            this.layerName[id] = layers[i].name;
+            tests.push([id, this.createLayerTestCode(layers[i])]);
+        }
+
+        // Run each code snippet
+        this.validateLayers(tests)
+            .then(errors => this.validationFinished(errors))
+            .fail(err => this.logger.error(`validation failed: ${err}`));
+    };
+
+    ValidateArchitecture.prototype.validationFinished = function (errors) {
+        if (!TORCH_INSTALLED) {
+            this.logger.warn('Torch is not installed. Architecture validation is not supported.');
+        } else {
+            this.logger.info(`found ${errors.length} validation errors`);
+        }
+
+        this.createMessage(null, {
+            errors: TORCH_INSTALLED ? errors : null
+        });
+        this.result.setSuccess(true);
+        this._callback(null, this.result);
+    };
+
+    ValidateArchitecture.prototype.createLayerTestCode = function (layer) {
+        var customLayerDefs = this.genLayerDefinitions([layer]);
+
+        return this.definitions.concat([
+            customLayerDefs,
+            this.createLayer(layer)
+        ]).join('\n');
+    };
+
+    ValidateArchitecture.prototype.validateLayers = function (layerTests) {
+        return Q.all(layerTests.map(layer => this.validateLayer(layer[0], layer[1])))
+            .then(results => Q.nfcall(rm_rf, this._tmpFileId)
+                .then(() => results.filter(result => !!result))
+            );
+    };
+
+    ValidateArchitecture.prototype.validateLayer = function (id, code) {
+        var deferred = Q.defer(),
+            tmpPath = path.join(this._tmpFileId, id.replace(/[^a-zA-Z\d]+/g, '_'));
+
+        if (!TORCH_INSTALLED) {
+            deferred.resolve(null);
+        } else {
+            // Write to a temp file
+            fs.writeFile(tmpPath, code, err => {
+                var job,
+                    stderr = '',
+                    stdout = '';
+
+                if (err) {
+                    return deferred.reject(`Could not create tmp file at ${tmpPath}: ${err}`);
+                }
+                // Run the file
+                job = spawn('th', [tmpPath]);
+                job.stderr.on('data', data => stderr += data.toString());
+                job.stdout.on('data', data => stdout += data.toString());
+                job.on('error', err => {
+                    if (err.code === 'ENOENT') {
+                        TORCH_INSTALLED = false;
+                    }
+                });
+                job.on('close', code => {
+                    if (code === 0) {
+                        deferred.resolve(null);
+                    } else {
+                        // If it errored, clean the error and return it
+                        deferred.resolve(this.parseError(id, stderr));
+                    }
+                });
+            });
+        }
+
+        return deferred.promise;
+    };
+
+    ValidateArchitecture.prototype.parseError = function (id, stderr) {
+        var msg = stderr
+            .split('\n').shift()  // first line
+            .replace(/^[^:]*: /, '')  // remove the file path
+            .replace(/ at [^ ]*\)/, ')')  // remove last line number
+            .replace(/ to '\?'/, '');  // remove unknown symbol
+
+        // convert 'bad argument #[num]' to the argument name
+        if (msg.indexOf('bad argument') === 0) {
+            var layerName = this.layerName[id],
+                args = this.LayerDict[layerName].args,
+                argIndex = +(stderr.match(GET_ARG_INDEX)[1]),
+                argName = args[argIndex-1].name;
+
+            // FIXME: This is not the correct index...
+            // This is the index for the incorrect argument passed to the
+            // tensor...
+            msg = msg.replace(`#${argIndex}`, `"${argName}"`);
+        }
+
+        return {
+            id: id,
+            msg: msg
+        };
+    };
+
+    ValidateArchitecture.prototype._saveOutput = function () {};
+
+    // for testing
+    ValidateArchitecture.prototype.setTorchInstalled = function (value) {
+        TORCH_INSTALLED = !!value;
+    };
+
+    return ValidateArchitecture;
+});

src/plugins/ValidateArchitecture/metadata.json

@@ -0,0 +1,14 @@
+{
+  "id": "ValidateArchitecture",
+  "name": "ValidateArchitecture",
+  "version": "0.1.0",
+  "description": "",
+  "icon": {
+    "class": "glyphicon glyphicon-cog",
+    "src": ""
+  },
+  "disableServerSideExecution": false,
+  "disableBrowserSideExecution": true,
+  "writeAccessRequired": false,
+  "configStructure": []
+}

src/routers/ExecPulse/ExecPulse.js

@@ -0,0 +1,131 @@
+/*jshint node:true*/
+
+// This is a REST endpoint keeping track of the heartbeats of each execution. This
+// allows detection of "disconnected" executions (enabling the reconnection of the
+// executions - issue #821)
+'use strict';
+
+var express = require('express'),
+    MONGO_COLLECTION = 'ExecPulse',
+    CONSTANTS = require('../../common/Constants').PULSE,
+    mongo,
+    storage,
+    router = express.Router();
+
+/**
+ * Called when the server is created but before it starts to listening to incoming requests.
+ * N.B. gmeAuth, safeStorage and workerManager are not ready to use until the start function is called.
+ * (However inside an incoming request they are all ensured to have been initialized.)
+ *
+ * @param {object} middlewareOpts - Passed by the webgme server.
+ * @param {GmeConfig} middlewareOpts.gmeConfig - GME config parameters.
+ * @param {GmeLogger} middlewareOpts.logger - logger
+ * @param {function} middlewareOpts.ensureAuthenticated - Ensures the user is authenticated.
+ * @param {function} middlewareOpts.getUserId - If authenticated retrieves the userId from the request.
+ * @param {object} middlewareOpts.gmeAuth - Authorization module.
+ * @param {object} middlewareOpts.safeStorage - Accesses the storage and emits events (PROJECT_CREATED, COMMIT..).
+ * @param {object} middlewareOpts.workerManager - Spawns and keeps track of "worker" sub-processes.
+ */
+function initialize(middlewareOpts) {
+    var logger = middlewareOpts.logger.fork('ExecPulse'),
+        ensureAuthenticated = middlewareOpts.ensureAuthenticated,
+        STALE_THRESHOLD = 7500;
+
+    storage = require('../storage')(logger, middlewareOpts.gmeConfig);
+    logger.debug('initializing ...');
+
+    // Ensure authenticated can be used only after this rule.
+    router.use('*', function (req, res, next) {
+        res.setHeader('X-WebGME-Media-Type', 'webgme.v1');
+        next();
+    });
+
+    // Use ensureAuthenticated if the routes require authentication. (Can be set explicitly for each route.)
+    router.use('*', ensureAuthenticated);
+
+    router.get('/', function (req, res) {
+        mongo.find().toArray((err, all) => {
+            if (err) {
+                return res.status(500).send(err);
+            }
+            res.json(all.map(entry => {
+                delete entry._id;
+                return entry;
+            }));
+        });
+    });
+
+    router.get('/:hash', function (req, res) {
+        // Check if the given job is alive (has a valid heartbeat).
+        // If the data doesn't exist, then it is considered alive
+        if (!req.params.hash) {
+            return res.status(400).send('Missing hash');
+        }
+
+        logger.debug('getting pulse of ', req.params.hash);
+        mongo.findOne({hash: req.params.hash})
+            .then(job => {
+                var current = Date.now(),
+                    result = CONSTANTS.DOESNT_EXIST;
+
+                if (job) {
+                    result = (current - job.timestamp) < STALE_THRESHOLD ?
+                        CONSTANTS.ALIVE : CONSTANTS.DEAD;
+                }
+                return res.status(200).send(result.toString());
+            });
+    });
+
+    router.post('/:hash', function (req, res) {
+        var timestamp = Date.now(),
+            job = {
+                hash: req.params.hash,
+                timestamp: timestamp
+            };
+
+        // Validate the input
+        logger.debug('Received heartbeat for ', job.hash);
+        if (!job.hash) {
+            return res.status(400).send('Missing hash');
+        }
+
+        // Delete the given job from the database
+        mongo.update({hash: job.hash}, job, {upsert: true})
+            .then(() => res.sendStatus(201));
+    });
+
+    router.delete('/:hash', function (req, res) {
+        // Delete the given job from the database
+        return mongo.findOneAndDelete({hash: req.params.hash})
+            .then(() => res.sendStatus(204));
+    });
+
+    logger.debug('ready');
+}
+
+/**
+ * Called before the server starts listening.
+ * @param {function} callback
+ */
+function start(callback) {
+    storage.then(db => {
+        mongo = db.collection(MONGO_COLLECTION);
+        callback();
+    });
+}
+
+/**
+ * Called after the server stopped listening.
+ * @param {function} callback
+ */
+function stop(callback) {
+    callback();
+}
+
+
+module.exports = {
+    initialize: initialize,
+    router: router,
+    start: start,
+    stop: stop
+};

src/routers/JobLogsAPI/JobLogManager.js

@@ -71,7 +71,7 @@ JobLogManager.prototype.migrate = function(migrationInfo, jobIds) {
     }
 
     // Copy the job files and evaluate each of the finish functions
-    this.logger.info('migrating from ' + migrationInfo.srcBranch + ' to '+ migrationInfo.dstBranch);
+    this.logger.debug('migrating from ' + migrationInfo.srcBranch + ' to '+ migrationInfo.dstBranch);
     return Q.all(jobIds.map(jobId => {
         src = this._getFilePath({
             project: migrationInfo.project,
@@ -112,7 +112,7 @@ JobLogManager.prototype.appendTo = function(jobInfo, logs) {
         branchDirname = path.dirname(filename),
         projDirname = path.dirname(branchDirname);
 
-    this.logger.info(`Appending content to ${filename}`);
+    this.logger.debug(`Appending content to ${filename}`);
     // Make directory if needed
     return this.mkdirIfNeeded(this.rootDir)
         .then(() => this.mkdirIfNeeded(projDirname))
@@ -142,7 +142,7 @@ JobLogManager.prototype.delete = function(jobInfo) {
                 this.logger.debug(`Removing file ${filename}`);
                 return Q.nfcall(fs.unlink, filename);
             }
-            this.logger.info(`${filename} doesn't exist. No need to delete...`);
+            this.logger.debug(`${filename} doesn't exist. No need to delete...`);
         });
 };
 

src/routers/JobLogsAPI/JobLogsAPI.js

@@ -4,7 +4,10 @@
 
 var express = require('express'),
     JobLogManager = require('./JobLogManager'),
-    router = express.Router();
+    MONGO_COLLECTION = 'JobLogsMetadata',
+    mongo,
+    router = express.Router(),
+    storage;
 
 /**
  * Called when the server is created but before it starts to listening to incoming requests.
@@ -27,6 +30,7 @@ function initialize(middlewareOpts) {
         logManager = new JobLogManager(logger, gmeConfig);
 
     logger.debug('initializing ...');
+    storage = require('../storage')(logger, gmeConfig);
 
     // Ensure authenticated can be used only after this rule.
     router.use('*', function (req, res, next) {
@@ -40,29 +44,72 @@ function initialize(middlewareOpts) {
 
     router.get('/:project/:branch/:job', function (req, res/*, next*/) {
         // Retrieve the job logs for the given job
-        logManager.getLog(req.params).then(log => {
-            res.set('Content-Type', 'text/plain');
-            res.send(log);
-        });
+        logger.info(`Requested logs for ${req.params.job} in ${req.params.project}`);
+        logManager.getLog(req.params)
+            .then(log => {
+                res.set('Content-Type', 'text/plain');
+                res.send(log);
+            })
+            .catch(err => logger.error(`Log retrieval failed: ${err}`));
+    });
+
+    router.get('/metadata/:project/:branch/:job', function (req, res/*, next*/) {
+        logger.info(`Requested metadata for ${req.params.job} in ${req.params.project}`);
+        return mongo.findOne(req.params)
+            .then(info => {
+                var lineCount = info ? info.lineCount : -1;
+
+                return res.json({
+                    lineCount: lineCount
+                });
+            })
+            .catch(err => logger.error(`Metadata retrieval failed: ${err}`));
     });
 
     router.patch('/:project/:branch/:job', function (req, res/*, next*/) {
         var logs = req.body.patch;
         logger.info(`Received append request for ${req.params.job} in ${req.params.project}`);
-        logManager.appendTo(req.params, logs)
-            .then(() => res.send('Append successful'))
+        return logManager.appendTo(req.params, logs)
+            .then(() => {
+                if (req.body.lineCount || req.body.cmdCount || req.body.createdIds) {
+                    var info = {
+                        project: req.params.project,
+                        branch: req.params.branch,
+                        job: req.params.job,
+                        lineCount: req.body.lineCount || -1,
+                        createdIds: req.body.createdIds || [],
+                        cmdCount: req.body.cmdCount || 0
+                    };
+                    logger.debug('metadata is', info);
+                    return mongo.update(req.params, info, {upsert: true})
+                        .then(() => res.send('Append successful'));
+                } else {
+                    res.send('Append successful');
+                }
+            })
             .catch(err => logger.error(`Append failed: ${err}`));
     });
 
     router.delete('/:project/:branch/:job', function (req, res/*, next*/) {
-        logManager.delete(req.params).then(() => res.send('delete successful'));
+        logger.info(`Request to delete logs for ${req.params.job} in ${req.params.project}`);
+        logManager.delete(req.params)
+            .then(() => mongo.findOneAndDelete(req.params))
+            .then(() => {
+                logger.info('Job log deletion successful!');
+                res.status(204).send('delete successful');
+            })
+            .catch(err => logger.error(`Job log deletion failed: ${err}`));
     });
 
     router.post('/migrate/:project/:srcBranch/:dstBranch', function (req, res/*, next*/) {
         var jobs = req.body.jobs;
+        logger.info(`Migrating logs from ${req.params.srcBranch} to ${req.params.dstBranch} in ${req.params.project}`);
         logManager.migrate(req.params, jobs)
-            .then(() => res.send('migration successful'))
-            .fail(err => logger.error(err));
+            .then(() => {
+                logger.info('Log migration successful!');
+                res.send('migration successful');
+            })
+            .fail(err => logger.error(`migration failed: ${err}`));
     });
 
     logger.debug('ready');
@@ -73,7 +120,11 @@ function initialize(middlewareOpts) {
  * @param {function} callback
  */
 function start(callback) {
-    callback();
+    storage.then(db => {
+        mongo = db.collection(MONGO_COLLECTION);
+        callback();
+    });
+
 }
 
 /**

src/routers/JobOriginAPI/JobOriginAPI.js

@@ -0,0 +1,162 @@
+/*jshint node:true*/
+
+'use strict';
+
+var express = require('express'),
+    MONGO_COLLECTION = 'JobOrigins',
+    utils = require('../utils'),
+    mongo,
+    router = express.Router(),
+    storage;
+
+/**
+ * Called when the server is created but before it starts to listening to incoming requests.
+ * N.B. gmeAuth, safeStorage and workerManager are not ready to use until the start function is called.
+ * (However inside an incoming request they are all ensured to have been initialized.)
+ *
+ * @param {object} middlewareOpts - Passed by the webgme server.
+ * @param {GmeConfig} middlewareOpts.gmeConfig - GME config parameters.
+ * @param {GmeLogger} middlewareOpts.logger - logger
+ * @param {function} middlewareOpts.ensureAuthenticated - Ensures the user is authenticated.
+ * @param {function} middlewareOpts.getUserId - If authenticated retrieves the userId from the request.
+ * @param {object} middlewareOpts.gmeAuth - Authorization module.
+ * @param {object} middlewareOpts.safeStorage - Accesses the storage and emits events (PROJECT_CREATED, COMMIT..).
+ * @param {object} middlewareOpts.workerManager - Spawns and keeps track of "worker" sub-processes.
+ */
+// When testing, use in memory storage...
+function initialize(middlewareOpts) {
+    var logger = middlewareOpts.logger.fork('JobOriginAPI'),
+        gmeConfig = middlewareOpts.gmeConfig,
+        ensureAuthenticated = middlewareOpts.ensureAuthenticated,
+        REQUIRED_FIELDS = ['hash', 'project', 'execution', 'job', 'nodeId', 'branch'];
+
+    storage = require('../storage')(logger, gmeConfig);
+
+    logger.debug('initializing ...');
+    // Ensure authenticated can be used only after this rule.
+    router.use('*', function (req, res, next) {
+        // This header ensures that any failures with authentication won't redirect.
+        res.setHeader('X-WebGME-Media-Type', 'webgme.v1');
+        next();
+    });
+
+    // Use ensureAuthenticated if the routes require authentication. (Can be set explicitly for each route.)
+    router.use('*', ensureAuthenticated);
+
+    // Connect to mongo...
+
+    router.get('/', function (req, res) {
+        mongo.find().toArray((err, all) => {
+            if (err) {
+                return res.status(500).send(err);
+            }
+            res.json(all.map(entry => {
+                delete entry._id;
+                return entry;
+            }));
+        });
+    });
+
+    router.get('/:jobHash', function (req, res/*, next*/) {
+        var hash = req.params.jobHash,
+            jobInfo = {};
+
+        mongo.findOne({hash: hash})
+            .then(result => {
+                if (result) {
+                    // Filter the result object
+                    for (var i = REQUIRED_FIELDS.length; i--;) {
+                        jobInfo[REQUIRED_FIELDS[i]] = result[REQUIRED_FIELDS[i]];
+                    }
+                    return res.json(jobInfo);
+                }
+                res.sendStatus(404);
+            })
+            .catch(err => {
+                logger.error(`Storing job info failed: ${err}`);
+                res.status(500).send(err);
+            });
+    });
+
+    router.post('/:jobHash', function (req, res/*, next*/) {
+        var hash = req.params.jobHash,
+            jobInfo = {
+                hash: hash,
+                project: req.body.project,
+                execution: req.body.execution,
+                branch: req.body.branch,
+                job: req.body.job,  // job name
+                nodeId: req.body.nodeId
+            };
+
+        // Check that none of the fields are undefined
+        var missing = utils.getMissingField(jobInfo, REQUIRED_FIELDS);
+        if (missing) {
+            return res.status(400).send(`Missing required field: ${missing}`);
+        }
+
+        logger.debug(`Storing job info for ${hash}`);
+        return mongo.insertOne(jobInfo)
+            .then(() => res.sendStatus(201))
+            .catch(err => {
+                logger.error(`Storing job info failed: ${err}`);
+                res.status(500).send(err.toString());
+            });
+    });
+
+    router.delete('/:jobHash', function (req, res/*, next*/) {
+        var hash = req.params.jobHash;
+
+        mongo.findOneAndDelete({hash: hash})
+            .then(() => res.sendStatus(204));
+    });
+
+    // on fork
+    router.patch('/:jobHash', function (req, res) {
+        var hash = req.params.jobHash;
+
+        if (!req.body.branch) {
+            return res.status(400).send('Missing "branch" field');
+        }
+
+        return mongo.findOneAndUpdate({hash: hash}, {$set: {branch: req.body.branch}})
+            .then(() => {
+                logger.debug('Finished updateOne!');
+                res.sendStatus(200);
+            })
+            .catch(err => {
+                logger.error(`Job update failed: ${err}`);
+                res.status(500).send(err);
+            });
+    });
+
+    logger.debug('ready');
+}
+
+/**
+ * Called before the server starts listening.
+ * @param {function} callback
+ */
+function start(callback) {
+    storage.then(db => {
+        mongo = db.collection(MONGO_COLLECTION);
+        callback();
+    });
+
+}
+
+/**
+ * Called after the server stopped listening.
+ * @param {function} callback
+ */
+function stop(callback) {
+    callback();
+}
+
+
+module.exports = {
+    initialize: initialize,
+    router: router,
+    start: start,
+    stop: stop
+};

src/routers/storage.js

@@ -0,0 +1,19 @@
+// Get a mongodb connection
+
+var mongodb = require('mongodb'),
+    connection;
+
+module.exports = function(logger, gmeConfig) {
+    if (!connection) {
+        connection = mongodb.MongoClient.connect(gmeConfig.mongo.uri, gmeConfig.mongo.options)
+            .then(db => {
+                logger.debug('Connected to mongo!');
+                return db;
+            })
+            .catch(err => {
+                logger.error(`Could not connect to mongo: ${err}`);
+                throw err;
+            });
+    }
+    return connection;
+};

src/routers/utils.js

@@ -0,0 +1,10 @@
+module.exports = {
+    getMissingField: function(array, fields) {
+        for (var i = fields.length; i--;) {
+            if (!array[fields[i]]) {
+                return fields[i];
+            }
+        }
+        return null;
+    }
+};

src/seeds/nn/version.txt

@@ -1 +1 @@
-0.2.0
+0.5.0

src/seeds/pipeline/version.txt

@@ -1 +1 @@
-0.3.0
+0.4.1

src/visualizers/Visualizers.json

@@ -31,8 +31,8 @@
   },
   {
     "id": "RootViz",
-    "title": "MainView",
-    "panel": "panels/MainView/MainViewPanel",
+    "title": "ForwardViz",
+    "panel": "panels/ForwardViz/ForwardVizPanel",
     "DEBUG_ONLY": false
   },
   {
@@ -71,6 +71,30 @@
     "panel": "panels/PipelineIndex/PipelineIndexPanel",
     "DEBUG_ONLY": false
   },
+  {
+    "id": "ArchitectureIndex",
+    "title": "ArchitectureIndex",
+    "panel": "panels/ArchIndex/ArchIndexPanel",
+    "DEBUG_ONLY": false
+  },
+  {
+    "id": "OperationIndex",
+    "title": "OperationIndex",
+    "panel": "js/Panels/ModelEditor/ModelEditorPanel",
+    "DEBUG_ONLY": false
+  },
+  {
+    "id": "LayerIndex",
+    "title": "LayerIndex",
+    "panel": "js/Panels/ModelEditor/ModelEditorPanel",
+    "DEBUG_ONLY": false
+  },
+  {
+    "id": "DataTypeIndex",
+    "title": "DataTypeIndex",
+    "panel": "js/Panels/ModelEditor/ModelEditorPanel",
+    "DEBUG_ONLY": false
+  },
   {
     "id": "JobEditor",
     "title": "JobEditor",
@@ -100,5 +124,29 @@
     "title": "ExecutionIndex",
     "panel": "panels/ExecutionIndex/ExecutionIndexPanel",
     "DEBUG_ONLY": false
+  },
+  {
+    "id": "WorkerHeader",
+    "title": "WorkerHeader",
+    "panel": "panels/WorkerHeader/WorkerHeaderPanel",
+    "DEBUG_ONLY": false
+  },
+  {
+    "id": "ArtifactIndex",
+    "title": "ArtifactIndex",
+    "panel": "panels/ArtifactIndex/ArtifactIndexPanel",
+    "DEBUG_ONLY": false
+  },
+  {
+    "id": "ArchIndex",
+    "title": "ArchIndex",
+    "panel": "panels/ArchIndex/ArchIndexPanel",
+    "DEBUG_ONLY": false
+  },
+  {
+    "id": "ForwardViz",
+    "title": "ForwardViz",
+    "panel": "panels/ForwardViz/ForwardVizPanel",
+    "DEBUG_ONLY": false
   }
-]
+]

src/visualizers/panels/ArchEditor/ArchEditorControl.js

@@ -4,27 +4,30 @@
 define([
     'deepforge/Constants',
     'deepforge/globals',
-    'panels/EasyDAG/EasyDAGControl',
+    'deepforge/viz/panels/ThumbnailControl',
     'js/NodePropertyNames',
     'js/Utils/ComponentSettings',
-    'underscore'
+    'underscore',
+    'q'
 ], function (
     Constants,
     DeepForge,
-    EasyDAGControl,
+    ThumbnailControl,
     nodePropertyNames,
     ComponentSettings,
-    _
+    _,
+    Q
 ) {
 
     'use strict';
 
     var ArchEditorControl,
         DEFAULT_CONFIG = {
-            DefaultColor: '#ffb74d',
+            DefaultColor: '#80cbc4',
             LayerColors: {
-                Containers: '#ffb74d',
-                Convolution: '#2196f3',
+                Container: '#ffb74d',
+                NestedContainer: '#ffe0b2',
+                Convolution: '#42a5f5',
                 Simple: '#ff9100',
                 Transfer: '#80deea',
                 Misc: '#ce93d8'
@@ -32,12 +35,13 @@ define([
         };
 
     ArchEditorControl = function (options) {
-        EasyDAGControl.call(this, options);
+        ThumbnailControl.call(this, options);
         this._config = DEFAULT_CONFIG;
         ComponentSettings.resolveWithWebGMEGlobal(this._config, this.getComponentId());
+        this.validateLayers = _.debounce(() => this.validateArchitecture(), 500);
     };
 
-    _.extend(ArchEditorControl.prototype, EasyDAGControl.prototype);
+    _.extend(ArchEditorControl.prototype, ThumbnailControl.prototype);
 
     ArchEditorControl.prototype.TERRITORY_RULE = {children: 1};
     ArchEditorControl.prototype.DEFAULT_DECORATOR = 'LayerDecorator';
@@ -46,7 +50,9 @@ define([
     };
 
     ArchEditorControl.prototype.selectedObjectChanged = function(id) {
-        EasyDAGControl.prototype.selectedObjectChanged.call(this, id);
+        this.nestedLevel = typeof id === 'string' ?
+            Math.floor(id.split('/').length/2) % 2 : 0;
+        ThumbnailControl.prototype.selectedObjectChanged.call(this, id);
 
         DeepForge.last.Architecture = id;
         if (typeof id === 'string') {
@@ -56,7 +62,8 @@ define([
     };
 
     ArchEditorControl.prototype._getObjectDescriptor = function(id) {
-        var desc = EasyDAGControl.prototype._getObjectDescriptor.call(this, id);
+        var node = this._client.getNode(id),
+            desc = ThumbnailControl.prototype._getObjectDescriptor.call(this, id);
 
         // Filter attributes
         if (!desc.isConnection) {
@@ -78,7 +85,7 @@ define([
 
             for (i = names.length; i--;) {
                 // check if it is a setter
-                schema = this._client.getAttributeSchema(id, names[i]);
+                schema = node.getAttributeMeta(names[i]);
                 if (names[i] === 'name' || schema.setterType) {
                     desc.attributes[names[i]] = allAttrs[names[i]];
                 }
@@ -87,8 +94,7 @@ define([
             // Add layer type (base class's base class)
             desc.layerType = null;
             if (desc.baseName) {
-                var node = this._client.getNode(id),
-                    base = this._client.getNode(node.getMetaTypeId()),
+                var base = this._client.getNode(node.getMetaTypeId()),
                     layerType = this._client.getNode(base.getBaseId()),
                     color;
 
@@ -97,11 +103,32 @@ define([
                     desc.layerType = layerType.getAttribute(nodePropertyNames.Attributes.name);
 
                     color = this._config.LayerColors[desc.layerType];
+                    if (desc.layerType === 'Container' && this.nestedLevel) {
+                        color = this._config.LayerColors.NestedContainer;
+                    }
                     if (!color) {
                         this._logger.warn(`No color found for ${desc.layerType}`);
                         color = this._config.DefaultColor;
                     }
                     desc.color = color;
+
+                    if (desc.layerType === 'Container') {
+                        desc.containedLayers = node.getMemberIds(Constants.CONTAINED_LAYER_SET)
+                            .map(layerId => {
+                                var index = node.getMemberRegistry(
+                                    Constants.CONTAINED_LAYER_SET,
+                                    layerId,
+                                    Constants.CONTAINED_LAYER_INDEX
+                                );
+                                return [layerId, index];
+                            })
+                            .sort((a, b) => a[1] < b[1] ? -1 : 1)
+                            .map(tuple => tuple[0]);
+
+                        // Set the decorator to ContainerLayerDecorator
+                        desc.Decorator = this._client.decoratorManager
+                            .getDecoratorForWidget('ContainerLayerDecorator', 'EasyDAG');
+                    }
                 }
             }
         }
@@ -109,30 +136,19 @@ define([
     };
 
     ////////////////////////// Layer Selection Logic //////////////////////////
-    ArchEditorControl.prototype._getValidInitialNodes = function() {
-        return this._client.getChildrenMeta(this._currentNodeId).items
-            // For now, anything is possible!
-            // FIXME
-            .map(info => this._getAllDescendentIds(info.id))
-            .reduce((prev, curr) => prev.concat(curr))
-            // Filter all abstract nodes
-            .filter(nodeId => {
-                return !this._client.getNode(nodeId).isAbstract();
-            })
-            .map(id => this._getObjectDescriptor(id))
-            .filter(obj => !obj.isConnection && obj.name !== 'Connection')
-            .filter(layer => layer.layerType !== 'Criterion');
-    };
-
-    ArchEditorControl.prototype._getValidSuccessorNodes =
+    ArchEditorControl.prototype.getValidSuccessors =
     ArchEditorControl.prototype._getValidInitialNodes =
     ArchEditorControl.prototype.getNonCriterionLayers = function() {
         // Return all (non-criterion) layer types
         var metanodes = this._client.getAllMetaNodes(),
             layerId,
+            connId,
+            conn,
             criterionId,
-            allLayerIds = [],
+            allLayers = [],
             layers = [],
+            tgts,
+            j,
             i;
 
         for (i = metanodes.length; i--;) {
@@ -142,24 +158,40 @@ define([
             }
         }
 
+        // Remove all criterion layers and abstract layers
         for (i = metanodes.length; i--;) {
             if (layerId) {
-                if (!metanodes[i].isAbstract() &&
-                    this._client.isTypeOf(metanodes[i].getId(), layerId)) {
+                if (!metanodes[i].isAbstract() && metanodes[i].isTypeOf(layerId)) {
 
                     if (metanodes[i].getAttribute('name') === 'Criterion') {
                         criterionId = metanodes[i].getId();
                     } else {
-                        allLayerIds.push(metanodes[i].getId());
+                        allLayers.push(metanodes[i]);
+                    }
+                } else if (!connId && metanodes[i].getAttribute('name') === 'Connection') {  // Detect the layer connection type...
+                    tgts = this._client.getPointerMeta(metanodes[i].getId(), 'src').items;
+                    for (j = tgts.length; j--;) {
+                        if (tgts[j].id === layerId) {
+                            connId = metanodes[i].getId();
+                        }
                     }
                 }
             }
         }
 
+        if (!connId) {
+            this._logger.warn('Could not find a layer connector');
+            return [];
+        }
+        // Convert the layers into the correct format
+        conn = this._getObjectDescriptor(connId);
         // Remove all criterion layers and abstract layers
-        for (i = allLayerIds.length; i--;) {
-            if (!this._client.isTypeOf(allLayerIds[i], criterionId)) {
-                layers.push({node: this._getObjectDescriptor(allLayerIds[i])});
+        for (i = allLayers.length; i--;) {
+            if (!allLayers[i].isTypeOf(criterionId)) {
+                layers.push({
+                    node: this._getObjectDescriptor(allLayers[i].getId()),
+                    conn: conn
+                });
             }
         }
 
@@ -172,8 +204,9 @@ define([
 
     // Widget extensions
     ArchEditorControl.prototype._initWidgetEventHandlers = function() {
-        EasyDAGControl.prototype._initWidgetEventHandlers.call(this);
+        ThumbnailControl.prototype._initWidgetEventHandlers.call(this);
         this._widget.getCreateNewDecorator = this.getCreateNewDecorator.bind(this);
+        this._widget.insertLayer = this.insertLayer.bind(this);
     };
 
     ArchEditorControl.prototype.getCreateNewDecorator = function() {
@@ -183,5 +216,63 @@ define([
         );
     };
 
+    ArchEditorControl.prototype.insertLayer = function(layerBaseId, connId) {
+        var conn = this._client.getNode(connId),
+            parentId = conn.getParentId(),
+            layerId,
+            nextLayerId = conn.getPointer('dst').to,
+            connBaseId = conn.getBaseId(),
+            newConnId,
+
+            baseName = this._client.getNode(layerBaseId).getAttribute('name'),
+            prevLayerId = conn.getPointer('src').to,
+            srcName = this._client.getNode(prevLayerId).getAttribute('name'),
+            dstName = this._client.getNode(nextLayerId).getAttribute('name'),
+            msg = `Inserting ${baseName} layer between ${srcName} and ${dstName}`;
+
+        this._client.startTransaction(msg);
+        // Create the new layer
+        layerId = this._client.createNode({
+            parentId: parentId,
+            baseId: layerBaseId
+        });
+
+        // Connect the new layer to the previous dst of 'connId'
+        newConnId = this._client.createNode({
+            parentId: parentId,
+            baseId: connBaseId
+        });
+        this._client.setPointer(newConnId, 'src', layerId);
+        this._client.setPointer(newConnId, 'dst', nextLayerId);
+
+        // Change the dst of 'connId' to the new layer
+        this._client.setPointer(connId, 'dst', layerId);
+
+        this._client.completeTransaction();
+    };
+
+    ArchEditorControl.prototype._eventCallback = function() {
+        ThumbnailControl.prototype._eventCallback.apply(this, arguments);
+        this.validateLayers();
+    };
+
+    ArchEditorControl.prototype.validateArchitecture = function() {
+        var pluginId = 'ValidateArchitecture',
+            context = this._client.getCurrentPluginContext(pluginId);
+
+        this._logger.info('about to validate arch');
+        // Run the plugin in the browser (set namespace)
+        context.managerConfig.namespace = 'nn';
+        context.pluginConfig = {};
+        Q.ninvoke(this._client, 'runServerPlugin', pluginId, context)
+            .then(res => {
+                var results = res.messages[0].message;
+                if (results.errors !== null) {
+                    this._widget.displayErrors(results.errors);
+                }
+            })
+            .fail(err => this._logger.warn(`Validation failed: ${err}`));
+    };
+
     return ArchEditorControl;
 });

src/visualizers/panels/ArchIndex/ArchIndexPanel.js

@@ -0,0 +1,100 @@
+/*globals define, _, WebGMEGlobal*/
+/*jshint browser: true*/
+
+define([
+    'js/PanelBase/PanelBaseWithHeader',
+    'js/PanelManager/IActivePanel',
+    'widgets/ArchIndex/ArchIndexWidget',
+    'panels/PipelineIndex/PipelineIndexControl'
+], function (
+    PanelBaseWithHeader,
+    IActivePanel,
+    ArchIndexWidget,
+    ArchIndexControl
+) {
+    'use strict';
+
+    var ArchIndexPanel;
+
+    ArchIndexPanel = function (layoutManager, params) {
+        var options = {};
+        //set properties from options
+        options[PanelBaseWithHeader.OPTIONS.LOGGER_INSTANCE_NAME] = 'ArchIndexPanel';
+        options[PanelBaseWithHeader.OPTIONS.FLOATING_TITLE] = true;
+
+        //call parent's constructor
+        PanelBaseWithHeader.apply(this, [options, layoutManager]);
+
+        this._client = params.client;
+        this._embedded = params.embedded;
+
+        //initialize UI
+        this._initialize();
+
+        this.logger.debug('ctor finished');
+    };
+
+    //inherit from PanelBaseWithHeader
+    _.extend(ArchIndexPanel.prototype, PanelBaseWithHeader.prototype);
+    _.extend(ArchIndexPanel.prototype, IActivePanel.prototype);
+
+    ArchIndexPanel.prototype._initialize = function () {
+        var self = this;
+
+        //set Widget title
+        this.setTitle('');
+
+        this.widget = new ArchIndexWidget(this.logger, this.$el);
+
+        this.widget.setTitle = function (title) {
+            self.setTitle(title);
+        };
+
+        this.control = new ArchIndexControl({
+            logger: this.logger,
+            client: this._client,
+            embedded: this._embedded,
+            widget: this.widget
+        });
+
+        this.onActivate();
+    };
+
+    /* OVERRIDE FROM WIDGET-WITH-HEADER */
+    /* METHOD CALLED WHEN THE WIDGET'S READ-ONLY PROPERTY CHANGES */
+    ArchIndexPanel.prototype.onReadOnlyChanged = function (isReadOnly) {
+        //apply parent's onReadOnlyChanged
+        PanelBaseWithHeader.prototype.onReadOnlyChanged.call(this, isReadOnly);
+
+    };
+
+    ArchIndexPanel.prototype.onResize = function (width, height) {
+        this.logger.debug('onResize --> width: ' + width + ', height: ' + height);
+    };
+
+    /* * * * * * * * Visualizer life cycle callbacks * * * * * * * */
+    ArchIndexPanel.prototype.destroy = function () {
+        this.control.destroy();
+        this.widget.destroy();
+
+        PanelBaseWithHeader.prototype.destroy.call(this);
+        WebGMEGlobal.KeyboardManager.setListener(undefined);
+        WebGMEGlobal.Toolbar.refresh();
+    };
+
+    ArchIndexPanel.prototype.onActivate = function () {
+        this.widget.onActivate();
+        this.control.onActivate();
+        WebGMEGlobal.KeyboardManager.setListener(this.widget);
+        WebGMEGlobal.Toolbar.refresh();
+    };
+
+    ArchIndexPanel.prototype.onDeactivate = function () {
+        this.widget.onDeactivate();
+        this.control.onDeactivate();
+        WebGMEGlobal.KeyboardManager.setListener(undefined);
+        WebGMEGlobal.Toolbar.refresh();
+    };
+
+    return ArchIndexPanel;
+});

src/visualizers/panels/ArtifactIndex/ArtifactIndexControl.js

@@ -0,0 +1,203 @@
+/*globals define, WebGMEGlobal*/
+/*jshint browser: true*/
+
+define([
+    'blob/BlobClient',
+    'js/Constants'
+], function (
+    BlobClient,
+    CONSTANTS
+) {
+
+    'use strict';
+
+    var ArtifactIndexControl;
+
+    ArtifactIndexControl = function (options) {
+
+        this._logger = options.logger.fork('Control');
+        this.blobClient = new BlobClient({
+            logger: this._logger.fork('BlobClient')
+        });
+
+        this._client = options.client;
+
+        // Initialize core collections and variables
+        this._widget = options.widget;
+
+        this._currentNodeId = null;
+        this._initWidgetEventHandlers();
+
+        this._logger.debug('ctor finished');
+    };
+
+    ArtifactIndexControl.prototype._initWidgetEventHandlers = function () {
+        this._widget.onNodeClick = (/*id*/) => {
+            // Change the current active object
+            // This is currently disabled as there are not any good
+            // visualizers for the data types
+            // WebGMEGlobal.State.registerActiveObject(id);
+        };
+
+        this._widget.onNodeDeleteClicked = id => {
+            var name = this._client.getNode(id).getAttribute('name'),
+                msg = `Deleted "${name}" artifact (${id}) --`;
+
+            this._client.startTransaction(msg);
+            this._client.deleteNode(id);
+            this._client.completeTransaction();
+        };
+    };
+
+    /* * * * * * * * Visualizer content update callbacks * * * * * * * */
+    // One major concept here is with managing the territory. The territory
+    // defines the parts of the project that the visualizer is interested in
+    // (this allows the browser to then only load those relevant parts).
+    ArtifactIndexControl.prototype.selectedObjectChanged = function (nodeId) {
+        this._logger.debug('activeObject nodeId \'' + nodeId + '\'');
+
+        // Remove current territory patterns
+        if (this._currentNodeId) {
+            this._client.removeUI(this._territoryId);
+        }
+
+        this._currentNodeId = nodeId;
+
+        if (typeof this._currentNodeId === 'string') {
+            // Put new node's info into territory rules
+            this._widget.currentNode = this._currentNodeId;
+            this._selfPatterns = {};
+
+            this._territoryId = this._client.addUI(this, events => {
+                this._eventCallback(events);
+            });
+
+            this._selfPatterns[nodeId] = {children: 1};
+            this._client.updateTerritory(this._territoryId, this._selfPatterns);
+        }
+    };
+
+    // This next function retrieves the relevant node information for the widget
+    ArtifactIndexControl.prototype._getObjectDescriptor = function (nodeId) {
+        var node = this._client.getNode(nodeId),
+            base,
+            hash,
+            objDescriptor;
+
+        if (node) {
+            base = this._client.getNode(node.getBaseId());
+            hash = node.getAttribute('data');
+            objDescriptor = {
+                id: node.getId(),
+                type: base ? base.getAttribute('name') : 'n/a',
+                name: node.getAttribute('name'),
+                createdAt: node.getAttribute('createdAt'),
+                dataURL: this.blobClient.getDownloadURL(hash),
+                parentId: node.getParentId()
+            };
+        }
+
+        return this.blobClient.getMetadata(hash)
+            .then(metadata => {
+                objDescriptor.size = this._humanFileSize(metadata.size);
+                return objDescriptor;
+            });
+    };
+
+    ArtifactIndexControl.prototype._humanFileSize = function (bytes, si) {
+        var thresh = si ? 1000 : 1024,
+            units = si ?
+                ['kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'] :
+                ['KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'],
+            u = -1;
+
+        if (bytes < thresh) {
+            return bytes + ' B';
+        }
+
+        do {
+            bytes = bytes / thresh;
+            u += 1;
+        } while (bytes >= thresh);
+
+        return bytes.toFixed(1) + ' ' + units[u];
+    };
+
+    /* * * * * * * * Node Event Handling * * * * * * * */
+    ArtifactIndexControl.prototype._eventCallback = function (events) {
+        var i = events ? events.length : 0,
+            event;
+
+        this._logger.debug('_eventCallback \'' + i + '\' items');
+
+        while (i--) {
+            event = events[i];
+            switch (event.etype) {
+
+            case CONSTANTS.TERRITORY_EVENT_LOAD:
+                this._onLoad(event.eid);
+                break;
+            case CONSTANTS.TERRITORY_EVENT_UPDATE:
+                this._onUpdate(event.eid);
+                break;
+            case CONSTANTS.TERRITORY_EVENT_UNLOAD:
+                this._onUnload(event.eid);
+                break;
+            default:
+                break;
+            }
+        }
+
+        this._logger.debug('_eventCallback \'' + events.length + '\' items - DONE');
+    };
+
+    ArtifactIndexControl.prototype._onLoad = function (gmeId) {
+        this._getObjectDescriptor(gmeId).then(desc => this._widget.addNode(desc));
+    };
+
+    ArtifactIndexControl.prototype._onUpdate = function (gmeId) {
+        this._getObjectDescriptor(gmeId).then(desc => this._widget.updateNode(desc));
+    };
+
+    ArtifactIndexControl.prototype._onUnload = function (gmeId) {
+        this._widget.removeNode(gmeId);
+    };
+
+    ArtifactIndexControl.prototype._stateActiveObjectChanged = function (model, activeObjectId) {
+        if (this._currentNodeId === activeObjectId) {
+            // The same node selected as before - do not trigger
+        } else {
+            this.selectedObjectChanged(activeObjectId);
+        }
+    };
+
+    /* * * * * * * * Visualizer life cycle callbacks * * * * * * * */
+    ArtifactIndexControl.prototype.destroy = function () {
+        this._detachClientEventListeners();
+    };
+
+    ArtifactIndexControl.prototype._attachClientEventListeners = function () {
+        this._detachClientEventListeners();
+        WebGMEGlobal.State.on('change:' + CONSTANTS.STATE_ACTIVE_OBJECT, this._stateActiveObjectChanged, this);
+    };
+
+    ArtifactIndexControl.prototype._detachClientEventListeners = function () {
+        WebGMEGlobal.State.off('change:' + CONSTANTS.STATE_ACTIVE_OBJECT, this._stateActiveObjectChanged);
+    };
+
+    ArtifactIndexControl.prototype.onActivate = function () {
+        this._attachClientEventListeners();
+
+        if (typeof this._currentNodeId === 'string') {
+            WebGMEGlobal.State.registerSuppressVisualizerFromNode(true);
+            WebGMEGlobal.State.registerActiveObject(this._currentNodeId);
+            WebGMEGlobal.State.registerSuppressVisualizerFromNode(false);
+        }
+    };
+
+    ArtifactIndexControl.prototype.onDeactivate = function () {
+        this._detachClientEventListeners();
+    };
+
+    return ArtifactIndexControl;
+});

src/visualizers/panels/ArtifactIndex/ArtifactIndexPanel.js

@@ -0,0 +1,99 @@
+/*globals define, _, WebGMEGlobal*/
+/*jshint browser: true*/
+
+define([
+    'js/PanelBase/PanelBaseWithHeader',
+    'js/PanelManager/IActivePanel',
+    'widgets/ArtifactIndex/ArtifactIndexWidget',
+    './ArtifactIndexControl'
+], function (
+    PanelBaseWithHeader,
+    IActivePanel,
+    ArtifactIndexWidget,
+    ArtifactIndexControl
+) {
+    'use strict';
+
+    var ArtifactIndexPanel;
+
+    ArtifactIndexPanel = function (layoutManager, params) {
+        var options = {};
+        //set properties from options
+        options[PanelBaseWithHeader.OPTIONS.LOGGER_INSTANCE_NAME] = 'ArtifactIndexPanel';
+        options[PanelBaseWithHeader.OPTIONS.FLOATING_TITLE] = true;
+
+        //call parent's constructor
+        PanelBaseWithHeader.apply(this, [options, layoutManager]);
+
+        this._client = params.client;
+
+        //initialize UI
+        this._initialize();
+
+        this.logger.debug('ctor finished');
+    };
+
+    //inherit from PanelBaseWithHeader
+    _.extend(ArtifactIndexPanel.prototype, PanelBaseWithHeader.prototype);
+    _.extend(ArtifactIndexPanel.prototype, IActivePanel.prototype);
+
+    ArtifactIndexPanel.prototype._initialize = function () {
+        var self = this;
+
+        //set Widget title
+        this.setTitle('');
+
+        this.widget = new ArtifactIndexWidget(this.logger, this.$el);
+
+        this.widget.setTitle = function (title) {
+            self.setTitle(title);
+        };
+
+        this.control = new ArtifactIndexControl({
+            logger: this.logger,
+            client: this._client,
+            widget: this.widget
+        });
+
+        this.onActivate();
+    };
+
+    /* OVERRIDE FROM WIDGET-WITH-HEADER */
+    /* METHOD CALLED WHEN THE WIDGET'S READ-ONLY PROPERTY CHANGES */
+    ArtifactIndexPanel.prototype.onReadOnlyChanged = function (isReadOnly) {
+        //apply parent's onReadOnlyChanged
+        PanelBaseWithHeader.prototype.onReadOnlyChanged.call(this, isReadOnly);
+
+    };
+
+    ArtifactIndexPanel.prototype.onResize = function (width, height) {
+        this.logger.debug('onResize --> width: ' + width + ', height: ' + height);
+        this.widget.onWidgetContainerResize(width, height);
+    };
+
+    /* * * * * * * * Visualizer life cycle callbacks * * * * * * * */
+    ArtifactIndexPanel.prototype.destroy = function () {
+        this.control.destroy();
+        this.widget.destroy();
+
+        PanelBaseWithHeader.prototype.destroy.call(this);
+        WebGMEGlobal.KeyboardManager.setListener(undefined);
+        WebGMEGlobal.Toolbar.refresh();
+    };
+
+    ArtifactIndexPanel.prototype.onActivate = function () {
+        this.widget.onActivate();
+        this.control.onActivate();
+        WebGMEGlobal.KeyboardManager.setListener(this.widget);
+        WebGMEGlobal.Toolbar.refresh();
+    };
+
+    ArtifactIndexPanel.prototype.onDeactivate = function () {
+        this.widget.onDeactivate();
+        this.control.onDeactivate();
+        WebGMEGlobal.KeyboardManager.setListener(undefined);
+        WebGMEGlobal.Toolbar.refresh();
+    };
+
+    return ArtifactIndexPanel;
+});

src/visualizers/panels/ClassCodeEditor/ClassCodeEditorControl.js

@@ -100,7 +100,7 @@ define([
 
         this._client.startTransaction(`Updating class "${name || nodeName}"`);
         if (name) {
-            this._client.setAttributes(id, 'name', name);
+            this._client.setAttribute(id, 'name', name);
         }
         if (basePath) {
             this._client.setBase(id, basePath);
@@ -137,7 +137,7 @@ define([
         }
 
         for (i = nameMatches.length; i--;) {
-            if (this._client.isTypeOf(nameMatches[i].getId(), classNode.getId())) {
+            if (nameMatches[i].isTypeOf(classNode.getId())) {
                 return nameMatches[i].getId();
             }
         }

src/visualizers/panels/ExecutionView/ExecutionViewControl.js

@@ -6,12 +6,14 @@ define([
     'deepforge/Constants',
     'panels/EasyDAG/EasyDAGControl',
     'deepforge/viz/PipelineControl',
+    'deepforge/viz/Execute',
     'underscore'
 ], function (
     GME_CONSTANTS,
     CONSTANTS,
     EasyDAGControl,
     PipelineControl,
+    Execute,
     _
 ) {
 
@@ -21,15 +23,18 @@ define([
 
     ExecutionViewControl = function (options) {
         EasyDAGControl.call(this, options);
+        Execute.call(this, this._client, this._logger);
         this.addedNodes = {};
         this.originTerritory = {};
         this.originTerritoryId = null;
+        this.readOnly = false;
     };
 
     _.extend(
         ExecutionViewControl.prototype,
         EasyDAGControl.prototype,
-        PipelineControl.prototype
+        PipelineControl.prototype,
+        Execute.prototype
     );
 
     /* * * * * * * * Visualizer content update callbacks * * * * * * * */
@@ -76,6 +81,9 @@ define([
                     this._client.removeUI(this.originTerritoryId);
                 }
             }
+            if (!this.readOnly) {
+                this.checkPipelineExecution(this._client.getNode(id));
+            }
         }
     };
 

src/visualizers/panels/ExecutionView/ExecutionViewPanel.js

@@ -73,7 +73,7 @@ define([
     ExecutionViewPanel.prototype.onReadOnlyChanged = function (isReadOnly) {
         //apply parent's onReadOnlyChanged
         PanelBaseWithHeader.prototype.onReadOnlyChanged.call(this, isReadOnly);
-
+        this.control.readOnly = isReadOnly;
     };
 
     ExecutionViewPanel.prototype.onResize = function (width, height) {

src/visualizers/panels/Footer/FooterPanel.js

@@ -1,19 +1,15 @@
-/*globals define, _, WebGMEGlobal, $ */
+/*globals define, _, $ */
 /*jshint browser: true*/
 
 define([
     'js/PanelBase/PanelBase',
     'js/Widgets/NetworkStatus/NetworkStatusWidget',
     'js/Widgets/BranchStatus/BranchStatusWidget',
-    'js/Widgets/BranchSelector/BranchSelectorWidget',
-    'js/Widgets/KeyboardManager/KeyboardManagerWidget',
     './FilteredNotificationWidget'
 ], function (
     PanelBase,
     NetworkStatusWidget,
     BranchStatusWidget,
-    BranchSelectorWidget,
-    KeyboardManagerWidget,
     NotificationWidget
 ) {
 
@@ -47,7 +43,6 @@ define([
             navBarInner = $('<div/>', {class: 'navbar-inner'}),
             separator = $('<div class="spacer pull-right"></div>'),
             widgetPlaceHolder = $('<div class="pull-right"></div>'),
-            keyBoardManagerEl,
             networkStatusEl,
             branchStatusEl,
             notificationEl;
@@ -58,13 +53,6 @@ define([
         //padding from screen right edge
         navBarInner.append(separator.clone());
 
-        //keyboard enable/disbale widget (NOTE: only on non touch device)
-        if (WebGMEGlobal.SUPPORTS_TOUCH !== true) {
-            keyBoardManagerEl = widgetPlaceHolder.clone();
-            new KeyboardManagerWidget(keyBoardManagerEl);
-            navBarInner.append(keyBoardManagerEl).append(separator.clone());
-        }
-
         networkStatusEl = widgetPlaceHolder.clone();
         new NetworkStatusWidget(networkStatusEl, this._client);
         navBarInner.append(networkStatusEl).append(separator.clone());

src/visualizers/panels/ForgeActionButton/Actions.js

@@ -264,6 +264,32 @@ define([
                 action: function() {
                     this.addOperation();
                 }
+            },
+            {
+                name: 'Export for local execution',
+                icon: 'play_for_work',
+                priority: -1,
+                action: function() {
+                    var pluginId = 'GenerateExecFile',
+                        context = this.client.getCurrentPluginContext(pluginId);
+
+                    // Run the plugin in the browser (set namespace)
+                    context.managerConfig.namespace = 'pipeline';
+                    context.pluginConfig = {};
+                    Q.ninvoke(this.client, 'runBrowserPlugin', pluginId, context)
+                        .then(res => {
+                            var id = this._currentNodeId,
+                                node = this.client.getNode(id),
+                                base = this.client.getNode(node.getBaseId()),
+                                type = base.getAttribute('name'),
+                                name = node.getAttribute('name');
+
+                            // Get the file and download it
+                            this.downloadFromBlob(res.artifacts[0]);
+                            Materialize.toast(`Exported ${name} ${type}!`, 2000);
+                        })
+                        .fail(err => Materialize.toast(`Export failed: ${err}`, 2000));
+                }
             }
         ],
         Architecture: [

src/visualizers/panels/ForgeActionButton/ForgeActionButton.js

@@ -3,6 +3,7 @@
 
 define([
     'blob/BlobClient',
+    'js/Utils/SaveToDisk',
     'js/Constants',
     'panel/FloatingActionButton/FloatingActionButton',
     'deepforge/viz/PipelineControl',
@@ -16,6 +17,7 @@ define([
     'deepforge/globals'
 ], function (
     BlobClient,
+    SaveToDisk,
     CONSTANTS,
     PluginButton,
     PipelineControl,
@@ -178,7 +180,7 @@ define([
 
     ForgeActionButton.prototype.createNamedNode = function(baseId, isMeta) {
         var parentId = this._currentNodeId,
-            newId = this.client.createChild({parentId, baseId}),
+            newId = this.client.createNode({parentId, baseId}),
             basename = 'New' + this.client.getNode(baseId).getAttribute('name'),
             newName = this.getUniqueName(parentId, basename);
 
@@ -186,7 +188,7 @@ define([
         if (!isMeta) {
             newName = newName.substring(0, 1).toLowerCase() + newName.substring(1);
         }
-        this.client.setAttributes(newId, 'name', newName);
+        this.client.setAttribute(newId, 'name', newName);
         return newId;
     };
 
@@ -355,10 +357,24 @@ define([
         var nodeId = this._currentNodeId;
         if (nodeId) {
             this.client.startTransaction(msg);
-            this.client.delMoreNodes([nodeId]);
-            this.client.completeTransaction(msg);
+            this.client.deleteNode(nodeId);
+            this.client.completeTransaction();
         }
     };
 
+    ForgeActionButton.prototype.downloadFromBlob = function(hash) {
+        var name;
+
+        this._blobClient.getMetadata(hash)
+            .then(metadata => {
+                name = metadata.name;
+                return this._blobClient.getObjectAsString(hash);
+            })
+            .then(text => {
+                SaveToDisk.downloadTextAsFile(name, text);
+            })
+            .fail(err => this.logger.error(`Blob download failed: ${err}`));
+    };
+
     return ForgeActionButton;
 });

src/visualizers/panels/ForwardViz/ForwardVizPanel.js

@@ -0,0 +1,67 @@
+/*globals define, _, WebGMEGlobal*/
+/*jshint browser: true*/
+
+define([
+    'js/Constants',
+    'deepforge/globals',
+    'js/PanelBase/PanelBaseWithHeader'
+], function (
+    CONSTANTS,
+    DeepForge,
+    PanelBaseWithHeader
+) {
+    'use strict';
+
+    var ForwardVizPanel;
+
+    ForwardVizPanel = function (layoutManager, params) {
+        var options = {};
+        //set properties from options
+        options[PanelBaseWithHeader.OPTIONS.LOGGER_INSTANCE_NAME] = 'ForwardViz';
+
+        //call parent's constructor
+        PanelBaseWithHeader.apply(this, [options, layoutManager]);
+
+        this._client = params.client;
+
+        //initialize UI
+        this._initialize();
+
+        this.logger.debug('ctor finished');
+    };
+
+    //inherit from PanelBaseWithHeader
+    _.extend(ForwardVizPanel.prototype, PanelBaseWithHeader.prototype);
+
+    ForwardVizPanel.prototype._initialize = function () {
+        this.control = this;
+        this.onActivate();
+    };
+
+    ForwardVizPanel.prototype.selectedObjectChanged = function(nodeId) {
+        if (nodeId === CONSTANTS.PROJECT_ROOT_ID) {
+            DeepForge.places.MyPipelines().then(id => WebGMEGlobal.State.registerActiveObject(id));
+        }
+    };
+
+    /* OVERRIDE FROM WIDGET-WITH-HEADER */
+    /* METHOD CALLED WHEN THE WIDGET'S READ-ONLY PROPERTY CHANGES */
+        //apply parent's onReadOnlyChanged
+    ForwardVizPanel.prototype.onReadOnlyChanged = function() {
+        PanelBaseWithHeader.prototype.onReadOnlyChanged.apply(this, arguments);
+
+    };
+
+    /* * * * * * * * Visualizer life cycle callbacks * * * * * * * */
+    ForwardVizPanel.prototype.destroy = function () {
+        PanelBaseWithHeader.prototype.destroy.call(this);
+    };
+
+    ForwardVizPanel.prototype.onReadOnlyChanged =
+    ForwardVizPanel.prototype.onResize =
+    ForwardVizPanel.prototype.onActivate =
+    ForwardVizPanel.prototype.onDeactivate = function () {
+    };
+
+    return ForwardVizPanel;
+});

src/visualizers/panels/JobEditor/JobEditorPanel.js

@@ -10,11 +10,13 @@ define([
     'panels/TilingViz/TilingVizPanel',
     'panels/OutputViewer/OutputViewerPanel',
     'panels/OperationCodeEditor/OperationCodeEditorPanel',
+    'deepforge/viz/Execute',
     'js/Constants'
 ], function (
     TilingViz,
     OutputViewer,
     OperationCodeEditor,
+    Execute,
     CONSTANTS
 ) {
     'use strict';
@@ -23,11 +25,16 @@ define([
 
     JobEditorPanel = function (layoutManager, params) {
         TilingViz.call(this, layoutManager, params);
+        Execute.call(this, this._client, this.logger);
         this.readOnly = false;
     };
 
     //inherit from PanelBaseWithHeader
-    _.extend(JobEditorPanel.prototype, TilingViz.prototype);
+    _.extend(
+        JobEditorPanel.prototype,
+        Execute.prototype,
+        TilingViz.prototype
+    );
 
     JobEditorPanel.prototype.getPanels = function () {
         if (this.readOnly) {
@@ -88,6 +95,10 @@ define([
             // update the OutputViewer controller
             var i = this._panels.length;
             this._panels[i-1].control.selectedObjectChanged(nodeId);
+            // Check if the job needs to be reconnected
+            if (!this.isReadOnly()) {
+                this.checkJobExecution(node);
+            }
         }
     };
 

src/visualizers/panels/LayerEditor/LayerEditorControl.js

@@ -94,10 +94,10 @@ define([
         this._client.startTransaction(msg);
 
         TextEditorControl.prototype.saveTextFor.call(this, id, text, true);
-        this._client.setAttributes(id, 'name', layerSchema.name);
+        this._client.setAttribute(id, 'name', layerSchema.name);
 
         this._logger.debug(`Setting ctor args to ${ctorAttrs.join(',')}`);
-        this._client.setAttributes(id, Constants.CTOR_ARGS_ATTR, ctorAttrs.join(','));
+        this._client.setAttribute(id, Constants.CTOR_ARGS_ATTR, ctorAttrs.join(','));
 
         types = layerSchema.types || {};
         schema = this.getPointerMeta();
@@ -114,29 +114,29 @@ define([
 
         // Remove old pointers
         node.getPointerNames().filter(ptr => !currentPtrs[ptr])
-            .forEach(ptr => this._client.deleteMetaPointer(id, ptr));
+            .forEach(ptr => this._client.delPointerMeta(id, ptr));
 
         // Remove old attributes
         setterNames = Object.keys(layerSchema.setters);
         _.difference(currentAttrs, ctorAttrs, setterNames)
-            .forEach(attr => this._client.removeAttributeSchema(id, attr));
+            .forEach(attr => this._client.delAttributeMeta(id, attr));
 
         // Add setters
         for (i = setterNames.length; i--;) {
             schema = utils.getSetterSchema(setterNames[i], layerSchema.setters, layerSchema.defaults);
             // Get setter attr schema
             if (schema.hasOwnProperty('default')) {
-                this._client.setAttributes(id, setterNames[i], schema.default);
+                this._client.setAttribute(id, setterNames[i], schema.default);
                 delete schema.default;
             }
             if (types[setterNames[i]]) {
                 schema.type = types[setterNames[i]];
             }
-            this._client.setAttributeSchema(id, setterNames[i], schema);
+            this._client.setAttributeMeta(id, setterNames[i], schema);
         }
 
         ctorAttrs.forEach(attr =>
-            this._client.setAttributeSchema(id, attr, {
+            this._client.setAttributeMeta(id, attr, {
                 type: types[attr] || 'string'
             })
         );

src/visualizers/panels/LogViewer/LogViewerControl.js

@@ -5,7 +5,7 @@
 // if the job is running, get the logs from the log-storage
 define([
     'q',
-    'deepforge/JobLogsClient',
+    'deepforge/api/JobLogsClient',
     'js/Constants',
     'deepforge/Constants',
     'panels/TextEditor/TextEditorControl'
@@ -82,7 +82,14 @@ define([
     };
 
     LogViewerControl.prototype._getRunningLogs = function (id) {
-        var logManager = new JobLogsClient({
+        var logManager;
+
+        if (!this._client.getActiveBranchName() || !this._client.getActiveProjectId()) {
+            // Logs are only stored for a given branch
+            return Q().then(() => '');
+        }
+
+        logManager = new JobLogsClient({
             logger: this._logger,
             projectId: this._client.getActiveProjectId(),
             branchName: this._client.getActiveBranchName()

src/visualizers/panels/MainView/MainViewControl.js

@@ -1,244 +0,0 @@
-/*globals define, WebGMEGlobal*/
-/*jshint browser: true*/
-
-define([
-    'blob/BlobClient',
-    'js/Constants',
-    'q',
-    'deepforge/globals'
-], function (
-    BlobClient,
-    CONSTANTS,
-    Q,
-    DeepForge
-) {
-
-    'use strict';
-
-    var MainViewControl;
-
-    MainViewControl = function (options) {
-
-        this._logger = options.logger.fork('Control');
-        this._client = options.client;
-        this._widget = options.widget;
-
-        this._currentNodeId = null;
-        this._embedded = options.embedded;
-        this.territory = {};
-        this.ui = {};
-        this._blobClient = new BlobClient({
-            logger: this._logger.fork('BlobClient')
-        });
-
-        this._initWidgetEventHandlers();
-        this._logger.debug('ctor finished');
-    };
-
-    MainViewControl.prototype._initWidgetEventHandlers = function () {
-        this._widget.deleteNode = id => {
-            var node = this._client.getNode(id),
-                baseId = node.getBaseId(),
-                base = this._client.getNode(baseId),
-                baseName = base.getAttribute('name'),
-                name = node.getAttribute('name'),
-                msg = `Deleting ${baseName} "${name}"`;
-
-            this._client.startTransaction(msg);
-            this._client.delMoreNodes([id]);
-            this._client.completeTransaction();
-        };
-
-        this._widget.dataUrlFor = id => {
-            var node = this._client.getNode(id),
-                hash = node.getAttribute('data');
-
-            if (hash) {
-                return this._blobClient.getDownloadURL(hash);
-            } else {
-                return null;
-            }
-        };
-
-        this._widget.toggleEmbeddedPanel = () => this.toggleEmbeddedPanel();
-        this._widget.updateLibraries = this.updateLibraries.bind(this);
-        this._widget.checkLibUpdates = this.checkLibUpdates.bind(this);
-        this._widget.getProjectName = this.getProjectName.bind(this);
-    };
-
-    MainViewControl.prototype.getProjectName = function () {
-        return this._client.getActiveProjectId().split('+')[1];
-    };
-
-    MainViewControl.prototype.checkLibUpdates = function () {
-        var pluginId = 'CheckLibraries',
-            context = this._client.getCurrentPluginContext(pluginId);
-
-        return Q.ninvoke(this._client, 'runServerPlugin', pluginId, context)
-            .then(res => {
-                return res.messages.map(msg => msg.message.split(' '));
-            });
-    };
-
-    MainViewControl.prototype.updateLibraries = function (libraries) {
-        var promises = libraries
-            .map(lib => Q.ninvoke(this._client, 'updateLibrary', lib[0], lib[1]));
-
-        return Q.all(promises);
-    };
-
-    /* * * * * * * * Visualizer content update callbacks * * * * * * * */
-    MainViewControl.prototype.selectedObjectChanged = function (nodeId) {
-        this._logger.debug('activeObject nodeId \'' + nodeId + '\'');
-
-        // Remove current territory patterns
-        this.clearTerritoryRules();
-
-        this._currentNodeId = nodeId;
-
-        if (typeof this._currentNodeId === 'string') {
-            var terrTypes = [
-                /* [type, root dir] */
-                ['arch', 'MyArchitectures'],
-                ['artifact', 'MyArtifacts']
-            ];
-
-            terrTypes.forEach(pair => {
-                var type = pair[0],
-                    dirname = pair[1];
-
-                // Update the territory
-                this.territory[type] = {};
-                DeepForge.places[dirname]().then(id => {
-                    this.territory[type][id] = {children: 1};
-                    this.ui[type] = this._client.addUI(this, this.handleEvents.bind(this, type));
-                    this._client.updateTerritory(this.ui[type], this.territory[type]);
-                });
-            });
-        }
-    };
-
-    MainViewControl.prototype.handleEvents = function (type, events) {
-        var event;
-
-        // Remove the containing dir
-        events = events.filter(e => !this.territory[type][e.eid]);
-        this._logger.debug('_eventCallback \'' + i + '\' items');
-
-        for (var i = events.length; i--;) {
-            event = events[i];
-            switch (event.etype) {
-
-            case CONSTANTS.TERRITORY_EVENT_LOAD:
-                this.onLoad(type, event.eid);
-                break;
-            case CONSTANTS.TERRITORY_EVENT_UPDATE:
-                this._onUpdate(type, event.eid);
-                break;
-            case CONSTANTS.TERRITORY_EVENT_UNLOAD:
-                this._onUnload(event.eid);
-                break;
-            default:
-                break;
-            }
-        }
-
-        this._logger.debug('_eventCallback \'' + events.length + '\' items - DONE');
-    };
-
-    MainViewControl.prototype.onLoad = function(type, id) {
-        // Load a node of the given type
-        var desc = this._getObjectDescriptor(type, id);
-        if (type === 'arch') {
-            this._widget.addArch(desc);
-        } else {  // artifacts
-            this._widget.addArtifact(desc);
-        }
-    };
-
-    // This next function retrieves the relevant node information for the widget
-    MainViewControl.prototype._getArtifactDesc = function (id) {
-        var node = this._client.getNode(id),
-            data = node.getAttribute('data'),
-            desc = this._getBasicDesc(id);
-
-        desc.data = data;
-        return desc;
-    };
-
-    MainViewControl.prototype._getArchDesc =
-    MainViewControl.prototype._getBasicDesc = function (id) {
-        var node = this._client.getNode(id);
-
-        return {
-            id: id,
-            name: node.getAttribute('name')
-        };
-    };
-
-    MainViewControl.prototype._getObjectDescriptor = function (type, id) {
-        return type === 'arch' ?
-            this._getArchDesc(id) :
-            this._getArtifactDesc(id);
-    };
-
-    /* * * * * * * * Node Event Handling * * * * * * * */
-    MainViewControl.prototype._onUpdate = function (type, gmeId) {
-        var description = this._getObjectDescriptor(type, gmeId);
-        this._widget.updateNode(description);
-    };
-
-    MainViewControl.prototype._onUnload = function (gmeId) {
-        this._widget.removeNode(gmeId);
-    };
-
-    MainViewControl.prototype._stateActiveObjectChanged = function (model, activeObjectId) {
-        if (this._currentNodeId !== activeObjectId) {
-            this.selectedObjectChanged(activeObjectId);
-        }
-    };
-
-    /* * * * * * * * Visualizer life cycle callbacks * * * * * * * */
-    MainViewControl.prototype.destroy = function () {
-        this._detachClientEventListeners();
-        this.clearTerritoryRules();
-    };
-
-    MainViewControl.prototype._attachClientEventListeners = function () {
-        this._detachClientEventListeners();
-        if (!this._embedded) {
-            WebGMEGlobal.State.on('change:' + CONSTANTS.STATE_ACTIVE_OBJECT,
-                this._stateActiveObjectChanged, this);
-        }
-    };
-
-    MainViewControl.prototype._detachClientEventListeners = function () {
-        if (!this._embedded) {
-            WebGMEGlobal.State.off('change:' + CONSTANTS.STATE_ACTIVE_OBJECT,
-                this._stateActiveObjectChanged);
-        }
-    };
-
-    MainViewControl.prototype.onActivate = function () {
-        this._attachClientEventListeners();
-
-        if (typeof this._currentNodeId === 'string') {
-            WebGMEGlobal.State.registerSuppressVisualizerFromNode(true);
-            WebGMEGlobal.State.registerActiveObject(this._currentNodeId);
-            WebGMEGlobal.State.registerSuppressVisualizerFromNode(false);
-        }
-    };
-
-    MainViewControl.prototype.clearTerritoryRules = function () {
-        if (Object.keys(this.ui).length) {
-            Object.keys(this.ui).forEach(id =>
-                this._client.removeUI(this.ui[id]));
-        }
-    };
-
-    MainViewControl.prototype.onDeactivate = function () {
-        this._detachClientEventListeners();
-    };
-
-    return MainViewControl;
-});

src/visualizers/panels/MainView/MainViewPanel.js

@@ -1,167 +0,0 @@
-/*globals define, $, _, WebGMEGlobal*/
-/*jshint browser: true*/
-
-// The main panel shows the PipelineIndex w/ a bar on the left for viewing architectures
-// and pipelines
-define([
-    'js/PanelBase/PanelBaseWithHeader',
-    'js/PanelManager/IActivePanel',
-    'widgets/MainView/MainViewWidget',
-    './MainViewControl',
-    'panels/PipelineIndex/PipelineIndexPanel',
-    'panels/ExecutionIndex/ExecutionIndexPanel',
-    'deepforge/globals'
-], function (
-    PanelBaseWithHeader,
-    IActivePanel,
-    MainViewWidget,
-    MainViewControl,
-    PipelineIndexPanel,
-    ExecutionIndexPanel,
-    DeepForge
-) {
-    'use strict';
-
-    var MainViewPanel;
-
-    MainViewPanel = function (layoutManager, params) {
-        var options = {};
-        //set properties from options
-        options[PanelBaseWithHeader.OPTIONS.LOGGER_INSTANCE_NAME] = 'MainViewPanel';
-        options[PanelBaseWithHeader.OPTIONS.FLOATING_TITLE] = true;
-
-        //call parent's constructor
-        PanelBaseWithHeader.apply(this, [options, layoutManager]);
-
-        this._client = params.client;
-        this._embedded = params.embedded;
-
-        //initialize UI
-        this.$nav = $('<div>', {id: 'nav-container'});
-        this.$el.css({padding: 0});
-
-        this.embeddedPanels = [
-            PipelineIndexPanel,
-            ExecutionIndexPanel
-        ];
-        this.nextPanelIndex = 0;
-        this._lm = layoutManager;
-        this._params = params;
-        this.$el.append(this.$nav);
-        this._initialize();
-
-        this.logger.debug('ctor finished');
-    };
-
-    //inherit from PanelBaseWithHeader
-    _.extend(MainViewPanel.prototype, PanelBaseWithHeader.prototype);
-    _.extend(MainViewPanel.prototype, IActivePanel.prototype);
-
-    MainViewPanel.prototype._initialize = function () {
-        //set Widget title
-        this.setTitle('');
-
-        this.widget = new MainViewWidget(this.logger, this.$nav);
-
-        this.control = new MainViewControl({
-            logger: this.logger,
-            client: this._client,
-            embedded: this._embedded,
-            widget: this.widget
-        });
-
-        this.control.toggleEmbeddedPanel = this.toggleEmbeddedPanel.bind(this);
-        var selectedObjectChanged = this.control.selectedObjectChanged;
-        this.control.selectedObjectChanged = id => {
-            this.getEmbeddedNode().then(nodeId =>
-                this.embeddedPanel.control.selectedObjectChanged(nodeId));
-            selectedObjectChanged.call(this.control, id);
-        };
-
-        this.toggleEmbeddedPanel(true);
-        this.onActivate();
-    };
-
-    MainViewPanel.prototype.getEmbeddedNode = function() {
-        if (this.nextPanelIndex === 1) {
-            return DeepForge.places.MyPipelines();
-        } else {
-            return DeepForge.places.MyExecutions();
-        }
-    };
-
-    MainViewPanel.prototype.toggleEmbeddedPanel = function (silent) {
-        var Panel = this.embeddedPanels[this.nextPanelIndex];
-        this.nextPanelIndex = (this.nextPanelIndex + 1) % this.embeddedPanels.length;
-
-        if (this.embeddedPanel) {  // Remove current
-            this.embeddedPanel.destroy();
-            this.$embedded.remove();
-        }
-
-        this.embeddedPanel = new Panel(this._lm, this._params);
-        this.$embedded = this.embeddedPanel.$el;
-        this.$embedded.addClass('main-view-embedded');
-        this.$el.append(this.$embedded);
-
-        // Call on Resize and selectedObjectChanged
-        this.onResize(this.width, this.height);
-        if (!silent) {
-            this.getEmbeddedNode().then(nodeId =>
-                this.embeddedPanel.control.selectedObjectChanged(nodeId));
-        }
-    };
-
-    /* OVERRIDE FROM WIDGET-WITH-HEADER */
-    /* METHOD CALLED WHEN THE WIDGET'S READ-ONLY PROPERTY CHANGES */
-    MainViewPanel.prototype.onReadOnlyChanged = function (isReadOnly) {
-        //apply parent's onReadOnlyChanged
-        PanelBaseWithHeader.prototype.onReadOnlyChanged.call(this, isReadOnly);
-
-    };
-
-    MainViewPanel.prototype.onResize = function (width, height) {
-        var navWidth,
-            embeddedWidth;
-
-        this.logger.debug('onResize --> width: ' + width + ', height: ' + height);
-        this.widget.onWidgetContainerResize(width, height);
-        navWidth = this.widget.width();
-        embeddedWidth = width-navWidth;
-        this.$embedded.css({
-            width: embeddedWidth,
-            height: height,
-            left: navWidth,
-            margin: 'inherit'
-        });
-        this.embeddedPanel.onResize(embeddedWidth, height);
-        this.width = width;
-        this.height = height;
-    };
-
-    /* * * * * * * * Visualizer life cycle callbacks * * * * * * * */
-    MainViewPanel.prototype.destroy = function () {
-        this.control.destroy();
-        this.widget.destroy();
-
-        PanelBaseWithHeader.prototype.destroy.call(this);
-        WebGMEGlobal.KeyboardManager.setListener(undefined);
-        WebGMEGlobal.Toolbar.refresh();
-    };
-
-    MainViewPanel.prototype.onActivate = function () {
-        this.widget.onActivate();
-        this.control.onActivate();
-        WebGMEGlobal.KeyboardManager.setListener(this.widget);
-        WebGMEGlobal.Toolbar.refresh();
-    };
-
-    MainViewPanel.prototype.onDeactivate = function () {
-        this.widget.onDeactivate();
-        this.control.onDeactivate();
-        WebGMEGlobal.KeyboardManager.setListener(undefined);
-        WebGMEGlobal.Toolbar.refresh();
-    };
-
-    return MainViewPanel;
-});

src/visualizers/panels/OperationInterfaceEditor/OperationInterfaceEditorControl.EventHandlers.js

@@ -135,7 +135,7 @@ define([
                 }
             ]
         });
-        this._client.makePointer(this._currentNodeId, ptrName, null);
+        this._client.setPointer(this._currentNodeId, ptrName, null);
         this._client.completeTransaction();
     };
 
@@ -174,10 +174,10 @@ define([
         this._client.startTransaction(msg);
 
         // Currently, this will not update children already using old name...
-        this._client.deleteMetaPointer(this._currentNodeId, from);
+        this._client.delPointerMeta(this._currentNodeId, from);
         this._client.delPointer(this._currentNodeId, from);
         this._client.setPointerMeta(this._currentNodeId, ptrName, meta);
-        this._client.makePointer(this._currentNodeId, ptrName, null);
+        this._client.setPointer(this._currentNodeId, ptrName, null);
 
         this._client.completeTransaction();
     };
@@ -188,7 +188,7 @@ define([
 
         this._client.startTransaction(msg);
         // Currently, this will not update children already using old name...
-        this._client.deleteMetaPointer(this._currentNodeId, name);
+        this._client.delPointerMeta(this._currentNodeId, name);
         this._client.delPointer(this._currentNodeId, name);
         this._client.completeTransaction();
     };
@@ -204,13 +204,13 @@ define([
 
         msg = `Adding ${isInput ? 'input' : 'output'} "${dataName}" to ${name} interface`;
         this._client.startTransaction(msg);
-        var id = this._client.createChild({
+        var id = this._client.createNode({
             parentId: cntrId,
             baseId: typeId
         });
 
         // Set the name of the new input
-        this._client.setAttributes(id, 'name', dataName);
+        this._client.setAttribute(id, 'name', dataName);
 
         this._client.completeTransaction();
     };

src/visualizers/panels/OperationInterfaceEditor/OperationInterfaceEditorControl.js

@@ -128,10 +128,10 @@ define([
             operation = metanodes.find(n => n.getAttribute('name') === 'Data');
 
         // Get all the meta nodes that are instances of Data
-        metanodes.map(n => n.getId())
-            .filter(nId => this._client.isTypeOf(nId, operation.getId()))
+        metanodes
+            .filter(node => node.isTypeOf(operation.getId()))
             // Add a rule for them
-            .forEach(opId => this._territories[opId] = {children: 0});
+            .forEach(op => this._territories[op.getId()] = {children: 0});
 
         this._client.updateTerritory(this._territoryId, this._territories);
     };
@@ -445,5 +445,9 @@ define([
         return false;
     };
 
+    OperationInterfaceEditorControl.prototype._isValidTerminalNode = function() {
+        return true;
+    };
+
     return OperationInterfaceEditorControl;
 });

src/visualizers/panels/OutputViewer/OutputViewerPanel.js

@@ -199,6 +199,7 @@ define([
 
     OutputViewerPanel.prototype.handleEvents = function (events) {
         var metadataId = this.getMetadataId(),
+            node,
             event;
 
         if (!metadataId) {
@@ -206,8 +207,13 @@ define([
             return;
         }
 
-        events = events.filter(event => event.eid && (this._pages[event.eid] ||
-            this._client.isTypeOf(event.eid, metadataId)));
+        events = events.filter(event => {
+            if (event.eid) {
+                node = this._client.getNode(event.eid);
+                return this._pages[event.eid] || node.isTypeOf(metadataId);
+            }
+            return false;
+        });
         for (var i = events.length; i--;) {
             event = events[i];
             switch (event.etype) {

src/visualizers/panels/PipelineEditor/PipelineEditorControl.js

@@ -4,7 +4,7 @@
 define([
     'deepforge/Constants',
     'js/Constants',
-    'panels/EasyDAG/EasyDAGControl',
+    'deepforge/viz/panels/ThumbnailControl',
     'deepforge/viz/PipelineControl',
     'deepforge/viz/Execute',
     'deepforge/globals',
@@ -15,7 +15,7 @@ define([
 ], function (
     CONSTANTS,
     GME_CONSTANTS,
-    EasyDAGControl,
+    ThumbnailControl,
     PipelineControl,
     Execute,
     DeepForge,
@@ -40,7 +40,7 @@ define([
     DECORATORS[CONSTANTS.OP.INPUT] = 'ArtifactOpDecorator';
 
     PipelineEditorControl = function (options) {
-        EasyDAGControl.call(this, options);
+        ThumbnailControl.call(this, options);
         Execute.call(this, this._client, this._logger);
         this.addedIds = {};
         this.executionTerritory = {};
@@ -52,7 +52,7 @@ define([
 
     _.extend(
         PipelineEditorControl.prototype,
-        EasyDAGControl.prototype,
+        ThumbnailControl.prototype,
         PipelineControl.prototype,
         Execute.prototype
     );
@@ -98,7 +98,7 @@ define([
         var msg = `Renaming pipeline "${from}" -> "${to}"`;
         if (from !== to && !/^\s*$/.test(to)) {
             this._client.startTransaction(msg);
-            this._client.setAttributes(this._currentNodeId, 'name', to);
+            this._client.setAttribute(this._currentNodeId, 'name', to);
             this._client.completeTransaction();
         }
     };
@@ -124,10 +124,10 @@ define([
             operation = metanodes.find(n => n.getAttribute('name') === 'Operation');
 
         // Get all the meta nodes that are instances of Operations
-        metanodes.map(n => n.getId())
-            .filter(nId => this._client.isTypeOf(nId, operation.getId()))
+        metanodes
+            .filter(n => n.isTypeOf(operation.getId()))
             // Add a rule for them
-            .forEach(opId => this._territories[opId] = this.TERRITORY_RULE);
+            .forEach(op => this._territories[op.getId()] = this.TERRITORY_RULE);
 
         // Add arch/artifact dir to the territory
         // loading more than necessary.... can restrict it in the future
@@ -138,12 +138,11 @@ define([
     };
 
     PipelineEditorControl.prototype._initWidgetEventHandlers = function () {
-        EasyDAGControl.prototype._initWidgetEventHandlers.call(this);
+        ThumbnailControl.prototype._initWidgetEventHandlers.call(this);
         this._widget.getExistingPortMatches = this.getExistingPortMatches.bind(this);
         this._widget.createConnection = this.createConnection.bind(this);
         this._widget.removeConnection = this.removeConnection.bind(this);
         this._widget.getDecorator = this.getDecorator.bind(this);
-        this._widget.updateThumbnail = this.updateThumbnail.bind(this);
     };
 
     PipelineEditorControl.prototype.isContainedInActive = function (gmeId) {
@@ -159,7 +158,7 @@ define([
             this.addedIds[desc.id] = true;
             // Validate any connections
             if (this.isValid(desc)) {
-                return EasyDAGControl.prototype._onLoad.call(this, gmeId);
+                return ThumbnailControl.prototype._onLoad.call(this, gmeId);
             }
         } else if (desc.parentId !== null &&
             this.isContainedInActive(desc.parentId) && desc.isDataPort) {
@@ -179,7 +178,7 @@ define([
 
                 this.invalidated[desc.id] = true;
                 this._client.startTransaction(msg);
-                this._client.delMoreNodes([desc.id]);
+                this._client.deleteNode(desc.id);
                 this._client.completeTransaction();
                 return false;
             }
@@ -198,7 +197,7 @@ define([
 
         if(this.addedIds[gmeId]) {
             delete this.addedIds[gmeId];
-            return EasyDAGControl.prototype._onUnload.call(this, gmeId);
+            return ThumbnailControl.prototype._onUnload.call(this, gmeId);
         }
     };
 
@@ -239,8 +238,13 @@ define([
     PipelineEditorControl.prototype.getValidOutputs = function (inputId, outputs) {
         // Valid input if one of the isTypeOf(<output>, inputId)
         // for at least one output
-        var inputType = this._client.getNode(inputId).getMetaTypeId();
-        return outputs.filter(type => this._client.isTypeOf(type, inputType)).length;
+        var inputType = this._client.getNode(inputId).getMetaTypeId(),
+            node;
+
+        return outputs.filter(type => {
+            node = this._client.getNode(type);
+            return node.isTypeOf(inputType);
+        }).length;
     };
 
     PipelineEditorControl.prototype._getValidSuccessorNodes = function (nodeId) {
@@ -283,7 +287,7 @@ define([
         msg = `Disconnecting ${names[0]} of ${names[1]} from ${names[2]} of ${names[3]}`;
 
         this._client.startTransaction(msg);
-        this._client.delMoreNodes([id]);
+        this._client.deleteNode(id);
         this._client.completeTransaction();
     };
 
@@ -325,10 +329,13 @@ define([
                 return [
                     node.getId(),
                     dstPorts.filter(id => {
-                        var typeId = this._client.getNode(id).getMetaTypeId();
+                        var typeId = this._client.getNode(id).getMetaTypeId(),
+                            portTypeNode = this._client.getNode(portType),
+                            typeNode = this._client.getNode(typeId);
+
                         return isOutput ?
-                            this._client.isTypeOf(portType, typeId) :
-                            this._client.isTypeOf(typeId, portType);
+                            portTypeNode.isTypeOf(typeId) :
+                            typeNode.isTypeOf(portType);
                     })
                 ];
             };
@@ -352,12 +359,12 @@ define([
 
         this._client.startTransaction(msg);
 
-        connId = this._client.createChild({
+        connId = this._client.createNode({
             parentId: this._currentNodeId,
             baseId: this.getConnectionId()
         });
-        this._client.makePointer(connId, CONN.SRC, srcId);
-        this._client.makePointer(connId, CONN.DST, dstId);
+        this._client.setPointer(connId, CONN.SRC, srcId);
+        this._client.setPointer(connId, CONN.DST, dstId);
 
         this._client.completeTransaction();
     };
@@ -369,14 +376,16 @@ define([
         // the dst operation
         var result = [],
             ipairs = inputs.map(id => [id, this._client.getNode(id).getMetaTypeId()]),
-            oType;
+            oType,
+            oTypeId;
 
         // For each output, get all possible (valid) input destinations
         outputs.forEach(outputId => {
-            oType = this._client.getNode(outputId).getMetaTypeId();
+            oTypeId = this._client.getNode(outputId).getMetaTypeId();
+            oType = this._client.getNode(oTypeId);
             result = result.concat(ipairs.filter(pair =>
                     // output type should be valid input type
-                    this._client.isTypeOf(oType, pair[1])
+                    oType.isTypeOf(pair[1])
                 )
                 .map(pair => [outputId, pair[0]])  // Get the input data id
             );
@@ -587,7 +596,7 @@ define([
         if (this.executionUI) {
             this._client.removeUI(this, this.executionEvents.bind(this));
         }
-        EasyDAGControl.prototype._detachClientEventListeners.call(this);
+        ThumbnailControl.prototype._detachClientEventListeners.call(this);
     };
 
     ////////////////////// Execution Support END //////////////////////
@@ -628,37 +637,15 @@ define([
         this._deleteTag(name);  // Remove execution tag
         if (this.isRunning(node)) {
             this.silentStopExecution(id, true).then(() => {
-                this._client.delMoreNodes([id]);
+                this._client.deleteNode(id);
                 this._client.completeTransaction();
             });
         } else {
-            this._client.delMoreNodes([id]);
+            this._client.deleteNode(id);
             this._client.completeTransaction();
         }
     };
 
-    PipelineEditorControl.prototype.updateThumbnail = function (svg) {
-        var node = this._client.getNode(this._currentNodeId),
-            name,
-            attrs,
-            currentThumbnail,
-            attrName = 'thumbnail',
-            msg;
-
-        if (node) {  // may have been deleted
-            name = node.getAttribute('name');
-            attrs = node.getValidAttributeNames();
-            currentThumbnail = node.getAttribute(attrName);
-            msg = `Updating pipeline thumbnail for "${name}"`;
-
-            if (attrs.indexOf(attrName) > -1 && currentThumbnail !== svg) {
-                this._client.startTransaction(msg);
-                this._client.setAttributes(this._currentNodeId, attrName, svg);
-                this._client.completeTransaction();
-            }
-        }
-    };
-
     ////////////////////// Criterion Support //////////////////////
     PipelineEditorControl.prototype._getValidTargetsFor = function (id, ptr) {
         // Check if the pointer is a Criterion pointer -> if so, only show the meta types
@@ -672,8 +659,8 @@ define([
         if (criterion) {
             // Get all criterion types
             criterionId = criterion.getId();
-            items = this._client.getAllMetaNodes().map(node => node.getId())
-                .filter(id => this._client.isTypeOf(id, criterionId));
+            items = this._client.getAllMetaNodes()
+                .filter(node => node.isTypeOf(criterionId));
 
             return items.map(id => {
                 return {
@@ -681,7 +668,7 @@ define([
                 };
             });
         } else {
-            return EasyDAGControl.prototype._getValidTargetsFor.apply(this, arguments);
+            return ThumbnailControl.prototype._getValidTargetsFor.apply(this, arguments);
         }
     };
 

src/visualizers/panels/PipelineIndex/PipelineIndexControl.js

@@ -54,7 +54,7 @@ define([
                 }
 
                 this._client.startTransaction(msg);
-                this._client.delMoreNodes(ids);
+                this._client.deleteNodes(ids);
                 this._client.completeTransaction();
 
                 this._client.removeUI(delUI);
@@ -68,7 +68,7 @@ define([
 
             if (oldName !== name && !/^\s*$/.test(name)) {
                 this._client.startTransaction(msg);
-                this._client.setAttributes(id, 'name', name);
+                this._client.setAttribute(id, 'name', name);
                 this._client.completeTransaction();
             }
         };
@@ -102,20 +102,20 @@ define([
     // This next function retrieves the relevant node information for the widget
     PipelineIndexControl.prototype._getObjectDescriptor = function (nodeId) {
         var node = this._client.getNode(nodeId),
+            base,
             objDescriptor;
 
         if (node) {
+            base = this._client.getNode(node.getBaseId());
             objDescriptor = {
-                id: undefined,
-                name: undefined,
-                parentId: undefined,
+                id: node.getId(),
+                name: node.getAttribute(nodePropertyNames.Attributes.name),
+                parentId: node.getParentId(),
                 thumbnail: node.getAttribute('thumbnail'),
+                type: base.getAttribute('name'),
                 executionCount: node.getMemberIds('executions').length
             };
 
-            objDescriptor.id = node.getId();
-            objDescriptor.name = node.getAttribute(nodePropertyNames.Attributes.name);
-            objDescriptor.parentId = node.getParentId();
         }
 
         return objDescriptor;

src/visualizers/panels/Sidebar/SidebarPanel.js

@@ -0,0 +1,170 @@
+/*globals define, WebGMEGlobal*/
+/*jshint browser: true*/
+
+define([
+    'js/Constants',
+    'js/PanelBase/PanelBase',
+    'panels/AutoViz/AutoVizPanel',
+    'widgets/Sidebar/SidebarWidget',
+    'deepforge/globals',
+    'q'
+], function (
+    CONSTANTS,
+    PanelBase,
+    AutoVizPanel,
+    SidebarWidget,
+    DeepForge,
+    Q
+) {
+    'use strict';
+
+    var SidebarPanel,
+        CATEGORY_TO_PLACE = {
+            pipelines: 'MyPipelines',
+            executions: 'MyExecutions',
+            architectures: 'MyArchitectures',
+            artifacts: 'MyArtifacts'
+        };
+
+    SidebarPanel = function (layoutManager, params) {
+        var opts = {};
+        opts[PanelBase.OPTIONS.LOGGER_INSTANCE_NAME] = 'SidebarPanel';
+        PanelBase.call(this, opts);
+
+        this._client = params.client;
+        this._embedded = params.embedded;
+
+        this._lm = layoutManager;
+        this._params = params;
+        this._panels = {};
+        this._initialize();
+
+        this.logger.debug('ctor finished');
+    };
+
+    SidebarPanel.prototype = Object.create(PanelBase.prototype);
+    SidebarPanel.prototype._initialize = function () {
+        this.widget = new SidebarWidget(this.logger, this.$el);
+        this.widget.getProjectName = this.getProjectName.bind(this);
+        this.widget.updateLibraries = this.updateLibraries.bind(this);
+        this.widget.checkLibUpdates = this.checkLibUpdates.bind(this);
+        this.widget.setEmbeddedPanel = this.setEmbeddedPanel.bind(this);
+
+        this.onActivate();
+    };
+
+    SidebarPanel.prototype._stateActiveBranchChanged = function (model, branchId) {
+        if (branchId) {
+            this.widget.checkLibraries();
+        }
+    };
+
+    SidebarPanel.prototype.setEmbeddedPanel = function (category) {
+        var placeName = CATEGORY_TO_PLACE[category];
+
+        return DeepForge.places[placeName]()
+            .then(nodeId => WebGMEGlobal.State.registerActiveObject(nodeId));
+    };
+
+    SidebarPanel.prototype.selectedObjectChanged = function (nodeId) {
+        var categories,
+            category,
+            place;
+
+        if (typeof nodeId === 'string') {
+            categories = Object.keys(CATEGORY_TO_PLACE);
+            
+            Q.all(categories.map(category => {
+                place = CATEGORY_TO_PLACE[category];
+                return DeepForge.places[place]();
+            }))
+            .then(nodeIdPrefixes => {
+                for (var i = nodeIdPrefixes.length; i--;) {
+                    if (nodeId.indexOf(nodeIdPrefixes[i]) > -1) {
+                        category = categories[i];
+                        return this.widget.highlight(category);
+                    }
+                }
+            });
+        }
+    };
+
+    /* OVERRIDE FROM WIDGET-WITH-HEADER */
+    SidebarPanel.prototype.onResize = function (width, height) {
+        var navWidth,
+            embeddedWidth;
+
+        this.logger.debug('onResize --> width: ' + width + ', height: ' + height);
+        navWidth = this.widget.width();
+        embeddedWidth = width-navWidth;
+        if (this.embeddedPanel) {
+            this.$embedded.css({
+                width: embeddedWidth,
+                height: height,
+                left: navWidth,
+                margin: 'inherit'
+            });
+            this.embeddedPanel.onResize(embeddedWidth, height);
+        }
+        this.width = width;
+        this.height = height;
+    };
+
+    /* * * * * * * * Visualizer life cycle callbacks * * * * * * * */
+    SidebarPanel.prototype.destroy = function () {
+        this.widget.destroy();
+        this.$el.remove();
+        WebGMEGlobal.KeyboardManager.setListener(undefined);
+        WebGMEGlobal.Toolbar.refresh();
+    };
+
+    SidebarPanel.prototype._stateActiveObjectChanged = function (model, activeObjectId) {
+        this.selectedObjectChanged(activeObjectId);
+    };
+
+    SidebarPanel.prototype.onActivate = function () {
+        WebGMEGlobal.State.on('change:' + CONSTANTS.STATE_ACTIVE_OBJECT,
+            this._stateActiveObjectChanged, this);
+        WebGMEGlobal.State.on('change:' + CONSTANTS.STATE_ACTIVE_BRANCH_NAME,
+            this._stateActiveBranchChanged, this);
+        this.widget.onActivate();
+        WebGMEGlobal.KeyboardManager.setListener(this.widget);
+        WebGMEGlobal.Toolbar.refresh();
+    };
+
+    SidebarPanel.prototype.onDeactivate = function () {
+        WebGMEGlobal.State.off('change:' + CONSTANTS.STATE_ACTIVE_OBJECT,
+            this._stateActiveObjectChanged);
+        WebGMEGlobal.State.off('change:' + CONSTANTS.STATE_ACTIVE_BRANCH_NAME,
+            this._stateActiveBranchChanged, this);
+        this.widget.onDeactivate();
+        WebGMEGlobal.KeyboardManager.setListener(undefined);
+        WebGMEGlobal.Toolbar.refresh();
+    };
+
+    /* * * * * * * * Library Updates * * * * * * * */
+
+    SidebarPanel.prototype.getProjectName = function () {
+        var projectId = this._client.getActiveProjectId();
+        return projectId && projectId.split('+')[1];
+    };
+
+    SidebarPanel.prototype.checkLibUpdates = function () {
+        var pluginId = 'CheckLibraries',
+            context = this._client.getCurrentPluginContext(pluginId);
+
+        return Q.ninvoke(this._client, 'runServerPlugin', pluginId, context)
+            .then(res => {
+                return res.messages.map(msg => msg.message.split(' '));
+            });
+    };
+
+    SidebarPanel.prototype.updateLibraries = function (libraries) {
+        var promises = libraries
+            .map(lib => Q.ninvoke(this._client, 'updateLibrary', lib[0], lib[1]));
+
+        return Q.all(promises);
+    };
+
+    return SidebarPanel;
+});

src/visualizers/panels/TextEditor/TextEditorControl.js

@@ -55,7 +55,7 @@ define([
         if (!inTransaction) {
             this._client.startTransaction(msg);
         }
-        this._client.setAttributes(id, this.ATTRIBUTE_NAME, text);
+        this._client.setAttribute(id, this.ATTRIBUTE_NAME, text);
         if (!inTransaction) {
             this._client.completeTransaction();
         }
@@ -67,7 +67,7 @@ define([
             msg = `Renaming ${oldName} -> ${name}`;
 
         this._client.startTransaction(msg);
-        this._client.setAttributes(this._currentNodeId, 'name', name);
+        this._client.setAttribute(this._currentNodeId, 'name', name);
         this._client.completeTransaction();
     };
 

src/visualizers/panels/WorkerHeader/NodePathNavWithHiddenNodes.js

@@ -0,0 +1,82 @@
+/* globals define, WebGMEGlobal*/
+define([
+    'js/Constants',
+    'panels/BreadcrumbHeader/NodePathNavigator'
+], function(
+    CONSTANTS,
+    NodePathNavigator
+) {
+    var PATH_SEP = '/';
+    var NodePathWithHidden = function() {
+        NodePathNavigator.apply(this, arguments);
+    };
+
+    NodePathWithHidden.prototype = Object.create(NodePathNavigator.prototype);
+
+    NodePathWithHidden.prototype.getNodePath = function() {
+        var nodeIds = NodePathNavigator.prototype.getNodePath.apply(this, arguments),
+            lastRootChildIndex = -1,
+            pathSepRegex = new RegExp(PATH_SEP, 'g'),
+            i;
+
+        // Treat any nodeIds in the root object as the same node then remove them
+        // Hide any nodeIds in the root object
+        for (i = nodeIds.length; i-- && lastRootChildIndex === -1;) {
+            // Check for multiple '/' separators in the id (else it's a child of
+            // the root node)
+            if (nodeIds[i] && nodeIds[i].match(pathSepRegex).length === 1) {
+                lastRootChildIndex = i;
+            }
+        }
+
+        if (lastRootChildIndex > -1) {
+            for (i = 1; i <= lastRootChildIndex; i++) {
+                delete this.territories[nodeIds[i]];
+            }
+            nodeIds.splice(1, lastRootChildIndex);
+        }
+
+        return nodeIds;
+    };
+
+    NodePathWithHidden.prototype.addNode = function(id, isActive) {
+        if (id === CONSTANTS.PROJECT_ROOT_ID && !isActive) {
+            var item = document.createElement('li'),
+                anchor = document.createElement('a');
+
+            this._nodes[id] = anchor;
+            item.appendChild(anchor);
+            item.addEventListener('click', () => {
+                var nodeId = this._nodeHistory[1],
+                    node;
+
+                if (nodeId) {
+                    // Get the id for the child of the root node
+                    node = this.client.getNode(nodeId);
+                    if (node.getParentId() !== CONSTANTS.PROJECT_ROOT_ID) {
+                        nodeId = node.getParentId();
+                    }
+                } else {
+                    // Try to load the 'MyPipelines' child of the root node
+                    node = this.client.getNode(CONSTANTS.PROJECT_ROOT_ID)
+                        // Get the child nodes
+                        .getChildrenIds().map(id => this.client.getNode(id))
+                        // Find the child named 'MyPipelines'
+                        .find(child => child && child.getAttribute('name') === 'MyPipelines');
+
+                    if (node) {
+                        nodeId = node.getId();
+                    }
+                }
+                // If none are loaded, try to register MyPipelines
+                WebGMEGlobal.State.registerActiveObject(nodeId || id);
+            });
+            this.territories[id] = {children: 0};
+            this.pathContainer.append(item);
+        } else {
+            return NodePathNavigator.prototype.addNode.apply(this, arguments);
+        }
+    };
+
+    return NodePathWithHidden;
+});

src/visualizers/panels/WorkerHeader/ProjectNavigatorController.js

@@ -0,0 +1,106 @@
+/* globals define, WebGMEGlobal */
+define([
+    'js/Dialogs/Projects/ProjectsDialog',
+    './WorkerDialog',
+    'js/Panels/Header/ProjectNavigatorController'
+], function(
+    ProjectsDialog,
+    WorkerDialog,
+    GMEProjectNavigatorController
+) {
+    'use strict';
+    var ProjectNavigatorController = function() {
+        GMEProjectNavigatorController.apply(this, arguments);
+    };
+
+    ProjectNavigatorController.prototype = Object.create(GMEProjectNavigatorController.prototype);
+
+    ProjectNavigatorController.prototype.initialize = function () {
+        var self = this,
+            newProject,
+            manageProjects,
+            manageWorkers;
+
+
+        // initialize model structure for view
+        self.$scope.navigator = {
+            items: [],
+            separator: true
+        };
+
+
+        manageProjects = function (/*data*/) {
+            var pd = new ProjectsDialog(self.gmeClient);
+            pd.show();
+        };
+        newProject = function (data) {
+            var pd = new ProjectsDialog(self.gmeClient, true, data.newType);
+            pd.show();
+        };
+        self.userId = WebGMEGlobal.userInfo._id;
+
+        manageWorkers = function() {
+            // Create the worker dialog
+            var pd = new WorkerDialog(self.logger);
+            pd.show();
+        };
+
+        // initialize root menu
+        // projects id is mandatory
+        if (self.config.disableProjectActions === false) {
+            self.root.menu = [
+                {
+                    id: 'top',
+                    items: [
+                        {
+                            id: 'manageProject',
+                            label: 'Manage projects ...',
+                            iconClass: 'glyphicon glyphicon-folder-open',
+                            action: manageProjects,
+                            actionData: {}
+                        },
+                        {
+                            id: 'newProject',
+                            label: 'New project ...',
+                            disabled: WebGMEGlobal.userInfo.canCreate !== true,
+                            iconClass: 'glyphicon glyphicon-plus',
+                            action: newProject,
+                            actionData: {newType: 'seed'}
+                        },
+                        {
+                            id: 'importProject',
+                            label: 'Import project ...',
+                            disabled: WebGMEGlobal.userInfo.canCreate !== true,
+                            iconClass: 'glyphicon glyphicon-import',
+                            action: newProject,
+                            actionData: {newType: 'import'}
+                        },
+                        {
+                            id: 'manageWorkers',
+                            label: 'View workers ...',
+                            iconClass: 'glyphicon glyphicon-cloud',
+                            action: manageWorkers
+                        }
+                    ]
+                },
+                {
+                    id: 'projects',
+                    label: 'Recent projects',
+                    totalItems: 20,
+                    items: [],
+                    showAllItems: manageProjects
+                }
+            ];
+        }
+
+        self.initWithClient();
+
+        // only root is selected by default
+        self.$scope.navigator = {
+            items: self.config.disableProjectActions ? [] : [self.root],
+            separator: true
+        };
+    };
+
+    return ProjectNavigatorController;
+});

src/visualizers/panels/WorkerHeader/WorkerDialog.js

@@ -0,0 +1,239 @@
+/* globals define, $ */
+define([
+    'q',
+    'superagent',
+    'deepforge/viz/Utils',
+    'deepforge/api/JobOriginClient',
+    'text!./WorkerModal.html',
+    'text!./WorkerTemplate.html.ejs',
+    'text!./WorkerJobItem.html',
+    'css!./WorkerModal.css'
+], function(
+    Q,
+    superagent,
+    utils,
+    JobOriginClient,
+    WorkerHtml,
+    WorkerTemplate,
+    WorkerJobItem
+) {
+    'use strict';
+
+    var WORKER_ENDPOINT = '/rest/executor/worker',
+        JOBS_ENDPOINT = '/rest/executor';
+
+    var WorkerDialog = function(logger) {
+        this.workerDict = {};
+        this.workers = {};
+        this.runningWorkers = [];
+        this.jobsDict = {};
+        this.jobs = {};
+        this.active = false;
+        this.logger = logger.fork('WorkerDialog');
+        this.originManager = new JobOriginClient({
+            logger: this.logger
+        });
+    };
+
+    WorkerDialog.prototype.initialize = function() {
+        this._dialog = $(WorkerHtml);
+        this._table = this._dialog.find('.worker-list');
+        this.$noJobs = this._dialog.find('.no-jobs-msg');
+        this.$noWorkers = this._dialog.find('.no-workers-msg');
+        this._isShowingJobs = false;
+        this._isShowingWorkers = true;
+        this._queue = this._dialog.find('.job-queue-list');
+        this._dialog.modal('show');
+        this._dialog.on('hidden.bs.modal', () => this.active = false);
+    };
+
+    WorkerDialog.prototype.show = function() {
+        this.active = true;
+        this.update();
+        this.initialize();
+    };
+
+    WorkerDialog.prototype.get = function(url) {
+        var deferred = Q.defer();
+
+        superagent.get(url)
+            .end((err, res) => {
+                if (err) {
+                    return deferred.reject(err);
+                }
+                deferred.resolve(JSON.parse(res.text));
+            });
+
+        return deferred.promise;
+    };
+
+    WorkerDialog.prototype.update = function() {
+        // Poll the workers
+        return Q.all([
+            this.get(WORKER_ENDPOINT).then(workers => this.updateWorkers(workers)),
+            this.get(JOBS_ENDPOINT).then(jobs => this.updateJobs(jobs))
+        ]).then(() => {
+            if (this.active) {
+                setTimeout(this.update.bind(this), 1000);
+            }
+        })
+        .catch(err => this.logger.error('Update failed:', err));
+    };
+
+    WorkerDialog.prototype.updateWorkers = function(workerDict) {
+        var ids = Object.keys(workerDict),
+            oldWorkerIds,
+            visibleWorkers = false,
+            i;
+
+        this.runningWorkers = [];
+        for (i = ids.length; i--;) {
+            this.updateWorker(workerDict[ids[i]]);
+            visibleWorkers = true;
+            delete this.workerDict[ids[i]];
+        }
+        this.toggleNoWorkersMsg(!visibleWorkers);
+
+        // Clear old workers
+        oldWorkerIds = Object.keys(this.workerDict);
+        for (i = oldWorkerIds.length; i--;) {
+            this.removeWorker(oldWorkerIds[i]);
+        }
+
+        this.workerDict = workerDict;
+    };
+
+    WorkerDialog.prototype.updateWorker = function(worker) {
+        var row = this.workers[worker.clientId] || $(WorkerTemplate),
+            clazz;
+
+        worker.lastSeen = utils.getDisplayTime(worker.lastSeen*1000);
+        worker.status = worker.jobs.length ? 'RUNNING' : 'READY';
+
+        clazz = worker.status === 'RUNNING' ? 'warning' : 'success';
+        row[0].className = clazz;
+
+        row.find('.lastSeen').text(worker.lastSeen);
+        row.find('.clientId').text(worker.clientId);
+        row.find('.status').text(worker.status);
+        if (!this.workers[worker.clientId]) {
+            this._table.append(row);
+            this.workers[worker.clientId] = row;
+        }
+
+        if (worker.status === 'RUNNING') {
+            this.runningWorkers.push(worker);
+        }
+    };
+
+    WorkerDialog.prototype.removeWorker = function(workerId) {
+        this.workers[workerId].remove();
+        delete this.workers[workerId];
+    };
+
+    WorkerDialog.prototype.updateJobs = function(jobsDict) {
+        var allJobIds = Object.keys(jobsDict),
+            hasJobs = false,
+            id;
+
+        this.jobsDict = jobsDict;
+        for (var i = allJobIds.length; i--;) {
+            id = allJobIds[i];
+            if (this.jobs[id] || !this.isFinished(id)) {
+                hasJobs = this.updateJobItem(id) || hasJobs;
+            }
+        }
+        this.setNoJobsMessage(!hasJobs);  // hide if no queue
+    };
+
+    WorkerDialog.prototype.setNoJobsMessage = function(visible) {
+        var visibility = visible ? 'inherit' : 'none',
+            wasVisible = !this._isShowingJobs;
+
+        if (visible !== wasVisible) {
+            this.$noJobs.css('display', visibility);
+            this._isShowingJobs = !visible;
+        }
+    };
+
+    WorkerDialog.prototype.toggleNoWorkersMsg = function(visible) {
+        var visibility = visible ? 'inherit' : 'none';
+
+        if (visible !== this._isShowingWorkers) {
+            this.$noWorkers.css('display', visibility);
+            this._isShowingWorkers = visible;
+        }
+    };
+
+    WorkerDialog.prototype.isFinished = function(jobId) {
+        return this.jobsDict[jobId].status === 'FAILED_TO_EXECUTE' ||
+            this.jobsDict[jobId].status === 'SUCCESS' ||
+            this.jobsDict[jobId].status === 'CANCELED';
+    };
+
+    WorkerDialog.prototype.updateJobItemName = function(jobId) {
+        return this.originManager.getOrigin(jobId)
+            .then(info => {
+                var job = this.jobs[jobId],
+                    project = info.project.replace(/^guest\+/, '');
+
+                if (job && this.active) {
+                    if (info.branch !== 'master') {
+                        project += ' (' + info.branch + ')';
+                    }
+                    job.find('.job-id').text(info.job);
+                    job.find('.execution').text(info.execution);
+                    job.find('.project').text(project);
+                }
+            });
+    };
+
+    WorkerDialog.prototype.getWorkerWithJob = function(jobId) {
+        var jobs;
+
+        for (var i = this.runningWorkers.length; i--;) {
+            jobs = this.runningWorkers[i].jobs;
+            for (var j = jobs.length; j--;) {
+                if (jobs[j].hash === jobId) {
+                    return this.runningWorkers[i].clientId;
+                }
+            }
+        }
+
+        return 'unknown';
+    };
+
+    WorkerDialog.prototype.updateJobItem = function(jobId) {
+        var job = this.jobs[jobId] || $(WorkerJobItem),
+            info = this.jobsDict[jobId],
+            createdTime = new Date(info.createTime).getTime(),
+            clazz = utils.ClassForJobStatus[info.status.toLowerCase()],
+            status = info.status;
+
+        job[0].className = `job-tag ${clazz}`;
+
+        // Add the worker id if running
+        if (info.status.toLowerCase() === 'running') {
+            var workerId = this.getWorkerWithJob(jobId);
+            status += ' (' + workerId + ')';
+        }
+        job.find('.status').text(status);
+
+        if (!this.jobs[jobId]) {
+            job.find('.job-id').text('Loading');
+            job.find('.createdAt').text(utils.getDisplayTime(createdTime));
+            this.updateJobItemName(jobId);
+            this._queue.append(job);
+            this.jobs[jobId] = job;
+        }
+
+        if (this.isFinished(jobId)) {
+            job.remove();
+            delete this.jobs[jobId];
+            return false;
+        }
+        return true;
+    };
+
+    return WorkerDialog;
+});

src/visualizers/panels/WorkerHeader/WorkerHeaderPanel.js

@@ -0,0 +1,54 @@
+/*globals define, angular, _, $, WebGMEGlobal*/
+/*jshint browser: true*/
+
+define([
+    'js/Panels/Header/HeaderPanel',
+    'panels/BreadcrumbHeader/BreadcrumbHeaderPanel',
+    'js/Widgets/UserProfile/UserProfileWidget',
+    'js/Widgets/ConnectedUsers/ConnectedUsersWidget',
+    'js/Panels/Header/DefaultToolbar',
+    './NodePathNavWithHiddenNodes',
+    'js/Toolbar/Toolbar',
+    './ProjectNavigatorController'
+], function (
+    HeaderBase,
+    BreadcrumbHeader,
+    UserProfileWidget,
+    ConnectedUsersWidget,
+    DefaultToolbar,
+    NodePathNavWithHiddenNodes,
+    Toolbar,
+    ProjectNavigatorController
+) {
+    'use strict';
+
+    var HeaderPanel;
+
+    HeaderPanel = function (layoutManager, params) {
+        BreadcrumbHeader.call(this, layoutManager, params);
+    };
+
+    //inherit from PanelBaseWithHeader
+    _.extend(HeaderPanel.prototype, BreadcrumbHeader.prototype);
+
+    HeaderPanel.prototype._initialize = function () {
+        HeaderBase.prototype._initialize.call(this);
+        var app = angular.module('gmeApp'),
+            nodePath = new NodePathNavWithHiddenNodes({
+                container: $('<div/>', {class: 'toolbar-container'}),
+                client: this._client,
+                logger: this.logger
+            });
+
+        app.controller('ProjectNavigatorController', ['$scope', 'gmeClient', '$timeout', '$window', '$http',
+            ProjectNavigatorController]);
+
+        this.$el.find('.toolbar-container').remove();
+        this.$el.append(nodePath.$el);
+
+        WebGMEGlobal.Toolbar = Toolbar.createToolbar($('<div/>'));
+        new DefaultToolbar(this._client);
+    };
+
+    return HeaderPanel;
+});

src/visualizers/panels/WorkerHeader/WorkerJobItem.html

@@ -0,0 +1,7 @@
+<tr>
+    <td class="job-id"></td>
+    <td class="execution">unknown</td>
+    <td class="project">unknown</td>
+    <td class="createdAt">unknown</td>
+    <td class="status">unknown</td>
+</tr>

src/visualizers/panels/WorkerHeader/WorkerModal.css

@@ -0,0 +1,47 @@
+.worker-modal .modal-content .modal-header span {
+  font-size: 28px;
+  vertical-align: middle;
+}
+
+.worker-modal .modal-content .modal-header .header-icon {
+  height: 28px;
+  width: 28px;
+  margin-right: 1ex;
+  display: inline-block;
+  vertical-align: middle;
+  background-size: 28px 28px;
+}
+
+.worker-modal .modal-content .modal-body th {
+  text-align: left;
+}
+
+.worker-modal .job-tag {
+  margin-right: 5px;
+}
+
+.worker-modal .queue-title {
+  float: left;
+  margin-right: 5px;
+  font-size: 1.4em;
+  color: #555555;
+  font-weight: bold;
+  width: 100%;
+  text-align: center;
+}
+
+.worker-modal .no-jobs-msg {
+  font-style: italic;
+  font-size: 1.3em;
+  color: #777;
+}
+
+.worker-modal .no-workers-msg {
+  font-style: italic;
+  font-size: 1.3em;
+  color: #777;
+}
+
+.worker-modal .job-queue {
+  padding-top: 1em;
+}

src/visualizers/panels/WorkerHeader/WorkerModal.html

@@ -0,0 +1,60 @@
+<div class="worker-modal modal fade in" tabindex="-1" role="dialog">
+    <div class="modal-dialog modal-lg">
+        <div class="modal-content">
+            <div class="modal-header">
+                <button type="button" class="close" data-dismiss="modal">x</button>
+                <i class="header-icon gme-icon"></i>
+                <span>Overview</span>
+            </div>
+            <div class="modal-body">
+                <div>
+                    <div class="queue-title">Connected Workers</div>
+                    <table class="table table-projects">
+                        <thead>
+                            <tr class="with-children">
+                                <th class="title-owner">Worker Id
+                                    <!--<i class="glyphicon glyphicon-sort-by-attributes-alt sorted in-order"></i>
+                                    <i class="glyphicon glyphicon-sort-by-attributes sorted rev-order"></i>-->
+                                </th>
+                                <th class="title-name">Last Seen
+                                    <!--<i class="glyphicon glyphicon-sort-by-attributes-alt sorted in-order"></i>
+                                    <i class="glyphicon glyphicon-sort-by-attributes sorted rev-order"></i>-->
+                                </th>
+                                <th class="title-modified">Status
+                                    <!--<i class="glyphicon glyphicon-sort-by-attributes-alt sorted in-order"></i>
+                                    <i class="glyphicon glyphicon-sort-by-attributes sorted rev-order"></i>-->
+                                </th>
+                            </tr>
+                            <!--<tr class="with-no-children">
+                                <th>No projects in this group...</th>
+                            </tr>-->
+                        </thead>
+                        <tbody class="worker-list">
+                            <tr><td class="no-workers-msg">No Connected Workers...</td></tr>
+                        </tbody>
+                    </table>
+                </div>
+                <div class="job-queue">
+                    <div class="queue-title">Job Queue</div>
+                    <table class="table table-projects">
+                        <thead>
+                            <tr class="with-children">
+                                <th>Job</th>
+                                <th>Execution</th>
+                                <th>Project</th>
+                                <th>Creation Date</th>
+                                <th>Status</th>
+                            </tr>
+                            <!--<tr class="with-no-children">-->
+                                <!--<th>No queued jobs</th>-->
+                            <!--</tr>-->
+                        </thead>
+                        <tbody class="job-queue-list">
+                            <tr><td class="no-jobs-msg">No Running Jobs...</td></tr>
+                        </tbody>
+                    </table>
+                </div>
+            </div>
+        </div>
+    </div>
+</div>