v0.21.0

* Removed custom disable checking. Fixes #917

* WIP #917 Removed unused variable

* WIP #917 removed unused var

.codeclimate.yml

@@ -31,6 +31,8 @@ exclude_paths:
 - test/
 - src/common/lua.js
 - src/common/js-yaml.min.js
+- src/visualizers/widgets/Sidebar/lib/
 - src/visualizers/widgets/TextEditor/lib/
 - src/visualizers/widgets/PipelineIndex/styles/PipelineIndex.css
 - src/visualizers/widgets/LineGraph/lib/
+- src/visualizers/widgets/PipelineEditor/klay.js

.gitignore

@@ -33,3 +33,5 @@ node_modules
 tmp/
 test-tmp/
 blob-local-storage/
+src/seeds/nn/hash.txt
+src/seeds/pipeline/hash.txt

.npmignore

@@ -0,0 +1,40 @@
+src/worker/tmp
+.env
+*.swp
+*.swo
+**.sass-cache
+# Logs
+logs
+*.log
+
+# Runtime data
+pids
+*.pid
+*.seed
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (http://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directory
+# https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git
+node_modules
+tmp/
+test-tmp/
+blob-local-storage/
+src/seeds/nn/hash.txt
+src/seeds/pipeline/hash.txt
+
+# docs
+images/

README.md

@@ -4,10 +4,22 @@
 [![Join the chat at https://gitter.im/dfst/deepforge](https://badges.gitter.im/dfst/deepforge.svg)](https://gitter.im/dfst/deepforge?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 [![Stories in Ready](https://badge.waffle.io/dfst/deepforge.png?label=ready&title=Ready)](https://waffle.io/dfst/deepforge)
 
-**Notice**: DeepForge is still a work in progress and is also lacking significant documentation! That being said, any contributions and/or feedback is greatly appreciated (and feel free to always ask any questions on the gitter)!
+**Notice**: DeepForge is still a work in progress and in beta! That being said, any contributions and/or feedback is greatly appreciated. If you have any questions, check out the [wiki](https://github.com/dfst/deepforge/wiki/) or drop me a line on the gitter!
+
 
 # DeepForge
-DeepForge is an open-source visual development environment for deep learning. Currently, it supports Convolutional Neural Networks but we are planning on supporting additional deep learning classifiers such as RNNs and LSTMs. Additional features include real-time collaborative editing and version control.
+DeepForge is an open-source visual development environment for deep learning providing end-to-end support for creating deep learning models. This is achieved through providing the ability to design **architectures**, create training **pipelines**, and then execute these pipelines over a cluster. Using a notebook-esque api, users can get real-time feedback about the status of any of their **executions** including compare them side-by-side in real-time.
+
+![overview](images/overview.png "")
+
+Additional features include:
+- Graphical architecture editor
+- Training/testing pipeline creation
+- Distributed pipeline execution
+- Real-time pipeline feedback
+- Collaborative editing
+- Automatic version control.
+- Facilitates defining custom layers 
 
 ## Quick Start
 Simply run the following command to install deepforge with its dependencies:
@@ -22,9 +34,9 @@ Or, if you already have NodeJS (v6) installed, simply run
 npm install -g deepforge
 ```
 
-Next, start deepforge with `deepforge start`!
+Finally, start deepforge with `deepforge start`and navigate to [http://localhost:8888](http://localhost:8888) to start using DeepForge! For more, detailed instructions, check out the [wiki](https://github.com/dfst/deepforge/wiki/Installation-Guide).
 
-Finally, navigate to [http://localhost:8888](http://localhost:8888) to start using DeepForge! For more, detailed instructions, check out the [wiki](https://github.com/dfst/deepforge/wiki/Installation-Guide).
+**Note**: running deepforge w/ `deepforge start` will also require [MongoDB](https://www.mongodb.com/download-center?jmp=nav#community) to be installed locally.
 
 Also, be sure to check out the other available features of the `deepforge` cli; it can be used to update, manage your torch installation, uninstall deepforge and run individual components!
 

app.js

@@ -3,10 +3,22 @@
 
 var gmeConfig = require('./config'),
     webgme = require('webgme'),
+    path = require('path'),
+    fs = require('fs'),
+    rm_rf = require('rimraf'),
+    gracefulFs = require('graceful-fs'),
     myServer;
 
+process.chdir(__dirname);
 webgme.addToRequireJsPaths(gmeConfig);
 
+// Patch the 'fs' module to fix 'too many files open' error
+gracefulFs.gracefulify(fs);
+
+// Clear seed hash info
+['nn', 'pipeline'].map(lib => path.join(__dirname, 'src', 'seeds', lib, 'hash.txt'))
+    .forEach(file => rm_rf.sync(file));
+
 myServer = new webgme.standaloneServer(gmeConfig);
 myServer.start(function (err) {
     if (err) {

bin/deepforge

@@ -117,7 +117,7 @@ var isLocalUri = function(protocol, uri) {
         uri.indexOf(protocol + '://127.0.0.1') === 0;
 };
 
-var checkMongo = function(args) {
+var checkMongo = function(args, notSilent) {
     // check the webgme config
     var gmeConfig = require('../config'),
         mongoUri = gmeConfig.mongo.uri;
@@ -129,15 +129,26 @@ var checkMongo = function(args) {
             console.log('MongoDB is already running!');
         } catch (e) {  // no pIds
             console.log('Starting MongoDB...');
-            startMongo(args, true);
+            var match = mongoUri.match(/:([0-9]+)/),
+                port = '80';
+
+            if (match) {
+                port = match[1];
+            }
+
+            startMongo(args, port, !notSilent);
         }
+    } else if (notSilent) {
+        console.log(`Cannot start remote mongo locally: ${mongoUri}`);
+    } else {
+        console.log(`Using remote mongo: ${mongoUri}`);
     }
 };
 
-var startMongo = function(args, silent) {
-    var job = rawSpawn('mongod', ['--dbpath', p(config.mongo.dir)], {
-        cwd: process.env.HOME
-    });
+var startMongo = function(args, port, silent) {
+    var opts = ['--dbpath', p(config.mongo.dir), '--port', port],
+        job = rawSpawn('mongod', opts, {cwd: process.env.HOME});
+
     if (!silent) {
         job.stdout.on('data',
             data => process.stdout.write(data.toString()));
@@ -293,7 +304,7 @@ program.command('start')
         }
 
         if (args.mongo) {
-            startMongo(args);
+            checkMongo(args, true);
         }
 
         if (!args.server && !args.worker && !args.mongo) {

config/components.json

@@ -2,13 +2,29 @@
     "AutoViz": {
         "preloadIds": [
             "ArchEditor",
+            "ArchIndex",
+            "PipelineIndex",
             "PipelineEditor",
             "OperationEditor",
             "ExecutionView"
-        ]
+        ],
+        "visualizerOverrides": {
+            "": "ForwardViz",
+            "MyArtifacts": "ArtifactIndex",
+            "MyArchitectures": "ArchIndex",
+            "MyExecutions": "ExecutionIndex",
+            "MyPipelines": "PipelineIndex"
+        }
+    },
+    "PipelineEditor": {
+        "itemName": "operation"
+    },
+    "ExecutionView": {
+        "itemName": "job"
     },
     "ArchEditor": {
         "hotkeys": "none",
+        "itemName": "layer",
         "LayerColors": {}
     },
     "BreadcrumbHeader": {
@@ -16,17 +32,38 @@
         "cachePrefix": "deepforge-header"
     },
     "FloatingActionButton": {
-        "hideOnEmpty": true
+        "hideOnEmpty": true,
+        "pluginUIConfigs": {
+            "GenerateArchitecture": {
+                "icon": "description",
+                "hotkey": "shift enter",
+                "priority": -1
+            },
+            "ExecutePipeline": {
+                "icon": "play_arrow",
+                "hotkey": "shift enter",
+                "color": "green",
+                "priority": 1
+            },
+            "ImportTorch": {
+                "icon": "import_export",
+                "priority": -1
+            },
+            "GenerateExecFile": {
+                "icon": "play_for_work",
+                "priority": -1
+            }
+        }
     },
     "GenericUIProjectNavigatorController": {
         "rootMenuClass": "deepforge-logo",
         "rootDisplayName": "DeepForge"
     },
-    "CHFLayout": {
+    "SidebarLayout": {
         "panels": [
             {
-                "id": "Header",
-                "panel": "BreadcrumbHeader/BreadcrumbHeaderPanel",
+                "id": "WorkerHeader",
+                "panel": "WorkerHeader/WorkerHeaderPanel",
                 "container": "header",
                 "DEBUG_ONLY": false
             },
@@ -42,6 +79,12 @@
                 "container": "center",
                 "DEBUG_ONLY": false
             },
+            {
+                "id": "Sidebar",
+                "panel": "Sidebar/SidebarPanel",
+                "container": "sidebar",
+                "DEBUG_ONLY": false
+            },
             {
                 "id": "ForgeActionButton",
                 "panel": "ForgeActionButton/ForgeActionButton",

config/config.base.js

@@ -19,7 +19,6 @@ config.plugin.allowBrowserExecution = true;
 config.plugin.allowServerExecution = true;
 
 config.executor.enable = true;
-config.executor.clearOldDataAtStartUp = true;
 
 config.visualization.extraCss.push('deepforge/styles/global.css');
 

config/config.local.js

@@ -7,7 +7,6 @@ var config = require('./config.default'),
 
 // Turn up the worker polling rate
 config.executor.workerRefreshInterval = 150;
-config.executor.clearOldDataAtStartUp = true,
 
 validateConfig(config);
 module.exports = config;

config/config.test.js

@@ -5,9 +5,11 @@
 
 'use strict';
 
-var config = require('./config.default');
+var config = require('./config.default'),
+    path = require('path');
 
 config.server.port = 9001;
 config.mongo.uri = 'mongodb://127.0.0.1:27017/webgme_tests';
+config.blob.fsDir = path.join(__dirname, '..', 'test-tmp', 'blob');
 
-module.exports = config;
+module.exports = config;

config/config.webgme.js

@@ -6,33 +6,38 @@
 var config = require('webgme/config/config.default'),
     validateConfig = require('webgme/config/validator');
 
-
 // The paths can be loaded from the webgme-setup.json
-config.plugin.basePaths.push('src/plugins');
-config.plugin.basePaths.push('node_modules/webgme-simple-nodes/src/plugins');
-config.visualization.layout.basePaths.push('node_modules/webgme-chflayout/src/layouts');
-config.visualization.decoratorPaths.push('src/decorators');
-config.visualization.decoratorPaths.push('node_modules/webgme-easydag/src/decorators');
-config.seedProjects.basePaths.push('src/seeds/nn');
-config.seedProjects.basePaths.push('src/seeds/devTests');
-config.seedProjects.basePaths.push('src/seeds/devUtilTests');
-config.seedProjects.basePaths.push('src/seeds/pipeline');
-config.seedProjects.basePaths.push('src/seeds/devPipelineTests');
-config.seedProjects.basePaths.push('src/seeds/project');
-config.seedProjects.basePaths.push('src/seeds/cifar10');
-config.seedProjects.basePaths.push('src/seeds/xor');
+config.plugin.basePaths.push(__dirname + '/../src/plugins');
+config.plugin.basePaths.push(__dirname + '/../node_modules/webgme-simple-nodes/src/plugins');
+config.visualization.layout.basePaths.push(__dirname + '/../src/layouts');
+config.visualization.layout.basePaths.push(__dirname + '/../node_modules/webgme-chflayout/src/layouts');
+config.visualization.decoratorPaths.push(__dirname + '/../src/decorators');
+config.visualization.decoratorPaths.push(__dirname + '/../node_modules/webgme-easydag/src/decorators');
+config.seedProjects.basePaths.push(__dirname + '/../src/seeds/nn');
+config.seedProjects.basePaths.push(__dirname + '/../src/seeds/devTests');
+config.seedProjects.basePaths.push(__dirname + '/../src/seeds/devUtilTests');
+config.seedProjects.basePaths.push(__dirname + '/../src/seeds/pipeline');
+config.seedProjects.basePaths.push(__dirname + '/../src/seeds/devPipelineTests');
+config.seedProjects.basePaths.push(__dirname + '/../src/seeds/project');
+config.seedProjects.basePaths.push(__dirname + '/../src/seeds/cifar10');
+config.seedProjects.basePaths.push(__dirname + '/../src/seeds/xor');
+config.seedProjects.basePaths.push(__dirname + '/../src/seeds/devProject');
 
 
 
-config.visualization.panelPaths.push('node_modules/webgme-fab/src/visualizers/panels');
-config.visualization.panelPaths.push('node_modules/webgme-breadcrumbheader/src/visualizers/panels');
-config.visualization.panelPaths.push('node_modules/webgme-autoviz/src/visualizers/panels');
-config.visualization.panelPaths.push('node_modules/webgme-easydag/src/visualizers/panels');
-config.visualization.panelPaths.push('src/visualizers/panels');
+config.visualization.panelPaths.push(__dirname + '/../node_modules/webgme-fab/src/visualizers/panels');
+config.visualization.panelPaths.push(__dirname + '/../node_modules/webgme-breadcrumbheader/src/visualizers/panels');
+config.visualization.panelPaths.push(__dirname + '/../node_modules/webgme-autoviz/src/visualizers/panels');
+config.visualization.panelPaths.push(__dirname + '/../node_modules/webgme-easydag/src/visualizers/panels');
+config.visualization.panelPaths.push(__dirname + '/../src/visualizers/panels');
 
 
+config.rest.components['execution/logs'] = __dirname + '/../src/routers/JobLogsAPI/JobLogsAPI.js';
+config.rest.components['job/origins'] = __dirname + '/../src/routers/JobOriginAPI/JobOriginAPI.js';
+config.rest.components['execution/pulse'] = __dirname + '/../src/routers/ExecPulse/ExecPulse.js';
+
 // Visualizer descriptors
-config.visualization.visualizerDescriptors.push('./src/visualizers/Visualizers.json');
+config.visualization.visualizerDescriptors.push(__dirname + '/../src/visualizers/Visualizers.json');
 // Add requirejs paths
 config.requirejsPaths = {
   'EllipseDecorator': 'node_modules/webgme-easydag/src/decorators/EllipseDecorator',
@@ -54,7 +59,7 @@ config.requirejsPaths = {
   'widgets/FloatingActionButton': './node_modules/webgme-fab/src/visualizers/widgets/FloatingActionButton'
 };
 
-config.visualization.layout.default = 'CHFLayout';
+config.visualization.layout.default = 'SidebarLayout';
 config.mongo.uri = 'mongodb://127.0.0.1:27017/deepforge';
 validateConfig(config);
 module.exports = config;

install.sh

@@ -41,6 +41,16 @@ detect_profile() {
 }
 detect_profile
 
+set_node_version() {
+    # Install nodejs v6.2.1
+    echo "Installing NodeJS v6.2.1"
+    nvm install v6.2.1
+    nvm alias default v6.2.1
+
+    # Install npm@2
+    npm install npm@2 -g
+}
+
 command -v node >/dev/null 2>&1 || {
     # No node! Install nvm
     echo >&2 "NodeJS is not found. Installing (using nvm)...";
@@ -48,50 +58,15 @@ command -v node >/dev/null 2>&1 || {
     source $DETECTED_PROFILE
     . $NVM_DIR/nvm.sh
 
-    # Install nodejs v6.2.0
-    echo "Installing nodejs v6.2.0"
-    nvm install v6.2.0
-    nvm alias default v6.2.0
-
-    # Install npm@2
-    npm install npm@2 -g
+    set_node_version
 }
 
-command -v node >/dev/null 2>&1 || {
-    # No mongod!
-    echo >&2 "MongoDB is not found. Installing...";
-    if [[ `uname` == "Darwin" ]]; then
-        brew install mongodb
-    elif [[ "$(uname)" == 'Linux' ]]; then
-
-        if [[ -r /etc/os-release ]]; then
-            # this will get the required information without dirtying any env state
-            DIST_VERS="$( ( . /etc/os-release &>/dev/null
-                            echo "$ID $VERSION_ID") )"
-            DISTRO="${DIST_VERS%% *}" # get our distro name
-            VERSION="${DIST_VERS##* }" # get our version number
-        elif [[ -r /etc/lsb-release ]]; then
-            DIST_VERS="$( ( . /etc/lsb-release &>/dev/null
-                            echo "${DISTRIB_ID,,} $DISTRIB_RELEASE") )"
-            DISTRO="${DIST_VERS%% *}" # get our distro name
-            VERSION="${DIST_VERS##* }" # get our version number
-        else # well, I'm out of ideas for now
-            echo '==> Failed to determine distro and version.'
-            exit 1
-        fi
-
-        # Detect archlinux
-        if [[ "$DISTRO" = "arch" ]]; then
-            distribution="archlinux"
-            sudo pacman -S mongodb
-        # Detect Ubuntu
-        elif [[ "$DISTRO" = "ubuntu" ]]; then
-            export DEBIAN_FRONTEND=noninteractive
-            sudo apt-get install mongodb
-        else
-            NEEDS_MONGO=true
-        fi
-    fi
+# Check node version supports arrow fns and string templates
+node -e '() => console.log(`print "3": ${1+2}`)' >/dev/null 2>&1 || {
+    echo "Unsupported version of NodeJS."
+    echo ""
+    echo "Please update NodeJS to version 4.x.x or later (6.x.x recommended)"
+    exit 1
 }
 
 echo >&2 "Installing DeepForge...";

package.json

@@ -8,22 +8,25 @@
     "start-dev": "NODE_ENV=dev node app.js",
     "local": "node ./bin/start-local.js",
     "worker": "node ./bin/start-worker.js",
-    "test": "node ./node_modules/mocha/bin/mocha --recursive test",
+    "test": "mkdir ./test-tmp; node ./node_modules/mocha/bin/mocha --recursive test",
     "watch-test": "./node_modules/nodemon/bin/nodemon.js --exec 'node ./node_modules/mocha/bin/mocha --recursive test'",
     "build-nn": "node ./utils/nn-parser.js"
   },
-  "version": "0.14.1",
+  "version": "0.21.0",
   "dependencies": {
     "commander": "^2.9.0",
     "dotenv": "^2.0.0",
     "exists-file": "^2.1.0",
+    "express": "^4.14.0",
     "lodash.difference": "^4.1.2",
+    "graceful-fs": "^4.1.10",
     "lodash.merge": "^4.5.1",
+    "mongodb": "^2.2.10",
     "nodemon": "^1.9.2",
     "q": "1.4.1",
     "rimraf": "^2.4.0",
-    "webgme": "^2.0.0",
-    "webgme-autoviz": "dfst/webgme-autoviz",
+    "webgme": "^2.7.1",
+    "webgme-autoviz": "^2.2.0",
     "webgme-breadcrumbheader": "^2.1.1",
     "webgme-chflayout": "^2.0.0",
     "webgme-easydag": "dfst/webgme-easydag",

src/common/Constants.js

@@ -1,16 +1,53 @@
 /* globals define */
-define({
-    LINE_OFFSET: 'lineOffset',
+(function(root, factory){
+    if(typeof define === 'function' && define.amd) {
+        define([], function(){
+            return factory();
+        });
+    } else if(typeof module === 'object' && module.exports) {
+        module.exports = factory();
+    } else {
+        root.CONSTANTS = factory();
+    }
+}(this, function() {
+    return {
+        CONTAINED_LAYER_SET: 'addLayers',
+        CONTAINED_LAYER_INDEX: 'index',
 
-    // DeepForge metadata creation in dist execution
-    START_CMD: 'deepforge-cmd',
+        LINE_OFFSET: 'lineOffset',
 
-    IMAGE: 'IMG',
+        // DeepForge metadata creation in dist execution
+        START_CMD: 'deepforge-cmd',
 
-    GRAPH_CREATE: 'GRAPH',
-    GRAPH_PLOT: 'PLOT',
-    GRAPH_CREATE_LINE: 'LINE',
+        IMAGE: {  // all prefixed w/ 'IMG' for simple upload detection
+            PREFIX: 'IMG',
+            BASIC: 'IMG-B',
+            CREATE: 'IMG-C',
+            UPDATE: 'IMG-U',
+            NAME: 'IMAGE-N'  // No upload required
+        },
 
-    // Code Generation Constants
-    CTOR_ARGS_ATTR: 'ctor_arg_order'
-});
+        GRAPH_CREATE: 'GRAPH',
+        GRAPH_PLOT: 'PLOT',
+        GRAPH_CREATE_LINE: 'LINE',
+
+        // Code Generation Constants
+        CTOR_ARGS_ATTR: 'ctor_arg_order',
+
+        // Operation types
+        OP: {
+            INPUT: 'Input',
+            OUTPUT: 'Output'
+        },
+
+        // Heartbeat constants (ExecPulse router)
+        PULSE: {
+            DEAD: 0,
+            ALIVE: 1,
+            DOESNT_EXIST: 2
+        },
+
+        // Job stdout update
+        STDOUT_UPDATE: 'stdout_update'
+    };
+}));

src/common/LayerParser.js

@@ -265,6 +265,7 @@
     var findTorchClass = function(ast){ 
         var torchClassArgs,  // args for `torch.class(...)`
             name = '',
+            alias,
             baseType,
             params,
             setters = {},
@@ -283,6 +284,7 @@
                     name = torchClassArgs[0];
                     if(name !== ''){
                         name = name.replace('nn.', '');
+                        alias = func.names[0] || name;
                         if (torchClassArgs.length > 1) {
                             baseType = torchClassArgs[1].replace('nn.', '');
                         }
@@ -302,7 +304,7 @@
                 attrName;
 
             // Record the setter functions
-            if (isSetterMethod(curr, parent, name)) {
+            if (isSetterMethod(curr, parent, alias)) {
                 firstLine = curr.block.stats[0];
                 // just use the attribute attrName for now...
                 attrName = getSettingAttrName(firstLine);
@@ -316,7 +318,7 @@
                 } else {
                     setters[attrName] = schema;
                 }
-            } else if (isInitFn(curr, name)) {  // Record the defaults
+            } else if (isInitFn(curr, alias)) {  // Record the defaults
                 paramDefs = getAttrsAndVals(curr);
                 attrDefs = getClassAttrDefs(curr);
                 types = inferParamTypes(curr, paramDefs);

src/common/api/APIClient.js

@@ -0,0 +1,75 @@
+/*globals define*/
+define([
+    'q',
+    'superagent'
+], function(
+    Q,
+    superagent
+) {
+    'use strict';
+
+    // Wrap the ability to read, update, and delete logs using the JobLogsAPI
+    var APIClient = function(params) {
+        params = params || {};
+
+        this.logger = this.logger || params.logger.fork('APIClient');
+
+        // Get the server url
+        this.token = params.token;
+        this.origin = this._getServerUrl(params);
+        this.relativeUrl = this.relativeUrl || '';
+        this.url = this.origin + this.relativeUrl;
+
+        this.logger.debug(`Setting url to ${this.url}`);
+
+        this.branch = params.branchName;
+        this.project = params.projectId;
+        this._modifiedJobs = [];
+
+        this.logger.debug(`Using <project>:<branch>: "${this.project}"/"${this.branch}"`);
+        this.logger.info('ctor finished');
+    };
+
+    APIClient.prototype._getServerUrl = function(params) {
+        if (typeof window !== 'undefined') {
+            return window.location.origin;
+        }
+
+        // If not in browser, set using the params
+        var server = params.server || '127.0.0.1',
+            port = params.port || '80',
+            protocol = params.httpsecure ? 'https' : 'http';  // default is http
+
+        return params.origin || `${protocol}://${server}:${port}`;
+    };
+
+    APIClient.prototype.getUrl = function() {
+        return this.url;
+    };
+
+    APIClient.prototype._request = function(method, jobId, content) {
+        var deferred = Q.defer(),
+            req = superagent[method](this.getUrl(jobId));
+
+        this.logger.debug(`sending ${method} request to ${this.getUrl(jobId)}`);
+        if (this.token) {
+            req.set('Authorization', 'Bearer ' + this.token);
+        }
+
+        if (content) {
+            req = req.send(content);
+        }
+
+        req.end((err, res) => {
+            if (err || res.status > 399) {
+                return deferred.reject(res || err);
+            }
+
+            return deferred.resolve(res);
+        });
+
+        return deferred.promise;
+    };
+
+    return APIClient;
+});

src/common/api/ExecPulseClient.js

@@ -0,0 +1,44 @@
+/* globals define */
+define([
+    './APIClient'
+], function(
+    APIClient
+) {
+    'use strict';
+
+    var ExecPulseClient = function(params) {
+        this.relativeUrl = '/execution/pulse/';
+        this.logger = params.logger.fork('ExecPulseClient');
+        APIClient.call(this, params);
+    };
+
+    ExecPulseClient.prototype = Object.create(APIClient.prototype);
+
+    ExecPulseClient.prototype.getUrl = function(hash) {
+        return this.url + hash;
+    };
+
+    // - update the heartbeat
+    // - check the heartbeat
+    // - delete the heartbeat
+    ExecPulseClient.prototype.update = function(hash) {
+        return this._request('post', hash)
+            .catch(err => {
+                throw err.text || err;
+            });
+    };
+
+    ExecPulseClient.prototype.check = function(hash) {
+        return this._request('get', hash)
+            .then(res => JSON.parse(res.text))
+            .catch(err => {
+                throw err.text || err;
+            });
+    };
+
+    ExecPulseClient.prototype.clear = function(hash) {
+        return this._request('delete', hash);
+    };
+
+    return ExecPulseClient;
+});

src/common/api/JobLogsClient.js

@@ -0,0 +1,120 @@
+/* globals define */
+define([
+    './APIClient',
+    'q',
+    'superagent'
+], function(
+    APIClient,
+    Q,
+    superagent
+) {
+    'use strict';
+
+    // Wrap the ability to read, update, and delete logs using the JobLogsAPI
+    var METADATA_FIELDS = [
+        'lineCount'
+    ];
+    var JobLogsClient = function(params) {
+        params = params || {};
+
+        this.relativeUrl = '/execution/logs';
+        this.logger = params.logger.fork('JobLogsClient');
+        APIClient.call(this, params);
+
+        // Get the project, branch name
+        if (!(params.branchName && params.projectId)) {
+            throw Error('"branchName" and "projectId" required');
+        }
+
+        this._modifiedJobs = [];
+
+        this.logger.debug(`Using <project>:<branch>: "${this.project}"/"${this.branch}"`);
+        this.logger.info('ctor finished');
+    };
+
+    JobLogsClient.prototype = Object.create(APIClient.prototype);
+
+    // This method could be optimized - it could make a log of requests
+    JobLogsClient.prototype.fork = function(forkName) {
+        var jobIds = this._modifiedJobs,
+            deferred = Q.defer(),
+            url = [
+                this.url,
+                'migrate',
+                encodeURIComponent(this.project),
+                encodeURIComponent(this.branch),
+                encodeURIComponent(forkName)
+            ].join('/'),
+            req = superagent.post(url);
+
+        this.logger.info(`migrating ${jobIds.length} jobs from ${this.branch} to ${forkName} in ${this.project}`);
+        if (this.token) {
+            req.set('Authorization', 'Bearer ' + this.token);
+        }
+
+        req.send({jobs: jobIds})
+            .end((err, res) => {
+                if (err || res.status > 399) {
+                    return deferred.reject(err || res.status);
+                }
+
+                return deferred.resolve(res);
+            });
+
+        this.branch = forkName;
+        return deferred.promise;
+    };
+
+    JobLogsClient.prototype.getUrl = function(jobId) {
+        var url = this.url;
+            
+        if (typeof jobId !== 'string') {
+            url = this.url + jobId.route;
+            jobId = jobId.jobId;
+        }
+
+        return [
+            url,
+            encodeURIComponent(this.project),
+            encodeURIComponent(this.branch),
+            encodeURIComponent(jobId)
+        ].join('/');
+    };
+
+    var hasRequiredFields = function(md) {
+        return METADATA_FIELDS.reduce((passing, nextField) => {
+            return passing && md.hasOwnProperty(nextField);
+        }, true);
+    };
+
+    JobLogsClient.prototype.appendTo = function(jobId, text, metadata) {
+        this._modifiedJobs.push(jobId);
+        this.logger.info(`Appending logs to ${jobId}`);
+
+        if (metadata && !hasRequiredFields(metadata)) {
+            throw Error(`Required metadata fields: ${METADATA_FIELDS.join(', ')}`);
+        }
+        metadata = metadata || {};
+        metadata.patch = text;
+        return this._request('patch', jobId, metadata);
+    };
+
+    JobLogsClient.prototype.getLog = function(jobId) {
+        this.logger.info(`Getting logs for ${jobId}`);
+        return this._request('get', jobId)
+            .then(res => res.text);
+    };
+
+    JobLogsClient.prototype.deleteLog = function(jobId) {
+        this.logger.info(`Deleting logs for ${jobId}`);
+        return this._request('delete', jobId);
+    };
+
+    JobLogsClient.prototype.getMetadata = function(jobId) {
+        this.logger.info(`Getting line count for ${jobId}`);
+        return this._request('get', {jobId: jobId, route: '/metadata'})
+            .then(res => JSON.parse(res.text));
+    };
+
+    return JobLogsClient;
+});

src/common/api/JobOriginClient.js

@@ -0,0 +1,60 @@
+/* globals define */
+define([
+    './APIClient'
+], function(
+    APIClient
+) {
+    'use strict';
+
+    var JobOriginClient = function(params) {
+        this.relativeUrl = '/job/origins/';
+        this.logger = params.logger.fork('JobOriginClient');
+        APIClient.call(this, params);
+    };
+
+    JobOriginClient.prototype = Object.create(APIClient.prototype);
+
+    // - Record the origin
+    // - Look up the origin
+    // - Delete record
+    JobOriginClient.prototype.getUrl = function(hash) {
+        return this.url + hash;
+    };
+
+    JobOriginClient.prototype.record = function(hash, info) {
+        var jobInfo = {
+            hash: hash,
+            nodeId: info.nodeId,
+            job: info.job,
+            project: info.project || this.project,
+            branch: info.branch || this.branch,
+            execution: info.execution
+        };
+
+        return this._request('post', hash, jobInfo)
+            .catch(err => {
+                throw err.text || err;
+            });
+    };
+
+    JobOriginClient.prototype.getOrigin = function(hash) {
+        return this._request('get', hash)
+            .then(res => JSON.parse(res.text))
+            .catch(res => {
+                if (res.status && res.status === 404) {
+                    return null;
+                }
+                throw res;
+            });
+    };
+
+    JobOriginClient.prototype.fork = function(hash, forkName) {
+        return this._request('patch', hash, {branch: forkName});
+    };
+
+    JobOriginClient.prototype.deleteRecord = function(hash) {
+        return this._request('delete', hash);
+    };
+
+    return JobOriginClient;
+});

src/common/globals.js

@@ -57,7 +57,7 @@ define([
     };
 
     var createNamedNode = function(baseId, parentId, isMeta) {
-        var newId = client.createChild({parentId, baseId}),
+        var newId = client.createNode({parentId, baseId}),
             baseNode = client.getNode(baseId),
             basename = 'New' + baseNode.getAttribute('name'),
             newName = getUniqueName(parentId, basename);
@@ -72,7 +72,7 @@ define([
             client.setRegistry(newId, 'isAbstract', false);
         }
 
-        client.setAttributes(newId, 'name', newName);
+        client.setAttribute(newId, 'name', newName);
         return newId;
     };
 
@@ -191,7 +191,7 @@ define([
                 .getId();
 
         // Look up the parent container
-        DeepForge.places[placeName]().then(parentId => {
+        return DeepForge.places[placeName]().then(parentId => {
 
             client.startTransaction(msg);
             newId = createNamedNode(baseId, parentId, !!metasheetName);
@@ -265,7 +265,7 @@ define([
         }
 
         dataBaseId = dataBase.getId();
-        dataTypes = metanodes.filter(n => client.isTypeOf(n.getId(), dataBaseId))
+        dataTypes = metanodes.filter(n => n.isTypeOf(dataBaseId))
             .filter(n => !n.getRegistry('isAbstract'))
             .map(node => node.getAttribute('name'));
 
@@ -297,7 +297,8 @@ define([
     };
 
     DeepForge.last = {};
-    DeepForge.create  = {};
+    DeepForge.create = {};
+    DeepForge.register = {};
     instances.forEach(type => {
         DeepForge.create[type] = function() {
             return createNew.call(null, type);
@@ -308,6 +309,10 @@ define([
         DeepForge.create[type] = function() {
             return createNew.call(null, type, type);
         };
+        DeepForge.register[type] = function(id) {
+            // Add the given element to the metasheet!
+            return addToMetaSheet(id, type);
+        };
     });
 
     DeepForge.create.Layer = createCustomLayer;

src/common/lua.js

@@ -2565,7 +2565,7 @@ function LuaContext(){
             case 'number':
                 return c;
             case 'string':
-                return parseInt(s) || dummy0;
+                return parseInt(c) || dummy0;
             default:
                 if (c == dummy0){
                     return c;

src/common/plugin/LocalExecutor.js

@@ -2,14 +2,16 @@
 // This is an 'executor' containing the implementations of all local operations
 // These are all primitives in DeepForge
 define([
+    'deepforge/Constants'
 ], function(
+    CONSTANTS
 ) {
     'use strict';
     var LocalExecutor = function() {
     };
 
     // Should these be in lua?
-    LocalExecutor.prototype.ArtifactLoader = function(node) {
+    LocalExecutor.prototype[CONSTANTS.OP.INPUT] = function(node) {
         // Get the hash from the output node
         var hash;
         return this.core.loadChildren(node)
@@ -18,13 +20,13 @@ define([
                 var output = cntrs
                     .find(cntr => {
                         var metaNode = this.core.getMetaType(cntr),
-                            metaName = this.core.getAttribute(metaNode, 'name');
+                            metaName = this.getAttribute(metaNode, 'name');
                         return metaName === 'Outputs';
                     });
                 return this.core.loadChildren(output);
             })
             .then(dataNodes => {
-                hash = this.core.getAttribute(dataNodes[0], 'data');
+                hash = this.getAttribute(dataNodes[0], 'data');
                 return this.getOutputs(node);
             })
             .then(outputTuples => {
@@ -46,7 +48,7 @@ define([
         var hash,
             typeId = this.core.getPointerPath(node, 'type'),
             type,
-            artifactName = this.core.getAttribute(node, 'artifactName');
+            artifactName = this.getAttribute(node, 'artifactName');
 
         return this.core.loadByPath(this.rootNode, typeId)
             .then(_type => {
@@ -56,25 +58,20 @@ define([
             .then(saveDir => this.core.loadChildren(saveDir))
             .then(artifacts => {
                 return artifacts.find(artifact =>
-                    this.core.getAttribute(artifact, 'name') === artifactName &&
+                    this.getAttribute(artifact, 'name') === artifactName &&
                         this.isMetaTypeOf(artifact, type));
             })
             .then(matchingArtifact => {
-                hash = matchingArtifact && this.core.getAttribute(matchingArtifact, 'data');
+                hash = matchingArtifact && this.getAttribute(matchingArtifact, 'data');
                 // If no hash, just continue (the subsequent ops will receive 'nil')
                 if (!hash) {
                     return this.onOperationComplete(node);
                 } else {
                     return this.getOutputs(node)
                         .then(outputPairs => {
-                            var outputs = outputPairs.map(pair => pair[2]),
-                                paths;
-
-                            paths = outputs.map(output => this.core.getPath(output));
+                            var outputs = outputPairs.map(pair => pair[2]);
                             // Get the 'data' hash and store it in the output data ports
-                            this.logger.info(`Loading blob data (${hash}) to ${paths.map(p => `"${p}"`)}`);
-
-                            outputs.forEach(output => this.core.setAttribute(output, 'data', hash));
+                            outputs.forEach(output => this.setAttribute(output, 'data', hash));
 
                             this.onOperationComplete(node);
                         });
@@ -97,7 +94,7 @@ define([
 
                 if (containers.length > 1) {
                     saveDir = containers.find(c =>
-                        this.core.getAttribute(c, 'name').toLowerCase().indexOf('artifacts') > -1
+                        this.getAttribute(c, 'name').toLowerCase().indexOf('artifacts') > -1
                     ) || containers[0];
                 }
 
@@ -105,7 +102,7 @@ define([
             });
     };
 
-    LocalExecutor.prototype.Save = function(node) {
+    LocalExecutor.prototype[CONSTANTS.OP.OUTPUT] = function(node) {
         var parentNode,
             currNameHashPairs;
         
@@ -119,8 +116,8 @@ define([
             .then(artifacts => {
                 currNameHashPairs = artifacts
                     .map(node => [
-                        this.core.getAttribute(node, 'name'),
-                        this.core.getAttribute(node, 'data')
+                        this.getAttribute(node, 'name'),
+                        this.getAttribute(node, 'data')
                     ]);
                 return this.getInputs(node);
             })
@@ -140,9 +137,9 @@ define([
 
                 // Remove nodes that already exist
                 dataNodes = allDataNodes.filter(dataNode => {
-                    var hash = this.core.getAttribute(dataNode, 'data'),
+                    var hash = this.getAttribute(dataNode, 'data'),
                         name = this.core.getOwnAttribute(node, 'saveName') ||
-                            this.core.getAttribute(dataNode, 'name');
+                            this.getAttribute(dataNode, 'name');
 
                     return !(currNameHashPairs
                         .find(pair => pair[0] === name && pair[1] === hash));
@@ -151,13 +148,16 @@ define([
                 // get the input node
                 if (dataNodes.length !== 0) {
                     var newNodes = this.core.copyNodes(dataNodes, parentNode),
-                        newName = this.core.getOwnAttribute(node, 'saveName');
+                        newName = this.core.getOwnAttribute(node, 'saveName'),
+                        createdAt = Date.now();
+
                     if (newName) {
-                        newNodes.forEach(node =>
-                            this.core.setAttribute(node, 'name', newName)
-                        );
+                        newNodes.forEach(node => {
+                            this.setAttribute(node, 'name', newName);
+                            this.setAttribute(node, 'createdAt', createdAt);
+                        });
                     }
-                    var hashes = dataNodes.map(n => this.core.getAttribute(n, 'data'));
+                    var hashes = dataNodes.map(n => this.getAttribute(n, 'data'));
                     this.logger.info(`saving hashes: ${hashes.map(h => `"${h}"`)}`);
                 } else if (allDataNodes.length === 0) {
                     this.logger.warn('No data nodes found!');

src/common/styles/global.css

@@ -1,22 +1,18 @@
-/* latin-ext */
-@font-face {
-  font-family: 'Audiowide';
-  font-style: normal;
-  font-weight: 400;
-  src: local('Audiowide'), local('Audiowide-Regular'), url(https://fonts.gstatic.com/s/audiowide/v4/7pSgz2MbVvTCvvm7vukSHxJtnKITppOI_IvcXXDNrsc.woff2) format('woff2');
-  unicode-range: U+0100-024F, U+1E00-1EFF, U+20A0-20AB, U+20AD-20CF, U+2C60-2C7F, U+A720-A7FF;
-}
-/* latin */
-@font-face {
-  font-family: 'Audiowide';
-  font-style: normal;
-  font-weight: 400;
-  src: local('Audiowide'), local('Audiowide-Regular'), url(https://fonts.gstatic.com/s/audiowide/v4/8XtYtNKEyyZh481XVWfVOltXRa8TVwTICgirnJhmVJw.woff2) format('woff2');
-  unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02C6, U+02DA, U+02DC, U+2000-206F, U+2074, U+20AC, U+2212, U+2215, U+E0FF, U+EFFD, U+F000;
+li.deepforge-logo {
+    background-image: url(img/deepforge-logo.png);
+    background-repeat: no-repeat;
+    background-position: center;
+    width: 100px;
+    background-size: 95px;
 }
 
-.deepforge-logo .item-label {
-  font-family: 'Audiowide', cursive;
+li.deepforge-logo span {
+    visibility: hidden;
+}
+
+i.gme-icon {
+    background-image: url(img/deepforge-icon.png);
+    background-size: 15.20px 18px;
 }
 
 .create-node text {

src/common/utils.js

@@ -1,7 +1,13 @@
 /* globals define*/
-define([
-], function(
-) {
+(function(root, factory){
+    if(typeof define === 'function' && define.amd) {
+        define([], function(){
+            return (root.utils = factory());
+        });
+    } else if(typeof module === 'object' && module.exports) {
+        module.exports = (root.utils = factory());
+    }
+}(this, function() {
     var isBoolean = txt => {
         return typeof txt === 'boolean' || (txt === 'false' || txt === 'true');
     };
@@ -51,8 +57,37 @@ define([
         }
     };
 
+    // Resolving stdout
+    var resolveCarriageReturns = function(text) {
+        // resolve \r
+        var lines,
+            chars,
+            result,
+            i = 0;
+
+        text = text.replace(/\u0000/g, '');
+        lines = text.split('\n');
+        for (var l = lines.length-1; l >= 0; l--) {
+            i = 0;
+            chars = lines[l].split('');
+            result = [];
+
+            for (var c = 0; c < chars.length; c++) {
+                if (chars[c] === '\r') {
+                    i = 0;
+                }
+                result[i] = chars[c];
+                i++;
+            }
+            lines[l] = result.join('');
+        }
+        return lines;
+    };
+
+
     return {
         getSetterSchema: getSetterSchema,
+        resolveCarriageReturns: resolveCarriageReturns,
         abbr: abbr
     };
-});
+}));

src/common/viz/Buttons.js

@@ -1,8 +1,10 @@
 /*globals define, WebGMEGlobal*/
 define([
+    'deepforge/globals',
     'widgets/EasyDAG/Buttons',
     'widgets/EasyDAG/Icons'
 ], function(
+    DeepForge,
     EasyDAGButtons,
     Icons
 ) {
@@ -57,9 +59,79 @@ define([
         return n && n.getBaseId();
     };
 
+    var CloneAndEdit = function(params) {
+        GoToBase.call(this, params);
+    };
+
+    CloneAndEdit.prototype = Object.create(GoToBase.prototype);
+    CloneAndEdit.prototype.BTN_CLASS = 'clone-and-edit';
+
+    CloneAndEdit.prototype._render = function() {
+        var lineRadius = GoToBase.SIZE - GoToBase.BORDER,
+            btnColor = '#a5d6a7';
+
+        if (this.disabled) {
+            btnColor = '#e0e0e0';
+        }
+
+        this.$el
+            .append('circle')
+            .attr('r', GoToBase.SIZE)
+            .attr('fill', btnColor);
+
+        // Show the 'code' icon
+        Icons.addIcon('code', this.$el, {
+            radius: lineRadius
+        });
+    };
+
+    CloneAndEdit.prototype._onClick = function(item) {
+        var node = client.getNode(item.id),
+            baseId = node && node.getBaseId(),
+            base = baseId && client.getNode(baseId),
+            typeId = base && base.getBaseId(),
+            type = typeId && client.getNode(typeId),
+            ctrName,
+            typeName,
+            name,
+            newId;
+
+        // Clone the given node's base and change to it
+        if (type) {
+            typeName = type.getAttribute('name');
+            ctrName = `My${typeName}s`;
+            if (DeepForge.places[ctrName]) {
+                DeepForge.places[ctrName]().then(ctrId => {
+                    type = base.getAttribute('name');
+                    client.startTransaction(`Creating new ${typeName} from ${item.name}`);
+                    newId = client.copyNode(baseId, ctrId);
+                    name = node.getAttribute('name');
+                    client.setAttribute(newId, 'name', 'Copy of ' + name);
+                    DeepForge.register[typeName](newId);
+
+                    client.completeTransaction();
+                    WebGMEGlobal.State.registerActiveObject(newId);
+                });
+            }
+        } else {
+            this._logger.warn('Could not find the base node!');
+        }
+    };
+
+    var Insert = function(params) {
+        EasyDAGButtons.ButtonBase.call(this, params);
+    };
+
+    Insert.prototype = Object.create(EasyDAGButtons.Add.prototype);
+    Insert.prototype._onClick = function(item) {
+        this.onInsertButtonClicked(item);
+    };
+
     return {
         DeleteOne: EasyDAGButtons.DeleteOne,
-        GoToBase: GoToBase
+        GoToBase: GoToBase,
+        CloneAndEdit: CloneAndEdit,
+        Insert: Insert
     };
 });
 

src/common/viz/Execute.js

@@ -1,21 +1,33 @@
 /* globals define, WebGMEGlobal */
 // Mixin for executing jobs and pipelines
 define([
+    'q',
     'executor/ExecutorClient',
+    'deepforge/api/ExecPulseClient',
+    'deepforge/api/JobOriginClient',
+    'deepforge/Constants',
     'panel/FloatingActionButton/styles/Materialize'
 ], function(
+    Q,
     ExecutorClient,
+    ExecPulseClient,
+    JobOriginClient,
+    CONSTANTS,
     Materialize
 ) {
 
     var Execute = function(client, logger) {
         this.client = this.client || client;
         this.logger = this.logger || logger;
+        this.pulseClient = new ExecPulseClient({
+            logger: this.logger
+        });
         this._executor = new ExecutorClient({
             logger: this.logger.fork('ExecutorClient'),
             serverPort: WebGMEGlobal.gmeConfig.server.port,
             httpsecure: window.location.protocol === 'https:'
         });
+        this.originManager = new JobOriginClient({logger: this.logger});
     };
 
     Execute.prototype.executeJob = function(node) {
@@ -36,12 +48,20 @@ define([
         context.managerConfig.namespace = 'pipeline';
         context.managerConfig.activeNode = node.getId();
         method = opts.useSecondary ? 'runBrowserPlugin' : 'runServerPlugin';
+
+        if (method === 'runServerPlugin' &&
+            this.client.getBranchStatus() !== this.client.CONSTANTS.BRANCH_STATUS.SYNC) {
+
+            Materialize.toast('Cannot execute operations when client is out-of-sync', 2000);
+            return;
+        }
+
         this.client[method](pluginId, context, (err, result) => {
             var msg = err ? `${name} failed!` : `${name} executed successfully!`,
                 duration = err ? 4000 : 2000;
 
             // Check if it was canceled - if so, show that type of message
-            if (result) {
+            if (result && result.messages.length) {
                 msg = result.messages[0].message;
                 duration = 4000;
             }
@@ -75,13 +95,11 @@ define([
             job.getAttribute('secret') && job.getAttribute('jobId');
     };
 
-    Execute.prototype.stopJob = function(job) {
+    Execute.prototype.silentStopJob = function(job) {
         var jobHash,
-            jobId,
             secret;
 
         job = job || this.client.getNode(this._currentNodeId);
-        jobId = job.getId();
         jobHash = job.getAttribute('jobId');
         secret = job.getAttribute('secret');
         if (!jobHash || !secret) {
@@ -89,14 +107,179 @@ define([
             return;
         }
 
-        this.client.delAttributes(jobId, 'jobId');
-        this.client.delAttributes(jobId, 'secret');
-        this.client.setAttributes(jobId, 'status', 'canceled');
-
         return this._executor.cancelJob(jobHash, secret)
             .then(() => this.logger.info(`${jobHash} has been cancelled!`))
             .fail(err => this.logger.error(`Job cancel failed: ${err}`));
     };
 
+    Execute.prototype._setJobStopped = function(jobId, silent) {
+        if (!silent) {
+            var name = this.client.getNode(jobId).getAttribute('name');
+            this.client.startTransaction(`Stopping "${name}" job`);
+        }
+
+        this.client.delAttribute(jobId, 'jobId');
+        this.client.delAttribute(jobId, 'secret');
+        this.client.setAttribute(jobId, 'status', 'canceled');
+
+        if (!silent) {
+            this.client.completeTransaction();
+        }
+    };
+
+    Execute.prototype.stopJob = function(job, silent) {
+        var jobId;
+
+        job = job || this.client.getNode(this._currentNodeId);
+        jobId = job.getId();
+
+        this.silentStopJob(job);
+        this._setJobStopped(jobId, silent);
+    };
+
+
+    Execute.prototype.loadChildren = function(id) {
+        var deferred = Q.defer(),
+            execNode = this.client.getNode(id || this._currentNodeId),
+            jobIds = execNode.getChildrenIds(),
+            jobsLoaded = !jobIds.length || this.client.getNode(jobIds[0]);
+
+        // May need to load the jobs...
+        if (!jobsLoaded) {
+            // Create a territory and load the nodes
+            var territory = {},
+                ui;
+
+            territory[id] = {children: 1};
+            ui = this.client.addUI(this, () => {
+                this.client.removeUI(ui);
+                deferred.resolve();
+            });
+            this.client.updateTerritory(ui, territory);
+        } else {
+            deferred.resolve();
+        }
+
+        return deferred.promise;
+    };
+
+    Execute.prototype.stopExecution = function(id, inTransaction) {
+        var execNode = this.client.getNode(id || this._currentNodeId);
+
+        return this.loadChildren(id)
+            .then(() => this._stopExecution(execNode, inTransaction));
+    };
+
+    Execute.prototype.silentStopExecution = function(id) {
+        var execNode = this.client.getNode(id || this._currentNodeId);
+
+        // Stop the execution w/o setting any attributes
+        return this.loadChildren(id)
+            .then(() => this._silentStopExecution(execNode));
+    };
+
+    Execute.prototype._stopExecution = function(execNode, inTransaction) {
+        var msg = `Canceling ${execNode.getAttribute('name')} execution`,
+            jobIds;
+
+        if (!inTransaction) {
+            this.client.startTransaction(msg);
+        }
+
+        jobIds = this._silentStopExecution(execNode);
+
+        this.client.setAttribute(execNode.getId(), 'status', 'canceled');
+        jobIds.forEach(jobId => this._setJobStopped(jobId, true));
+
+        if (!inTransaction) {
+            this.client.completeTransaction();
+        }
+    };
+
+    Execute.prototype._silentStopExecution = function(execNode) {
+        var runningJobIds = execNode.getChildrenIds()
+            .map(id => this.client.getNode(id))
+            .filter(job => this.isRunning(job));  // get running jobs
+
+        runningJobIds.forEach(job => this.silentStopJob(job));  // stop them
+
+        return runningJobIds;
+    };
+
+    // Resuming Executions
+    Execute.prototype.checkJobExecution= function (job) {
+        var pipelineId = job.getParentId(),
+            pipeline = this.client.getNode(pipelineId);
+
+        // First check the parent execution. If it doesn't exist, then check the job
+        return this.checkPipelineExecution(pipeline)
+            .then(tryToStartJob => {
+                if (tryToStartJob) {
+                    return this._checkJobExecution(job);
+                }
+            });
+    };
+
+    Execute.prototype._checkJobExecution = function (job) {
+        var jobId = job.getAttribute('jobId'),
+            status = job.getAttribute('status');
+
+        if (status === 'running' && jobId) {
+            return this.pulseClient.check(jobId)
+                .then(status => {
+                    if (status !== CONSTANTS.PULSE.DOESNT_EXIST) {
+                        return this._onOriginBranch(jobId).then(onBranch => {
+                            if (onBranch) {
+                                this.runExecutionPlugin('ExecuteJob', {
+                                    node: job
+                                });
+                            }
+                        });
+                    } else {
+                        this.logger.warn(`Could not restart job: ${job.getId()}`);
+                    }
+                });
+        }
+        return Q();
+    };
+
+    Execute.prototype._onOriginBranch = function (hash) {
+        return this.originManager.getOrigin(hash)
+            .then(origin => {
+                var currentBranch = this.client.getActiveBranchName();
+                if (origin && origin.branch) {
+                    return origin.branch === currentBranch;
+                }
+                return false;
+            });
+    };
+
+    Execute.prototype.checkPipelineExecution = function (pipeline) {
+        var runId = pipeline.getAttribute('runId'),
+            status = pipeline.getAttribute('status'),
+            tryToStartJob = true;
+
+        if (status === 'running' && runId) {
+            return this.pulseClient.check(runId)
+                .then(status => {
+                    if (status === CONSTANTS.PULSE.DEAD) {
+                        // Check the origin branch
+                        return this._onOriginBranch(runId).then(onBranch => {
+                            if (onBranch) {
+                                this.runExecutionPlugin('ExecutePipeline', {
+                                    node: pipeline
+                                });
+                            }
+                        });
+                    }
+                    // only try to start if the pulse info doesn't exist
+                    tryToStartJob = status === CONSTANTS.PULSE.DOESNT_EXIST;
+                    return tryToStartJob;
+                });
+        } else {
+            return Q().then(() => tryToStartJob);
+        }
+    };
+
     return Execute;
 });

src/common/viz/PipelineControl.js

@@ -65,7 +65,7 @@ define([
 
     PipelineControl.prototype.createNode = function(baseId) {
         var parentId = this._currentNodeId,
-            newNodeId = this._client.createChild({parentId, baseId});
+            newNodeId = this._client.createNode({parentId, baseId});
 
         return newNodeId;
     };

src/common/viz/RenameablePanel.js

@@ -49,7 +49,7 @@ define([
 
                 if (!/^\s*$/.test(newValue)) {
                     this._client.startTransaction(msg);
-                    this._client.setAttributes(nodeId, 'name', newValue);
+                    this._client.setAttribute(nodeId, 'name', newValue);
                     this._client.completeTransaction();
                 }
             }

src/common/viz/panels/ThumbnailControl.js

@@ -0,0 +1,41 @@
+/* globals define */
+define([
+    'panels/EasyDAG/EasyDAGControl'
+], function(
+    EasyDAGControl
+) {
+    var ThumbnailControl = function() {
+        EasyDAGControl.apply(this, arguments);
+    };
+
+    ThumbnailControl.prototype = Object.create(EasyDAGControl.prototype);
+
+    ThumbnailControl.prototype._initWidgetEventHandlers = function () {
+        EasyDAGControl.prototype._initWidgetEventHandlers.call(this);
+        this._widget.updateThumbnail = this.updateThumbnail.bind(this);
+    };
+
+    ThumbnailControl.prototype.updateThumbnail = function (svg) {
+        var node = this._client.getNode(this._currentNodeId),
+            name,
+            attrs,
+            currentThumbnail,
+            attrName = 'thumbnail',
+            msg;
+
+        if (node) {  // may have been deleted
+            name = node.getAttribute('name');
+            attrs = node.getValidAttributeNames();
+            currentThumbnail = node.getAttribute(attrName);
+            msg = `Updating pipeline thumbnail for "${name}"`;
+
+            if (attrs.indexOf(attrName) > -1 && currentThumbnail !== svg) {
+                this._client.startTransaction(msg);
+                this._client.setAttribute(this._currentNodeId, attrName, svg);
+                this._client.completeTransaction();
+            }
+        }
+    };
+
+    return ThumbnailControl;
+});

src/common/viz/widgets/Thumbnail.js

@@ -0,0 +1,84 @@
+/* globals define, $, _ */
+define([
+    'widgets/EasyDAG/EasyDAGWidget'
+], function(
+    EasyDAGWidget
+) {
+
+    var ThumbnailWidget = function() {
+        EasyDAGWidget.apply(this, arguments);
+    };
+
+    ThumbnailWidget.prototype = Object.create(EasyDAGWidget.prototype);
+
+    ThumbnailWidget.prototype.addNode = function() {
+        var result = EasyDAGWidget.prototype.addNode.apply(this, arguments);
+
+        this.refreshThumbnail();
+        return result;
+    };
+
+    ThumbnailWidget.prototype.removeNode = function() {
+        var result = EasyDAGWidget.prototype.removeNode.apply(this, arguments);
+
+        this.refreshThumbnail();
+        return result;
+    };
+
+    ThumbnailWidget.prototype._removeConnection = function() {
+        var result = EasyDAGWidget.prototype._removeConnection.apply(this, arguments);
+
+        this.refreshThumbnail();
+        return result;
+    };
+
+    ThumbnailWidget.prototype.addConnection = function() {
+        var result = EasyDAGWidget.prototype.addConnection.apply(this, arguments);
+
+        this.refreshThumbnail();
+        return result;
+    };
+
+    ////////////////////////// Thumbnail updates //////////////////////////
+    ThumbnailWidget.prototype.getSvgDistanceDim = function(dim) {
+        var maxValue = this._getMaxAlongAxis(dim),
+            nodes,
+            minValue;
+
+        nodes = this.graph.nodes().map(id => this.graph.node(id));
+        minValue = nodes.length ? Math.min.apply(null, nodes.map(node => node[dim] || 0)) : 0;
+        return maxValue-minValue;
+    };
+
+    ThumbnailWidget.prototype.getSvgWidth = function() {
+        return this.getSvgDistanceDim('x');
+    };
+
+    ThumbnailWidget.prototype.getSvgHeight = function() {
+        return this.height - 25;
+    };
+
+    ThumbnailWidget.prototype.getViewBox = function() {
+        var maxX = this.getSvgWidth('x'),
+            maxY = this.getSvgHeight('y');
+
+        return `0 0 ${maxX} ${maxY}`;
+    };
+
+    ThumbnailWidget.prototype.refreshThumbnail = _.debounce(function() {
+        // Get the svg...
+        var svg = document.createElement('svg'),
+            group = this.$svg.node(),
+            child;
+
+        svg.setAttribute('viewBox', this.getViewBox());
+        for (var i = 0; i < group.children.length; i++) {
+            child = $(group.children[i]);
+            svg.appendChild(child.clone()[0]);
+        }
+
+        this.updateThumbnail(svg.outerHTML);
+    }, 1000);
+
+    return ThumbnailWidget;
+});

src/decorators/ArtifactOpDecorator/EasyDAG/ArtifactOpDecorator.EasyDAGWidget.js

@@ -1,12 +1,8 @@
 /*globals define, _*/
 /*jshint browser: true, camelcase: false*/
 
-/**
- * @author brollb / https://github.com/brollb
- */
-
 define([
-    'js/Constants',
+    'deepforge/Constants',
     'decorators/DcOpDecorator/EasyDAG/DcOpDecorator.EasyDAGWidget',
     'css!./ArtifactOpDecorator.EasyDAGWidget.css'
 ], function (
@@ -19,16 +15,17 @@ define([
     var ArtifactOpDecorator,
         DECORATOR_ID = 'ArtifactOpDecorator',
         CAST_OPTS = {
-            ArtifactLoader: {
-                ptr: 'artifact',
-                metaTgt: false
-            },
             ArtifactFinder: {
                 ptr: 'type',
                 metaTgt: true
             }
         };
 
+    CAST_OPTS[CONSTANTS.OP.INPUT] = {
+        ptr: 'artifact',
+        metaTgt: false
+    };
+
     // ArtifactOp nodes need to be able to...
     //     - dynamically change their outputs (downcast)
     ArtifactOpDecorator = function (options) {
@@ -60,13 +57,13 @@ define([
             this.client.startTransaction(`Removing output of ${this.name}`);
             this.client.delPointer(this._node.id, name);
             if (outputId) {
-                this.client.delAttributes(outputId, 'data');
+                this.client.delAttribute(outputId, 'data');
             }
             this.client.completeTransaction();
         } else if (name === this.castOpts.ptr) {  // set the casted value
             this.client.startTransaction(`Setting output of ${this.name} to ${to}`);
             this.castOutputType(to);
-            this.client.makePointer(this._node.id, name, to);
+            this.client.setPointer(this._node.id, name, to);
             this.client.completeTransaction();
         } else {
             DecoratorBase.prototype.savePointer.call(this, name, to);
@@ -74,7 +71,7 @@ define([
     };
 
     ArtifactOpDecorator.prototype.getDisplayName = function() {
-        var ptrName = this._node.baseName === 'ArtifactLoader' ? 'artifact' : 'type',
+        var ptrName = this._node.baseName === CONSTANTS.OP.INPUT ? 'artifact' : 'type',
             id = this._node.pointers[ptrName],
             name = this.nameFor[id] || this._node.name;
         return name;
@@ -91,7 +88,7 @@ define([
     ArtifactOpDecorator.prototype.updateTargetName = function(id, name) {
         DecoratorBase.prototype.updateTargetName.apply(this, arguments);
         // Update name
-        var ptrName = this._node.baseName === 'ArtifactLoader' ? 'artifact' : 'type';
+        var ptrName = this._node.baseName === CONSTANTS.OP.INPUT ? 'artifact' : 'type';
         if (this._node.pointers[ptrName] === id) {
             this._name = name;
             this.onResize();

src/decorators/ContainerLayerDecorator/ContainerLayerDecorator.js

@@ -0,0 +1,39 @@
+/*globals define, _*/
+/*jshint browser: true, camelcase: false*/
+
+define([
+    'js/Decorators/DecoratorBase',
+    './EasyDAG/ContainerLayerDecorator.EasyDAGWidget'
+], function (
+    DecoratorBase,
+    ContainerLayerDecoratorEasyDAGWidget
+) {
+
+    'use strict';
+
+    var ContainerLayerDecorator,
+        __parent__ = DecoratorBase,
+        __parent_proto__ = DecoratorBase.prototype,
+        DECORATOR_ID = 'ContainerLayerDecorator';
+
+    ContainerLayerDecorator = function (params) {
+        var opts = _.extend({loggerName: this.DECORATORID}, params);
+
+        __parent__.apply(this, [opts]);
+
+        this.logger.debug('ContainerLayerDecorator ctor');
+    };
+
+    _.extend(ContainerLayerDecorator.prototype, __parent_proto__);
+    ContainerLayerDecorator.prototype.DECORATORID = DECORATOR_ID;
+
+    /*********************** OVERRIDE DecoratorBase MEMBERS **************************/
+
+    ContainerLayerDecorator.prototype.initializeSupportedWidgetMap = function () {
+        this.supportedWidgetMap = {
+            EasyDAG: ContainerLayerDecoratorEasyDAGWidget
+        };
+    };
+
+    return ContainerLayerDecorator;
+});

src/decorators/ContainerLayerDecorator/EasyDAG/ContainerLayerDecorator.EasyDAGWidget.css

@@ -0,0 +1,25 @@
+.condense .nested-layers {
+  display: none;
+}
+
+.nested-layers .hover-box {
+  stroke: black;
+  stroke-dasharray: 4, 4;
+  stroke-opacity: 0;
+}
+
+.nested-layers .unhovered .button {
+  opacity: 0;
+}
+
+.nested-layers .hovered .button {
+  opacity: 1;
+}
+
+.nested-layers .unhovered .hover-box {
+  stroke-opacity: 0;
+}
+
+.nested-layers .hovered .hover-box {
+  stroke-opacity: 0;
+}

src/decorators/ContainerLayerDecorator/EasyDAG/ContainerLayerDecorator.EasyDAGWidget.js

@@ -0,0 +1,428 @@
+/*globals define, _, */
+/*jshint browser: true, camelcase: false*/
+
+define([
+    'decorators/LayerDecorator/EasyDAG/LayerDecorator.EasyDAGWidget',
+    'js/Constants',
+    'deepforge/Constants',
+    './NestedLayer',
+    'widgets/EasyDAG/Buttons',
+    'css!./ContainerLayerDecorator.EasyDAGWidget.css'
+], function (
+    LayerDecorator,
+    GME_CONSTANTS,
+    CONSTANTS,
+    NestedLayer,
+    Buttons
+) {
+
+    'use strict';
+
+    var ContainerLayerDecorator,
+        ZOOM = 0.8,
+        DECORATOR_ID = 'ContainerLayerDecorator';
+
+    // Container layer nodes need to be able to nest the containedLayers
+    // in order inside of themselves when expanded
+    ContainerLayerDecorator = function (options) {
+        this.nestedLayers = {};
+        LayerDecorator.call(this, options);
+        this.$nested = this.$el.append('g')
+            .attr('class', 'nested-layers');
+
+        // If clicked, deselect the given nested layer
+        this.$el.on('click', () => {
+            if (this.expanded) {
+                Object.keys(this.nestedLayers).forEach(id => {
+                    this.nestedLayers[id].widget.onBackgroundClick();
+                });
+            }
+        });
+        this.onNestedRefresh = _.debounce(this.updateExpand.bind(this), 50);
+
+        // Add event handlers
+        NestedLayer.prototype.addLayerBefore = function(layerId) {
+            var decorator = this._parent,
+                index = decorator._node.containedLayers.indexOf(this.id);
+            return decorator.addLayerAt(layerId, index - 1);
+        };
+
+        NestedLayer.prototype.addLayerAfter = function(layerId) {
+            var decorator = this._parent,
+                index = decorator._node.containedLayers.indexOf(this.id);
+            return decorator.addLayerAt(layerId, index + 1);
+        };
+
+        NestedLayer.prototype.isLast = function() {
+            var index = this._parent._node.containedLayers.length - 1;
+            return this._parent._node.containedLayers[index] === this.id;
+        };
+
+        NestedLayer.prototype.isFirst = function() {
+            return this._parent._node.containedLayers[0] === this.id;
+        };
+
+        NestedLayer.prototype.moveLayerForward = function() {
+            return this.moveLayer(true);
+        };
+
+        NestedLayer.prototype.moveLayerBackward = function() {
+            return this.moveLayer();
+        };
+
+        NestedLayer.prototype.moveLayer = function(forward) {
+            var decorator = this._parent,
+                index = decorator._node.containedLayers.indexOf(this.id),
+                client = decorator.client,
+                msg;
+
+            decorator._node.containedLayers.splice(index, 1);
+            if (forward) {
+                index = Math.max(0, index - 1);
+            } else {
+                index++;
+            }
+
+            decorator._node.containedLayers.splice(index, 0, this.id);
+
+            msg = `Swapping nested layers at ${index} and ${forward ? index-1 : index+1}`;
+            client.startTransaction(msg);
+            decorator._updateNestedIndices();
+            client.completeTransaction();
+        };
+
+        NestedLayer.prototype.onLastNodeRemoved = function() {
+            var decorator = this._parent,
+                index = decorator._node.containedLayers.indexOf(this.id),
+                msg = `Removing nested layer of ${decorator._node.name} at position ${index}`;
+
+            decorator.client.startTransaction(msg);
+            decorator.client.deleteNode(this.id);
+            decorator.client.completeTransaction();
+        };
+        this.updateNestedTerritory();
+    };
+
+    _.extend(ContainerLayerDecorator.prototype, LayerDecorator.prototype);
+
+    ContainerLayerDecorator.prototype.DECORATOR_ID = DECORATOR_ID;
+
+    ContainerLayerDecorator.prototype._updateNestedIndices = function() {
+        this._node.containedLayers.forEach((layerId, index) => {
+            // Set the layer's member registry to it's index
+            this.client.setMemberRegistry(
+                this._node.id,
+                layerId,
+                CONSTANTS.CONTAINED_LAYER_SET,
+                CONSTANTS.CONTAINED_LAYER_INDEX,
+                index
+            );
+        });
+    };
+
+    ContainerLayerDecorator.prototype.addLayerAt = function(baseId, index) {
+        var client = this.client,
+            parentId = this._node.id,
+            archNode,
+            newId,
+            msg;
+
+        // Get the index of the given layer
+        index = Math.max(index, 0);
+
+        archNode = client.getAllMetaNodes()
+            .find(node => node.getAttribute('name') === 'Architecture');
+
+        // Create a new Architecture node in the given node
+        msg = `Adding layer to ${this._node.name} at position ${index}`;
+        client.startTransaction(msg);
+
+        newId = client.createNode({
+            parentId: parentId,
+            baseId: archNode.getId()
+        });
+        // Create the selected layer
+        client.createNode({
+            parentId: newId,
+            baseId: baseId
+        });
+        client.addMember(parentId, newId, CONSTANTS.CONTAINED_LAYER_SET);
+        this._node.containedLayers.splice(index, 0, newId);
+        this._updateNestedIndices();
+
+        client.completeTransaction();
+    };
+
+    ContainerLayerDecorator.prototype.condense = function() {
+        // hide the nested layers
+        this.$el.attr('class', 'centering-offset condense');
+        this.removeCreateNestedBtn();
+        return LayerDecorator.prototype.condense.apply(this, arguments);
+    };
+
+    ContainerLayerDecorator.prototype.updateNestedTerritory = function() {
+        // Add the nested layers and update
+        if (!this._nestedTerritoryUI) {
+            this._nestedTerritoryUI = this.client.addUI(this, this._containedEvents.bind(this));
+        }
+        this._territory = {};
+        this._node.containedLayers.forEach(id => this._territory[id] = {children: 0});
+        this.client.updateTerritory(this._nestedTerritoryUI, this._territory);
+    };
+
+    ContainerLayerDecorator.prototype._containedEvents = function(events) {
+        for (var i = events.length; i--;) {
+            switch (events[i].etype) {
+            case GME_CONSTANTS.TERRITORY_EVENT_LOAD:
+                if (!this.nestedLayers[events[i].eid]) {
+                    this.createNestedWidget(events[i].eid);
+                }
+                break;
+
+            case GME_CONSTANTS.TERRITORY_EVENT_UNLOAD:
+                this.removeNestedWidget(events[i].eid);
+                break;
+            }
+        }
+        if (events.length > 1) {  // if more than just 'complete' event
+            this.updateExpand();
+        }
+    };
+
+    ContainerLayerDecorator.prototype.update = function(node) {
+        var attrsUpdated = false,
+            attrs = this._attributes;
+
+        this._node = node;
+        // Update the attributes
+        this.setAttributes();
+        attrsUpdated = !_.isEqual(attrs, this._attributes);
+
+        // Check for a new nested layer
+        var hasNewLayers = this._node.containedLayers
+            .filter(id => !this.nestedLayers[id])
+            .length > 0;
+
+        if (hasNewLayers) {
+            this.updateNestedTerritory();
+        } else {
+            // Update the order of the nested layers
+            if (this._selected) {
+                this.expand();
+            } else {
+                this.condense();
+            }
+        }
+        // Only reset fieldsWidth if the attribute has gotten larger
+        if (attrsUpdated) {
+            this.fieldsWidth = null;
+        }
+    };
+
+    ContainerLayerDecorator.prototype.updateExpand = function() {
+        if (this.expanded) {
+            this.expand();
+        }
+    };
+
+    ContainerLayerDecorator.prototype.createNestedWidget = function(id) {
+        if (!this.$nested) {
+            this.$nested = this.$el.append('g')
+                .attr('class', 'nested-layers');
+        }
+
+        this.nestedLayers[id] = new NestedLayer({
+            $container: this.$nested,
+            parent: this,
+            client: this.client,
+            logger: this.logger,
+            onRefresh: this.onNestedRefresh,
+            id: id
+        });
+        return this.nestedLayers[id];
+    };
+
+    ContainerLayerDecorator.prototype.removeNestedWidget = function(id) {
+        this.nestedLayers[id].destroy();
+        delete this.nestedLayers[id];
+        this.updateExpand();
+    };
+
+    ContainerLayerDecorator.prototype._renderInfo = function(top, width) {
+        var isAnUpdate = this.expanded,
+            y = top;
+
+        // Add the attribute fields
+        this.clearFields();
+        this.$attributes = this.$el.append('g')
+            .attr('fill', '#222222');
+
+        if (!isAnUpdate) {
+            this.$attributes.attr('opacity', 0);
+        }
+
+        y = this.createAttributeFields(y, width);
+        y = this.createPointerFields(y, width);
+
+        if (y !== top) {
+            y += this.ROW_HEIGHT/2;
+        }
+        return y;
+    };
+
+    ContainerLayerDecorator.prototype.expand = function() {
+        // This should be rendered with the attributes
+        var height,
+            width,
+
+            // Attributes
+            initialY = 25,
+            isAnUpdate = this.expanded,
+            NAME_MARGIN = 15,
+            nestedMargin = 15,  // minimum
+            margin = 5,
+            y = margin + initialY,
+            x = margin,
+            i;
+
+        // Shift name down
+        this.$name.attr('y', 20);
+
+        // Add the nested children
+        var ids = this._node.containedLayers.filter(id => this.nestedLayers[id]),
+            totalNestedWidth = 0,
+            maxNestedHeight = 0,
+            fieldWidth,
+            widget;
+
+        if (ids.length === 0) {
+            maxNestedHeight = CreateNestedBtn.SIZE * 2;
+        } else {
+            for (i = 0; i < ids.length; i++) {
+                widget = this.nestedLayers[ids[i]].widget;
+                totalNestedWidth += widget.getSvgWidth() * ZOOM;
+                maxNestedHeight = Math.max(widget.getSvgHeight() * ZOOM, maxNestedHeight);
+
+                // Update the buttons (in case of reorder)
+                this.nestedLayers[ids[i]].refreshButtons();
+            }
+        }
+
+        fieldWidth = this.fieldsWidth + 3 * NAME_MARGIN;
+        width = Math.max(
+            this.nameWidth + 2 * NAME_MARGIN,
+            this.size.width,
+            fieldWidth,
+            totalNestedWidth + (ids.length + 1) * nestedMargin
+        );
+
+        // Render attributes
+        y = this._renderInfo(y, fieldWidth);
+        y += nestedMargin;
+
+        // Update width, height
+        height = y + maxNestedHeight + nestedMargin;
+
+        // Equally space the nested widgets
+        nestedMargin = (width - totalNestedWidth)/(ids.length + 1);
+        x = nestedMargin - width/2;
+        for (i = 0; i < ids.length; i++) {
+            this.nestedLayers[ids[i]].$el
+                .attr('transform', `translate(${x}, ${y}) scale(${ZOOM})`);
+            x += this.nestedLayers[ids[i]].widget.getSvgWidth() * ZOOM + nestedMargin;
+        }
+
+        this.removeCreateNestedBtn();
+
+        if (ids.length === 0) {
+            // Add the 'create nested layer' button if no nested layers
+            this.$createNestedBtn = new CreateNestedBtn({
+                context: this,
+                $pEl: this.$el,
+                y: y + CreateNestedBtn.SIZE
+            });
+        }
+
+        this.$body
+            .transition()
+            .attr('x', -width/2)
+            .attr('y', 0)
+            .attr('rx', 0)
+            .attr('ry', 0)
+            .attr('width', width)
+            .attr('height', height)
+            .each('end', () => {
+                if (!isAnUpdate) {
+                    this.$attributes.attr('opacity', 1);
+                    this.$el.attr('class', 'centering-offset expand');
+                }
+            });
+
+        if (this.height !== height || this.width !== width) {
+            this.height = height;
+            this.width = width;
+            this.expanded = true;
+            this.$el
+                .attr('transform', `translate(${this.width/2}, 0)`);
+
+            this.onResize();
+        }
+    };
+
+    ContainerLayerDecorator.prototype.removeCreateNestedBtn = function() {
+        if (this.$createNestedBtn) {
+            this.$createNestedBtn.remove();
+            this.$createNestedBtn = null;
+        }
+    };
+
+    ContainerLayerDecorator.prototype.destroyNested = function() {
+        Object.keys(this.nestedLayers).forEach(id => this.nestedLayers[id].destroy());
+        this.nestedLayers = {};
+
+        if (this.$nested) {
+            this.$nested.remove();
+            this.$nested = this.$el.append('g')
+                .attr('class', 'nested-layers');
+        }
+    };
+
+    ContainerLayerDecorator.prototype.destroy = function() {
+        LayerDecorator.prototype.destroy.call(this);
+        if (this._nestedTerritoryUI) {
+            this.client.removeUI(this._nestedTerritoryUI);
+            this._nestedTerritoryUI = null;
+        }
+        this.destroyNested();
+    };
+
+    var CreateNestedBtn = function(params) {
+        params.title = 'Add nested layer';
+        Buttons.Add.call(this, params);
+    };
+
+    CreateNestedBtn.SIZE = Buttons.Add.SIZE;
+    CreateNestedBtn.prototype = Object.create(Buttons.Add.prototype);
+
+    CreateNestedBtn.prototype._onClick = function() {
+        // Call addLayerAfter and prompt for a layer
+        this.promptLayer()
+            .then(layerId => this.addLayerAt(layerId, 0));
+    };
+
+    ContainerLayerDecorator.prototype.expandAll = function() {
+        this.expand();
+        // For each of the nested layers, expand all their nodes
+        Object.keys(this.nestedLayers)
+            .forEach(id => this.nestedLayers[id].widget.expandAllNodes());
+    };
+
+    ContainerLayerDecorator.prototype.condenseAll = function() {
+        this.condense();
+        // For each of the nested layers, expand all their nodes
+        Object.keys(this.nestedLayers)
+            .forEach(id => this.nestedLayers[id].widget.expandAllNodes(true));
+    };
+
+    return ContainerLayerDecorator;
+});

src/decorators/ContainerLayerDecorator/EasyDAG/NestedLayer.js

@@ -0,0 +1,175 @@
+/*globals define, _ */
+define([
+    'panels/ArchEditor/ArchEditorControl',
+    'widgets/ArchEditor/ArchEditorWidget',
+    'widgets/EasyDAG/Buttons'
+], function(
+    ArchEditor,
+    ArchEditorWidget,
+    Buttons
+) {
+    var nop = () => {};
+    var NestedLayer = function(opts) {
+        this.$el = opts.$container.append('g')
+            .attr('class', 'nested-layer');
+
+        this.id = opts.id;
+        this._parent = opts.parent;
+        this.logger = opts.logger;
+
+        this.refreshButtons = _.debounce(this.updateButtons.bind(this), 100);
+        this.$outline = this.$el.append('rect')  // for hover detection
+            .attr('fill-opacity', 0)
+            .attr('x', 0)
+            .attr('y', 0);
+
+        this.$content = this.$el.append('g');
+        this.initHover();
+
+        this.widget = new ArchEditorWidget({
+            logger: this.logger.fork('ArchWidget'),
+            autoCenter: false,
+            svg: this.$content
+        });
+        this.widget.setTitle =
+        this.widget.updateEmptyMsg = nop;
+        this.onRefresh = opts.onRefresh;
+        this.widget.refreshExtras = this.onWidgetRefresh.bind(this);
+
+        this.control = new ArchEditor({
+            logger: this.logger.fork('ArchControl'),
+            client: opts.client,
+            embedded: true,
+            widget: this.widget
+        });
+        this.control._onUnload = () => {
+            ArchEditor.prototype._onUnload.apply(this.control, arguments);
+            // If it was the last node, remove it
+            var node = this.control._client.getNode(this.id);
+            if (node.getChildrenIds().length === 0) {
+                this.onLastNodeRemoved();
+            }
+        };
+
+        // hack :(
+        this.control.$btnModelHierarchyUp = {
+            show: nop,
+            hide: nop
+        };
+        this.widget.active = true;
+        this.control.selectedObjectChanged(this.id);
+    };
+
+    NestedLayer.prototype.initHover = function() {
+        this.$hover = this.$el.append('g')
+            .attr('class', 'hover-items');
+
+
+
+        this.$el.on('mouseenter', this.onHover.bind(this));
+        this.$el.on('mouseleave', this.onUnhover.bind(this));
+
+        // Buttons
+        this.$leftBtn = new Buttons.Add({
+            hide: true,
+            icon: this.isFirst() ? 'plus' : 'chevron-left',
+            $pEl: this.$hover
+        });
+
+        this.$rightBtn = new Buttons.Add({
+            hide: true,
+            icon: this.isLast() ? 'plus' : 'chevron-right',
+            $pEl: this.$hover
+        });
+
+        this.$deleteBtn = new Buttons.DeleteOne({
+            hide: true,
+            title: 'Delete',
+            $pEl: this.$hover
+        });
+
+        this.$leftBtn._onClick = this.clickLeft.bind(this);
+        this.$rightBtn._onClick = this.clickRight.bind(this);
+        this.$deleteBtn._onClick = () => this.onLastNodeRemoved();
+
+        this.$leftHint = this.$leftBtn.$el.append('title');
+        this.$rightHint = this.$rightBtn.$el.append('title');
+        this.refreshButtons();
+    };
+
+    NestedLayer.prototype.updateButtons = function() {
+        this.$leftBtn.icon = this.isFirst() ? 'plus' : 'chevron-left';
+        this.$rightBtn.icon = this.isLast() ? 'plus' : 'chevron-right';
+
+        this.$leftHint.text(this.isFirst() ?
+            'Add nested layer' :
+            'Move nested layer left'
+        );
+        this.$rightHint.text(this.isLast() ?
+            'Add nested layer' :
+            'Move nested layer right'
+        );
+
+        this.$leftBtn.render();
+        this.$rightBtn.render();
+    };
+
+    NestedLayer.prototype.clickLeft = function() {
+        if (this.isFirst()) {
+            this.promptLayer()
+                .then(layerId => this.addLayerBefore(layerId));
+        } else {
+            this.moveLayerForward();
+        }
+        this.onUnhover();
+    };
+
+    NestedLayer.prototype.promptLayer = function() {
+        var nodes = this.widget.getValidInitialNodes();
+
+        return this.widget.promptLayer(nodes)
+            .then(selected => selected.node.id);
+    };
+
+    NestedLayer.prototype.clickRight = function() {
+        if (this.isLast()) {
+            this.promptLayer()
+                .then(layerId => this.addLayerAfter(layerId));
+        } else {
+            this.moveLayerBackward();
+        }
+        this.onUnhover();
+    };
+
+    NestedLayer.prototype.onHover = function() {
+        this.refreshButtons();
+        this.$hover.attr('class', 'hover-items hovered');
+    };
+
+    NestedLayer.prototype.onUnhover = function() {
+        this.$hover.attr('class', 'hover-items unhovered');
+    };
+
+    NestedLayer.prototype.onWidgetRefresh = function() {
+        var width = this.widget.getSvgWidth(),
+            height = this.widget.getSvgHeight();
+
+        this.$outline
+            .attr('width', width)
+            .attr('height', height);
+
+        this.$leftBtn.$el.attr('transform', `translate(0, ${height/2})`);
+        this.$rightBtn.$el
+            .attr('transform', `translate(${width}, ${height/2})`);
+
+        this.onRefresh();
+    };
+
+    NestedLayer.prototype.destroy = function() {
+        this.control.destroy();
+        this.widget.destroy();
+        this.$el.remove();
+    };
+
+    return NestedLayer;
+});

src/decorators/DcOpDecorator/EasyDAG/DcOpDecorator.EasyDAGWidget.js

@@ -47,11 +47,11 @@ define([
             // create the outputId node
             outputId = this._createOutputNode(baseId);
         } else {
-            this.client.makePointer(outputId, CONSTANTS.POINTER_BASE, baseId);
+            this.client.setPointer(outputId, CONSTANTS.POINTER_BASE, baseId);
         }
         // Copy the data content to the output node
         hash = target.getAttribute('data');
-        this.client.setAttributes(outputId, 'data', hash);
+        this.client.setAttribute(outputId, 'data', hash);
     };
 
     DcOpDecorator.prototype._createOutputNode = function(baseId) {
@@ -69,7 +69,7 @@ define([
             return metaType.getAttribute('name') === 'Outputs';
         });
 
-        return this.client.createChild({
+        return this.client.createNode({
             baseId: baseId,
             parentId: outputCntrId
         });

src/decorators/JobDecorator/EasyDAG/JobDecorator.EasyDAGWidget.js

@@ -2,11 +2,13 @@
 /*jshint browser: true, camelcase: false*/
 
 define([
+    'deepforge/Constants',
     'decorators/EllipseDecorator/EasyDAG/EllipseDecorator.EasyDAGWidget',
     './PointerField.RO',
     './AttributeField.RO',
     'css!./JobDecorator.EasyDAGWidget.css'
 ], function (
+    CONSTANTS,
     EllipseDecorator,
     PointerField,
     AttributeField
@@ -49,12 +51,12 @@ define([
     JobDecorator.prototype.AttributeField = AttributeField;
     JobDecorator.prototype.PointerField = PointerField;
 
-    JobDecorator.prototype.isArtifactLoader = function() {
-        return this._node.name === 'ArtifactLoader';
+    JobDecorator.prototype.isInputOperation = function() {
+        return this._node.name === CONSTANTS.OP.INPUT;
     };
 
     JobDecorator.prototype.getDisplayName = function() {
-        if (this.isArtifactLoader()) {
+        if (this.isInputOperation()) {
             var id = this._node.pointers.artifact;
 
             // Try to look up the pointer name

src/decorators/LayerDecorator/EasyDAG/LayerDecorator.EasyDAGWidget.js

@@ -77,11 +77,11 @@ define([
         // Create a nested "architecture" node and set the ptr target to it
         baseId = base.getId();
         this.client.startTransaction(msg);
-        tgtId = this.client.createChild({
+        tgtId = this.client.createNode({
             parentId: this._node.id,
             baseId: baseId
         });
-        this.client.setAttributes(tgtId, 'name', `${ptr} (${this._node.name})`);
+        this.client.setAttribute(tgtId, 'name', `${ptr} (${this._node.name})`);
         this.savePointer(ptr, tgtId);
         this.client.completeTransaction();
         WebGMEGlobal.State.registerActiveObject(tgtId);
@@ -95,7 +95,7 @@ define([
             // If the target is contained in the current node, delete it!
             if (currentId.indexOf(this._node.id) === 0) {
                 this.client.startTransaction(`Removing layer for ${ptr} of ${name}`);
-                this.client.delMoreNodes([currentId]);
+                this.client.deleteNode(currentId);
                 this.client.completeTransaction();
                 this.logger.info(`Removed ${ptr} and deleted target (${currentId})`);
             } else {

src/decorators/OpIntDecorator/EasyDAG/OpIntDecorator.EasyDAGWidget.js

@@ -102,8 +102,8 @@ define([
             msg = `Deleting "${name}" attribute from "${opName}" operation`;
 
         this.client.startTransaction(msg);
-        this.client.removeAttributeSchema(this._node.id, name);
-        this.client.delAttributes(this._node.id, name);
+        this.client.delAttributeMeta(this._node.id, name);
+        this.client.delAttribute(this._node.id, name);
         this.client.completeTransaction();
     };
 
@@ -129,14 +129,14 @@ define([
 
         if (name !== desc.name) {  // Renaming attribute
             if (name) {
-                this.client.removeAttributeSchema(this._node.id, name);
-                this.client.delAttributes(this._node.id, name);
+                this.client.delAttributeMeta(this._node.id, name);
+                this.client.delAttribute(this._node.id, name);
             }
             name = desc.name;
         }
 
-        this.client.setAttributeSchema(this._node.id, name, schema);
-        this.client.setAttributes(this._node.id, name, desc.defaultValue);
+        this.client.setAttributeMeta(this._node.id, name, schema);
+        this.client.setAttribute(this._node.id, name, desc.defaultValue);
         this.client.completeTransaction();
     };
 

src/decorators/OperationDecorator/EasyDAG/OperationDecorator.EasyDAGWidget.js

@@ -73,14 +73,15 @@ define([
 
     OperationDecorator.prototype.showPorts = function(ids, areInputs) {
         var allPorts = areInputs ? this._node.inputs : this._node.outputs,
-            ports = ids ? allPorts.filter(port => ids.indexOf(port.id) > -1) : allPorts,
             x = -this.width/2,
-            dx = this.width/(ports.length+1),
+            dx = this.width/(allPorts.length+1),
             y = areInputs ? 0 : this.height;  // (this.height/2);
 
-        ports.forEach(port => {
+        allPorts.forEach(port => {
             x += dx;
-            this.renderPort(port, x, y, areInputs);
+            if (!ids || ids.indexOf(port.id) > -1) {
+                this.renderPort(port, x, y, areInputs);
+            }
         });
     };
 
@@ -117,7 +118,7 @@ define([
         tooltip = new Opentip(portIcon[0][0], PORT_TOOLTIP_OPTS);
         tooltip.setContent(port.name);
         portIcon.on('mouseenter', () => tooltip.show());
-        portIcon.on('mouseout', () => tooltip.hide());
+        portIcon.on('mouseleave', () => tooltip.hide());
         this.$portTooltips[port.id] = tooltip;
     };
 

src/layouts/SidebarLayout/SidebarLayout.css

@@ -0,0 +1,12 @@
+.ui-layout-center .layout-center {
+    left: 40px;
+    height: 100%;
+    width: 100%;
+    position: absolute;
+}
+
+.ui-layout-sidebar {
+    top: 64px;
+    width: 40px;
+    bottom: 27px;
+}

src/layouts/SidebarLayout/SidebarLayout.js

@@ -0,0 +1,57 @@
+/*globals define, */
+define([
+    'layout/CHFLayout/CHFLayout/CHFLayout',
+    'text!./templates/SidebarLayout.html',
+    'css!./SidebarLayout.css'
+], function(
+    CHFLayout,
+    SidebarTemplate
+) {
+    'use strict';
+    
+    var SidebarLayout = function(params) {
+        params = params || {};
+        params.template = SidebarTemplate;
+        CHFLayout.call(this, params);
+    };
+
+    SidebarLayout.prototype = Object.create(CHFLayout.prototype);
+
+    SidebarLayout.prototype.getComponentId = function () {
+        return 'SidebarLayout';
+    };
+
+    /**
+     * Initialize the html page. This example is using the jQuery Layout plugin.
+     *
+     * @return {undefined}
+     */
+    SidebarLayout.prototype.init = function() {
+        CHFLayout.prototype.init.apply(this, arguments);
+        this._sidebarPanel = this._body.find('div.ui-layout-sidebar');
+        this._centerPanel = this._body.find('div.layout-center');
+    };
+
+    /**
+     * Add a panel to a given container. This is defined in the corresponding
+     * layout config JSON file.
+     *
+     * @param {Panel} panel
+     * @param {String} container
+     * @return {undefined}
+     */
+    SidebarLayout.prototype.addToContainer = function(panel, container) {
+        if (container === 'sidebar') {
+            this._sidebarPanel.append(panel.$pEl);
+        } else {
+            CHFLayout.prototype.addToContainer.apply(this, arguments);
+        }
+    };
+
+    SidebarLayout.prototype._onCenterResize = function() {
+        var width = this._centerPanel.width() - this._sidebarPanel.width();
+        this._canvas.setSize(width, this._centerPanel.height());
+    };
+
+    return SidebarLayout;
+});

src/layouts/SidebarLayout/templates/SidebarLayout.html

@@ -0,0 +1,7 @@
+<div class="ui-layout-center" style="position: relative;">
+    <div class="layout-center"></div>
+    <div class="float"></div>
+</div>
+<div class="ui-layout-sidebar"></div>
+<div class="ui-layout-north"></div>
+<div class="ui-layout-south"></div>

src/plugins/CheckLibraries/CheckLibraries.js

@@ -0,0 +1,185 @@
+/*globals define*/
+/*jshint node:true, browser:true*/
+
+define([
+    'text!./metadata.json',
+    'module',
+    'path',
+    'fs',
+    'q',
+    'plugin/PluginBase'
+], function (
+    pluginMetadata,
+    module,
+    path,
+    fs,
+    Q,
+    PluginBase
+) {
+    'use strict';
+
+    pluginMetadata = JSON.parse(pluginMetadata);
+    var __dirname = path.dirname(module.uri),
+        SEEDS_DIR = path.join(__dirname, '..', '..', 'seeds');
+
+    /**
+     * Initializes a new instance of CheckLibraries.
+     * @class
+     * @augments {PluginBase}
+     * @classdesc This class represents the plugin CheckLibraries.
+     * @constructor
+     */
+    var CheckLibraries = function () {
+        // Call base class' constructor.
+        PluginBase.call(this);
+        this.pluginMetadata = pluginMetadata;
+        this.libraries = {};
+    };
+
+    /**
+     * Metadata associated with the plugin. Contains id, name, version, description, icon, configStructue etc.
+     * This is also available at the instance at this.pluginMetadata.
+     * @type {object}
+     */
+    CheckLibraries.metadata = pluginMetadata;
+
+    // Prototypical inheritance from PluginBase.
+    CheckLibraries.prototype = Object.create(PluginBase.prototype);
+    CheckLibraries.prototype.constructor = CheckLibraries;
+
+    /**
+     * Main function for the plugin to execute. This will perform the execution.
+     * Notes:
+     * - Always log with the provided logger.[error,warning,info,debug].
+     * - Do NOT put any user interaction logic UI, etc. inside this method.
+     * - callback always has to be called even if error happened.
+     *
+     * @param {function(string, plugin.PluginResult)} callback - the result callback
+     */
+    CheckLibraries.prototype.main = function (callback) {
+        var tuples;
+
+        return this.getAllLibraries()
+            .then(libs => {
+                tuples = libs.map(lib => {  // map to [name, version, dir]
+                    var version,
+                        hash,
+                        data,
+                        versionPath = this.getSeedVersionPath(lib);
+
+                    try {
+                        this.logger.info(`Checking for version info at ${versionPath}`);
+                        version = fs.readFileSync(versionPath, 'utf8');
+                        this.logger.debug(`${lib} version is ${version}`);
+                        data = fs.readFileSync(this.getSeedHashPath(lib), 'utf8').split(' ');
+                        if (data[1] === version) {
+                            hash = data[0];
+                            this.logger.debug(`${lib} hash is ${hash}`);
+                        }
+                    } catch (e) {
+                        if (!version) {
+                            this.logger.warn(`Could not find library version for ${lib}`);
+                        } else {
+                            this.logger.warn(`Could not find library hash for ${lib}`);
+                        }
+                    }
+
+                    return [lib, version, hash];
+                })
+                .filter(tuple => {
+                    var projVersion = this.getLoadedVersion(tuple[0]),
+                        latest = tuple[1].replace(/\s+/g, '');
+
+                    this.logger.info(`${tuple[0]} version info:\n${projVersion} ` +
+                        `(project)\n${latest} (latest)`);
+                    return projVersion < latest;
+                });
+
+                return Q.all(tuples.map(tuple => this.uploadSeed.apply(this, tuple)));
+            })
+            .then(hashes => {
+                var name;
+
+                for (var i = hashes.length; i--;) {
+                    name = tuples[i][0];
+                    this.createMessage(this.libraries[name], `${name} ${hashes[i]}`);
+                }
+
+                this.logger.info(`Found ${hashes.length} out of date libraries`);
+                this.result.setSuccess(true);
+                callback(null, this.result);
+            })
+            .fail(err => {
+                this.logger.error(`Could not check the libraries: ${err}`);
+                callback(err, this.result);
+            });
+    };
+
+    CheckLibraries.prototype.getSeedDir = function (name) {
+        return path.join(SEEDS_DIR, name);
+    };
+
+    CheckLibraries.prototype.getSeedDataPath = function (name) {
+        return path.join(this.getSeedDir(name), name + '.webgmex');
+    };
+
+    CheckLibraries.prototype.getSeedHashPath = function (name) {
+        return path.join(this.getSeedDir(name), 'hash.txt');
+    };
+
+    CheckLibraries.prototype.getSeedVersionPath = function (name) {
+        return path.join(this.getSeedDir(name), 'version.txt');
+    };
+
+    CheckLibraries.prototype.uploadSeed = function (name, version, hash) {
+        if (!hash) {  // Upload the seed
+            // Get the data
+            return Q.nfcall(fs.readFile, this.getSeedDataPath(name))
+                .then(data => {
+                    this.logger.info(`Uploading new version of ${name} (${version})`);
+                    return this.blobClient.putFile(`${name}.webgmex`, data);
+                })
+                .then(newHash => {  // Store the new hash
+                    this.logger.info(`Upload of ${name} finished!`);
+                    hash = newHash;
+                    return Q.nfcall(
+                        fs.writeFile,
+                        this.getSeedHashPath(name),
+                        `${hash} ${version}`
+                    );
+                }).then(() => hash);
+        }
+        return hash;
+    };
+
+    CheckLibraries.prototype.getAllLibraries = function () {
+        var name,
+            names = [];
+
+        return this.core.loadChildren(this.rootNode)
+            .then(children => {
+                for (var i = children.length; i--;) {
+                    if (this.core.isLibraryRoot(children[i])) {
+                        name = this.core.getAttribute(children[i], 'name');
+                        this.libraries[name] = children[i];
+                        names.push(name);
+                    }
+                }
+                if (names.length) {
+                    this.logger.debug(`Found libraries: ${names.join(', ')}`);
+                } else {
+                    this.logger.debug('Found no libraries!');
+                }
+                return names;
+            });
+    };
+
+    CheckLibraries.prototype.getLoadedVersion = function (libName) {
+        var node = this.libraries[libName],
+            version = this.core.getAttribute(node, 'version');  // using library root hash
+
+        return version;
+    };
+
+    return CheckLibraries;
+});

src/plugins/CheckLibraries/metadata.json

@@ -0,0 +1,14 @@
+{
+  "id": "CheckLibraries",
+  "name": "CheckLibraries",
+  "version": "0.1.0",
+  "description": "",
+  "icon": {
+    "class": "glyphicon glyphicon-cog",
+    "src": ""
+  },
+  "disableServerSideExecution": false,
+  "disableBrowserSideExecution": true,
+  "writeAccessRequired": false,
+  "configStructure": []
+}

src/plugins/CreateTorchMeta/schemas/nn.json

@@ -65,7 +65,14 @@
       "affine"
     ],
     "setters": {},
-    "defaults": {},
+    "types": {
+      "eps": "number",
+      "momentum": "number"
+    },
+    "defaults": {
+      "momentum": 0.1,
+      "eps": 0.00001
+    },
     "type": "Misc"
   },
   {
@@ -82,6 +89,35 @@
     "defaults": {},
     "type": "Misc"
   },
+  {
+    "name": "Bottle",
+    "baseType": "Container",
+    "params": [
+      "module",
+      "nInputDim",
+      "nOutputDim"
+    ],
+    "setters": {},
+    "types": {
+      "nInputDim": "number",
+      "module": "nn.Module"
+    },
+    "defaults": {
+      "nInputDim": 2
+    },
+    "type": "Container"
+  },
+  {
+    "name": "CAdd",
+    "baseType": "Module",
+    "params": [
+      "params"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Misc"
+  },
   {
     "name": "CAddTable",
     "baseType": "Module",
@@ -102,6 +138,24 @@
     "defaults": {},
     "type": "Misc"
   },
+  {
+    "name": "CMaxTable",
+    "baseType": "Module",
+    "params": [],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Misc"
+  },
+  {
+    "name": "CMinTable",
+    "baseType": "Module",
+    "params": [],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Misc"
+  },
   {
     "name": "CMul",
     "baseType": "Module",
@@ -175,7 +229,7 @@
     "setters": {},
     "types": {},
     "defaults": {},
-    "type": "Containers"
+    "type": "Simple"
   },
   {
     "name": "ConcatTable",
@@ -184,7 +238,7 @@
     "setters": {},
     "types": {},
     "defaults": {},
-    "type": "Misc"
+    "type": "Container"
   },
   {
     "name": "Contiguous",
@@ -296,13 +350,16 @@
     "params": [
       "p",
       "v1",
-      "inplace"
+      "inplace",
+      "stochasticInference"
     ],
     "setters": {},
     "types": {
-      "p": "number"
+      "p": "number",
+      "stochasticInference": "boolean"
     },
     "defaults": {
+      "stochasticInference": false,
       "p": 0.5
     },
     "type": "Simple"
@@ -357,8 +414,11 @@
   {
     "name": "GradientReversal",
     "baseType": "Module",
-    "params": [],
+    "params": [
+      "lambda"
+    ],
     "setters": {},
+    "types": {},
     "defaults": {},
     "type": "Misc"
   },
@@ -382,7 +442,8 @@
     "baseType": "Module",
     "params": [
       "min_value",
-      "max_value"
+      "max_value",
+      "inplace"
     ],
     "setters": {},
     "types": {
@@ -508,9 +569,7 @@
   {
     "name": "Log",
     "baseType": "Module",
-    "params": [
-      "inputSize"
-    ],
+    "params": [],
     "setters": {},
     "types": {},
     "defaults": {},
@@ -865,7 +924,7 @@
     "setters": {},
     "types": {},
     "defaults": {},
-    "type": "Misc"
+    "type": "Container"
   },
   {
     "name": "ParallelCriterion",
@@ -885,7 +944,7 @@
     "setters": {},
     "types": {},
     "defaults": {},
-    "type": "Misc"
+    "type": "Container"
   },
   {
     "name": "PartialLinear",
@@ -900,6 +959,17 @@
     "defaults": {},
     "type": "Misc"
   },
+  {
+    "name": "PixelShuffle",
+    "baseType": "Module",
+    "params": [
+      "upscaleFactor"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Misc"
+  },
   {
     "name": "Power",
     "baseType": "Module",
@@ -939,6 +1009,17 @@
     "defaults": {},
     "type": "Transfer"
   },
+  {
+    "name": "ReLU6",
+    "baseType": "Module",
+    "params": [
+      "inplace"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Transfer"
+  },
   {
     "name": "Replicate",
     "baseType": "Module",
@@ -1156,6 +1237,18 @@
     "defaults": {},
     "type": "Convolution"
   },
+  {
+    "name": "SpatialClassNLLCriterion",
+    "baseType": "Criterion",
+    "params": [
+      "weights",
+      "sizeAverage"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Criterion"
+  },
   {
     "name": "SpatialContrastiveNormalization",
     "baseType": "Module",
@@ -1279,6 +1372,56 @@
     },
     "type": "Convolution"
   },
+  {
+    "name": "SpatialDilatedConvolution",
+    "baseType": "SpatialConvolution",
+    "params": [
+      "nInputPlane",
+      "nOutputPlane",
+      "kW",
+      "kH",
+      "dW",
+      "dH",
+      "padW",
+      "padH",
+      "dilationW",
+      "dilationH"
+    ],
+    "setters": {},
+    "types": {
+      "dilationW": "number",
+      "dilationH": "number"
+    },
+    "defaults": {
+      "dilationH": 1,
+      "dilationW": 1
+    },
+    "type": "Convolution"
+  },
+  {
+    "name": "SpatialDilatedMaxPooling",
+    "baseType": "SpatialMaxPooling",
+    "params": [
+      "kW",
+      "kH",
+      "dW",
+      "dH",
+      "padW",
+      "padH",
+      "dilationW",
+      "dilationH"
+    ],
+    "setters": {},
+    "types": {
+      "dilationW": "number",
+      "dilationH": "number"
+    },
+    "defaults": {
+      "dilationH": 1,
+      "dilationW": 1
+    },
+    "type": "Misc"
+  },
   {
     "name": "SpatialDivisiveNormalization",
     "baseType": "Module",
@@ -1301,13 +1444,16 @@
     "name": "SpatialDropout",
     "baseType": "Module",
     "params": [
-      "p"
+      "p",
+      "stochasticInference"
     ],
     "setters": {},
     "types": {
-      "p": "number"
+      "p": "number",
+      "stochasticInference": "boolean"
     },
     "defaults": {
+      "stochasticInference": false,
       "p": 0.5
     },
     "type": "Convolution"
@@ -1387,6 +1533,14 @@
     "defaults": {},
     "type": "Convolution"
   },
+  {
+    "name": "SpatialLogSoftMax",
+    "baseType": "Module",
+    "params": [],
+    "setters": {},
+    "defaults": {},
+    "type": "Misc"
+  },
   {
     "name": "SpatialMaxPooling",
     "baseType": "Module",
@@ -1497,6 +1651,17 @@
     },
     "type": "Convolution"
   },
+  {
+    "name": "SpatialUpSamplingBilinear",
+    "baseType": "Module",
+    "params": [
+      "params"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Misc"
+  },
   {
     "name": "SpatialUpSamplingNearest",
     "baseType": "Module",
@@ -1627,6 +1792,22 @@
     "defaults": {},
     "type": "Convolution"
   },
+  {
+    "name": "TemporalDynamicKMaxPooling",
+    "baseType": "Module",
+    "params": [
+      "minK",
+      "factor"
+    ],
+    "setters": {},
+    "types": {
+      "factor": "number"
+    },
+    "defaults": {
+      "factor": 0
+    },
+    "type": "Misc"
+  },
   {
     "name": "TemporalMaxPooling",
     "baseType": "Module",
@@ -1764,16 +1945,81 @@
     "type": "Convolution"
   },
   {
-    "name": "VolumetricDropout",
-    "baseType": "Module",
+    "name": "VolumetricDilatedConvolution",
+    "baseType": "VolumetricConvolution",
     "params": [
-      "p"
+      "nInputPlane",
+      "nOutputPlane",
+      "kT",
+      "kW",
+      "kH",
+      "dT",
+      "dW",
+      "dH",
+      "padT",
+      "padW",
+      "padH",
+      "dilationT",
+      "dilationW",
+      "dilationH"
     ],
     "setters": {},
     "types": {
-      "p": "number"
+      "dilationT": "number",
+      "dilationW": "number",
+      "dilationH": "number"
     },
     "defaults": {
+      "dilationH": 1,
+      "dilationW": 1,
+      "dilationT": 1
+    },
+    "type": "Misc"
+  },
+  {
+    "name": "VolumetricDilatedMaxPooling",
+    "baseType": "VolumetricMaxPooling",
+    "params": [
+      "kT",
+      "kW",
+      "kH",
+      "dT",
+      "dW",
+      "dH",
+      "padT",
+      "padW",
+      "padH",
+      "dilationT",
+      "dilationW",
+      "dilationH"
+    ],
+    "setters": {},
+    "types": {
+      "dilationT": "number",
+      "dilationW": "number",
+      "dilationH": "number"
+    },
+    "defaults": {
+      "dilationH": 1,
+      "dilationW": 1,
+      "dilationT": 1
+    },
+    "type": "Misc"
+  },
+  {
+    "name": "VolumetricDropout",
+    "baseType": "Module",
+    "params": [
+      "p",
+      "stochasticInference"
+    ],
+    "setters": {},
+    "types": {
+      "p": "number",
+      "stochasticInference": "boolean"
+    },
+    "defaults": {
+      "stochasticInference": false,
       "p": 0.5
     },
     "type": "Convolution"
@@ -1864,6 +2110,22 @@
     "defaults": {},
     "type": "Convolution"
   },
+  {
+    "name": "VolumetricReplicationPadding",
+    "baseType": "Module",
+    "params": [
+      "pleft",
+      "pright",
+      "ptop",
+      "pbottom",
+      "pfront",
+      "pback"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Misc"
+  },
   {
     "name": "WeightedEuclidean",
     "baseType": "Module",
@@ -1887,4 +2149,4 @@
     "defaults": {},
     "type": "Criterion"
   }
-]
+]

src/plugins/ExecuteJob/ExecuteJob.Files.js

@@ -0,0 +1,338 @@
+/*globals define*/
+define([
+    './templates/index',
+    'q',
+    'underscore',
+    'deepforge/Constants'
+], function(
+    Templates,
+    Q,
+    _,
+    CONSTANTS
+) {
+
+    var ExecuteJob = function() {
+    };
+
+    ExecuteJob.prototype.createOperationFiles = function (node) {
+        var files = {};
+        // For each operation, generate the output files:
+        //   inputs/<arg-name>/init.lua  (respective data deserializer)
+        //   pointers/<name>/init.lua  (result of running the main plugin on pointer target - may need a rename)
+        //   outputs/<name>/  (make dirs for each of the outputs)
+        //   outputs/init.lua  (serializers for data outputs)
+        //
+        //   attributes.lua (returns lua table of operation attributes)
+        //   init.lua (main file -> calls main and serializes outputs)
+        //   <name>.lua (entry point -> calls main operation code)
+
+        // add the given files
+        this.logger.info('About to create dist execution files');
+        files['start.js'] = _.template(Templates.START)(CONSTANTS);
+        return this.createEntryFile(node, files)
+            .then(() => this.createClasses(node, files))
+            .then(() => this.createCustomLayers(node, files))
+            .then(() => this.createInputs(node, files))
+            .then(() => this.createOutputs(node, files))
+            .then(() => this.createMainFile(node, files))
+            .then(() => {
+                this.createAttributeFile(node, files);
+                return Q.ninvoke(this, 'createPointers', node, files);
+            });
+    };
+
+    ExecuteJob.prototype.createEntryFile = function (node, files) {
+        this.logger.info('Creating entry files...');
+        return this.getOutputs(node)
+            .then(outputs => {
+                var name = this.getAttribute(node, 'name'),
+                    content = {};
+
+                // inputs and outputs
+                content.name = name;
+                content.outputs = outputs;
+
+                files['init.lua'] = _.template(Templates.ENTRY)(content);
+
+                // Create the deepforge file
+                files['deepforge.lua'] = _.template(Templates.DEEPFORGE)(CONSTANTS);
+            });
+    };
+
+    ExecuteJob.prototype.createClasses = function (node, files) {
+        var metaDict = this.core.getAllMetaNodes(this.rootNode),
+            isClass,
+            metanodes,
+            classNodes,
+            inheritanceLvl = {},
+            code;
+
+        this.logger.info('Creating custom layer file...');
+        metanodes = Object.keys(metaDict).map(id => metaDict[id]);
+        isClass = this.getTypeDictFor('Complex', metanodes);
+
+        classNodes = metanodes.filter(node => {
+            var base = this.core.getBase(node),
+                baseId = this.core.getPath(base),
+                count = 1;
+
+            // Count the sets back to a class node
+            while (base) {
+                if (isClass[baseId]) {
+                    inheritanceLvl[this.core.getPath(node)] = count;
+                    return true;
+                }
+                base = this.core.getBase(base);
+                baseId = this.core.getPath(base);
+                count++;
+            }
+
+            return false;
+        });
+
+        // Get the code definitions for each
+        // Sort by levels of inheritance...
+        code = classNodes.sort((a, b) => {
+            var aId = this.core.getPath(a),
+                bId = this.core.getPath(b);
+
+            return inheritanceLvl[aId] > inheritanceLvl[bId];
+        }).map(node =>
+            `require './${this.getAttribute(node, 'name')}.lua'`
+        ).join('\n');
+
+        // Create the class files
+        classNodes.forEach(node => {
+            var name = this.getAttribute(node, 'name');
+            files[`classes/${name}.lua`] = this.getAttribute(node, 'code');
+        });
+
+        // Create the custom layers file
+        files['classes/init.lua'] = code;
+    };
+
+    ExecuteJob.prototype.getTypeDictFor = function (name, metanodes) {
+        var isType = {};
+        // Get all the custom layers
+        for (var i = metanodes.length; i--;) {
+            if (this.getAttribute(metanodes[i], 'name') === name) {
+                isType[this.core.getPath(metanodes[i])] = true;
+            }
+        }
+        return isType;
+    };
+
+    ExecuteJob.prototype.createCustomLayers = function (node, files) {
+        var metaDict = this.core.getAllMetaNodes(this.rootNode),
+            isCustomLayer,
+            metanodes,
+            customLayers,
+            code;
+
+        this.logger.info('Creating custom layer file...');
+        metanodes = Object.keys(metaDict).map(id => metaDict[id]);
+        isCustomLayer = this.getTypeDictFor('CustomLayer', metanodes);
+
+        customLayers = metanodes.filter(node =>
+            this.core.getMixinPaths(node).some(id => isCustomLayer[id]));
+
+        // Get the code definitions for each
+        code = 'require \'nn\'\n\n' + customLayers
+            .map(node => this.getAttribute(node, 'code')).join('\n');
+
+        // Create the custom layers file
+        files['custom-layers.lua'] = code;
+    };
+
+    ExecuteJob.prototype.createInputs = function (node, files) {
+        var tplContents,
+            inputs;
+
+        this.logger.info('Retrieving inputs and deserialize fns...');
+        return this.getInputs(node)
+            .then(allInputs => {
+                // For each input, match the connection with the input name
+                //   [ name, type ] => [ name, type, node ]
+                //
+                // For each input,
+                //  - create the deserializer
+                //  - put it in inputs/<name>/init.lua
+                //  - copy the data asset to /inputs/<name>/init.lua
+                inputs = allInputs
+                    .filter(pair => !!this.getAttribute(pair[2], 'data'));  // remove empty inputs
+
+                files.inputAssets = {};  // data assets
+                return Q.all(inputs.map(pair => {
+                    var name = pair[0],
+                        node = pair[2],
+                        nodeId = this.core.getPath(node),
+                        fromNodeId;
+
+                    // Get the deserialize function. First, try to get it from
+                    // the source method (this guarantees that the correct
+                    // deserialize method is used despite any auto-upcasting
+                    fromNodeId = this.inputPortsFor[nodeId][0] || nodeId;
+
+                    return this.core.loadByPath(this.rootNode, fromNodeId)
+                        .then(fromNode => {
+                            var deserFn,
+                                base,
+                                className;
+
+                            deserFn = this.getAttribute(fromNode, 'deserialize');
+
+                            if (this.isMetaTypeOf(node, this.META.Complex)) {
+                                // Complex objects are expected to define their own
+                                // (static) deserialize factory method
+                                base = this.core.getMetaType(node);
+                                className = this.getAttribute(base, 'name');
+                                deserFn = `return ${className}.deserialize(path)`;
+                            }
+
+                            return {
+                                name: name,
+                                code: deserFn
+                            };
+                        });
+                }));
+            })
+            .then(_tplContents => {
+                tplContents = _tplContents;
+                var hashes = inputs.map(pair => {
+                    var hash = this.getAttribute(pair[2], 'data');
+                    files.inputAssets[pair[0]] = hash;
+                    return {
+                        hash: hash,
+                        name: pair[0]
+                    };
+                });
+
+                return Q.all(hashes.map(pair => 
+                    this.blobClient.getMetadata(pair.hash)
+                        .fail(err => this.onBlobRetrievalFail(node, pair.name, err))));
+            })
+            .then(metadatas => {
+                // Create the deserializer
+                tplContents.forEach((ctnt, i) => {
+                    // Get the name of the given asset
+                    ctnt.filename = metadatas[i].name;
+                    files['inputs/' + ctnt.name + '/init.lua'] = _.template(Templates.DESERIALIZE)(ctnt);
+                });
+                return files;
+            });
+    };
+
+    ExecuteJob.prototype.createOutputs = function (node, files) {
+        // For each of the output types, grab their serialization functions and
+        // create the `outputs/init.lua` file
+        this.logger.info('Creating outputs/init.lua...');
+        return this.getOutputs(node)
+            .then(outputs => {
+                var outputTypes = outputs
+                // Get the serialize functions for each
+                    .map(tuple => {
+                        var node = tuple[2],
+                            serFn = this.getAttribute(node, 'serialize');
+
+                        if (this.isMetaTypeOf(node, this.META.Complex)) {
+                            // Complex objects are expected to define their own
+                            // serialize methods
+                            serFn = 'if data ~= nil then data:serialize(path) end';
+                        }
+
+                        return [tuple[1], serFn];
+                    });
+
+                files['outputs/init.lua'] = _.template(Templates.SERIALIZE)({types: outputTypes});
+            });
+    };
+
+    ExecuteJob.prototype.createMainFile = function (node, files) {
+        this.logger.info('Creating main file...');
+        return this.getInputs(node)
+            .then(inputs => {
+                var name = this.getAttribute(node, 'name'),
+                    code = this.getAttribute(node, 'code'),
+                    pointers = this.core.getPointerNames(node).filter(ptr => ptr !== 'base'),
+                    content = {
+                        name: name
+                    };
+
+                // Get input data arguments
+                content.inputs = inputs
+                    .map(pair => [pair[0], !this.getAttribute(pair[2], 'data')]);  // remove empty inputs
+
+                // Defined variables for each pointers
+                content.pointers = pointers
+                    .map(id => [id, this.core.getPointerPath(node, id) === null]);
+
+                // Add remaining code
+                content.code = code;
+
+                files['main.lua'] = _.template(Templates.MAIN)(content);
+
+                // Set the line offset
+                var lineOffset = this.getLineOffset(files['main.lua'], code);
+                this.setAttribute(node, CONSTANTS.LINE_OFFSET, lineOffset);
+            });
+    };
+
+    ExecuteJob.prototype.getLineOffset = function (main, snippet) {
+        var i = main.indexOf(snippet),
+            lines = main.substring(0, i).match(/\n/g);
+
+        return lines ? lines.length : 0;
+    };
+
+    ExecuteJob.prototype.createAttributeFile = function (node, files) {
+        var skip = ['code', 'stdout', 'execFiles', 'jobId', 'secret'],
+            numOrBool = /^(-?\d+\.?\d*((e|e-)\d+)?|(true|false))$/,
+            table;
+
+        this.logger.info('Creating attributes file...');
+        table = '{\n\t' + this.core.getAttributeNames(node)
+            .filter(attr => skip.indexOf(attr) === -1)
+            .map(name => {
+                var value = this.getAttribute(node, name);
+                if (!numOrBool.test(value)) {
+                    value = `"${value}"`;
+                }
+                return [name, value];
+            })
+            .map(pair => pair.join(' = '))
+            .join(',\n\t') + '\n}';
+
+        files['attributes.lua'] = `-- attributes of ${this.getAttribute(node, 'name')}\nreturn ${table}`;
+    };
+
+    ExecuteJob.prototype.createPointers = function (node, files, cb) {
+        var pointers,
+            nIds;
+
+        this.logger.info('Creating pointers file...');
+        pointers = this.core.getPointerNames(node)
+            .filter(name => name !== 'base')
+            .filter(id => this.core.getPointerPath(node, id) !== null);
+
+        nIds = pointers.map(p => this.core.getPointerPath(node, p));
+        files.ptrAssets = {};
+        Q.all(
+            nIds.map(nId => this.getPtrCodeHash(nId))
+        )
+        .then(resultHashes => {
+            var name = this.getAttribute(node, 'name');
+            this.logger.info(`Pointer generation for ${name} FINISHED!`);
+            resultHashes.forEach((hash, index) => {
+                files.ptrAssets[`pointers/${pointers[index]}/init.lua`] = hash;
+            });
+            return cb(null, files);
+        })
+        .fail(e => {
+            this.logger.error(`Could not generate pointer files for ${this.getAttribute(node, 'name')}: ${e.toString()}`);
+            return cb(e);
+        });
+    };
+
+    return ExecuteJob;
+});
+

src/plugins/ExecuteJob/ExecuteJob.Metadata.js

@@ -0,0 +1,122 @@
+/*globals define*/
+define([
+    'deepforge/Constants'
+], function(
+    CONSTANTS
+) {
+
+    var ExecuteJob = function() {
+    };
+
+    ExecuteJob.prototype[CONSTANTS.GRAPH_CREATE] = function (job, id) {
+        var graph,
+            name = Array.prototype.slice.call(arguments, 2).join(' '),
+            jobId = this.core.getPath(job);
+
+        id = jobId + '/' + id;
+        this.logger.info(`Creating graph ${id} named ${name}`);
+
+        // Check if the graph already exists
+        graph = this._getExistingMetadata(jobId, 'Graph', name);
+        if (!graph) {
+            graph = this.createNode('Graph', job);
+
+            if (name) {
+                this.setAttribute(graph, 'name', name);
+            }
+            this.createIdToMetadataId[graph] = id;
+        }
+
+        this._metadata[id] = graph;
+    };
+
+    ExecuteJob.prototype[CONSTANTS.GRAPH_PLOT] = function (job, id, x, y) {
+        var jobId = this.core.getPath(job),
+            nonNum = /[^\d-\.]*/g,
+            line,
+            points;
+            
+
+        id = jobId + '/' + id;
+        this.logger.info(`Adding point ${x}, ${y} to ${id}`);
+        line = this._metadata[id];
+        if (!line) {
+            this.logger.warn(`Can't add point to non-existent line: ${id}`);
+            return;
+        }
+
+        // Clean the points by removing and special characters
+        x = x.replace(nonNum, '');
+        y = y.replace(nonNum, '');
+        points = this.getAttribute(line, 'points');
+        points += `${x},${y};`;
+        this.setAttribute(line, 'points', points);
+    };
+
+    ExecuteJob.prototype[CONSTANTS.GRAPH_CREATE_LINE] = function (job, graphId, id) {
+        var jobId = this.core.getPath(job),
+            graph = this._metadata[jobId + '/' + graphId],
+            name = Array.prototype.slice.call(arguments, 3).join(' '),
+            line;
+
+        // Create a 'line' node in the given Graph metadata node
+        name = name.replace(/\s+$/, '');
+        line = this.createNode('Line', graph);
+        this.setAttribute(line, 'name', name);
+        this._metadata[jobId + '/' + id] = line;
+        this.createIdToMetadataId[line] = jobId + '/' + id;
+    };
+
+    ExecuteJob.prototype[CONSTANTS.IMAGE.BASIC] =
+    ExecuteJob.prototype[CONSTANTS.IMAGE.UPDATE] =
+    ExecuteJob.prototype[CONSTANTS.IMAGE.CREATE] = function (job, hash, imgId) {
+        var name = Array.prototype.slice.call(arguments, 3).join(' '),
+            imageNode = this._getImageNode(job, imgId, name);
+
+        this.setAttribute(imageNode, 'data', hash);
+    };
+
+    ExecuteJob.prototype[CONSTANTS.IMAGE.NAME] = function (job, imgId) {
+        var name = Array.prototype.slice.call(arguments, 2).join(' '),
+            imageNode = this._getImageNode(job, imgId, name);
+
+        this.setAttribute(imageNode, 'name', name);
+    };
+
+    ExecuteJob.prototype._getImageNode = function (job, imgId, name) {
+        var jobId = this.core.getPath(job),
+            id = jobId + '/IMAGE/' + imgId,
+            imageNode = this._metadata[id];  // Look for the metadata imageNode
+
+        if (!imageNode) {
+
+            // Check if the imageNode already exists
+            imageNode = this._getExistingMetadata(jobId, 'Image', name);
+            if (!imageNode) {
+                this.logger.info(`Creating image ${id} named ${name}`);
+                imageNode = this.createNode('Image', job);
+                this.setAttribute(imageNode, 'name', name);
+                this.createIdToMetadataId[imageNode] = id;
+            }
+            this._metadata[id] = imageNode;
+        }
+        return imageNode;
+    };
+
+    ExecuteJob.prototype._getExistingMetadata = function (jobId, type, name) {
+        var oldMetadata = this._oldMetadataByName[jobId] && this._oldMetadataByName[jobId][type],
+            node,
+            id;
+
+        if (oldMetadata && oldMetadata[name]) {
+            id = oldMetadata[name];
+            node = this._markForDeletion[jobId][id];
+            delete this._markForDeletion[jobId][id];
+            this.createdMetadataIds[jobId].push(id);  // used for resuming jobs
+        }
+
+        return node || null;
+    };
+
+    return ExecuteJob;
+});

src/plugins/ExecuteJob/ExecuteJob.SafeSave.js

@@ -0,0 +1,377 @@
+/*globals define*/
+define([
+    'plugin/PluginBase',
+    'common/storage/constants',
+    'q',
+    'common/util/assert'
+], function(
+    PluginBase,
+    STORAGE_CONSTANTS,
+    Q,
+    assert
+) {
+
+    var CREATE_PREFIX = 'created_node_',
+        INDEX = 1;
+
+    var ExecuteJob = function() {
+        this.forkNameBase = null;
+        this.runningJobHashes = [];
+        this._currentSave = Q();
+    };
+
+    ExecuteJob.prototype.getCreateId = function () {
+        return CREATE_PREFIX + (++INDEX);
+    };
+
+    ExecuteJob.prototype.isCreateId = function (id) {
+        return (typeof id === 'string') && (id.indexOf(CREATE_PREFIX) === 0);
+    };
+
+    ExecuteJob.prototype.createNode = function (baseType, parent) {
+        var id = this.getCreateId(),
+            parentId = this.isCreateId(parent) ? parent : this.core.getPath(parent);
+
+        this.logger.info(`Creating ${id} of type ${baseType} in ${parentId}`);
+        assert(this.META[baseType], `Cannot create node w/ unrecognized type: ${baseType}`);
+        this.creations[id] = {
+            base: baseType,
+            parent: parentId
+        };
+        return id;
+    };
+
+    ExecuteJob.prototype.deleteNode = function (nodeId) {
+        this.deletions.push(nodeId);
+    };
+
+    ExecuteJob.prototype.delAttribute = function (node, attr) {
+        return this.setAttribute(node, attr, null);
+    };
+
+    ExecuteJob.prototype.setAttribute = function (node, attr, value) {
+        var nodeId;
+
+        if (this.isCreateId(node)) {
+            nodeId = node;
+        } else {
+            nodeId = this.core.getPath(node);
+            assert(typeof nodeId === 'string', `Cannot set attribute of ${nodeId}`);
+        }
+
+        if (value !== null) {
+            this.logger.info(`Setting ${attr} of ${nodeId} to ${value}`);
+        } else {
+            this.logger.info(`Deleting ${attr} of ${nodeId}`);
+        }
+
+        if (!this.changes[nodeId]) {
+            this.changes[nodeId] = {};
+        }
+        this.changes[nodeId][attr] = value;
+    };
+
+    ExecuteJob.prototype.getAttribute = function (node, attr) {
+        var nodeId;
+
+        assert(this.deletions.indexOf(nodeId) === -1,
+            `Cannot get ${attr} from deleted node ${nodeId}`);
+
+        // Check if it was newly created
+        if (this.isCreateId(node)) {
+            nodeId = node;
+            assert(this.creations[nodeId], `Creation node not updated: ${nodeId}`);
+            node = this.META[this.creations[nodeId].base];
+        } else {
+            nodeId = this.core.getPath(node);
+        }
+
+        // Check the most recent changes, then the currentChanges, then the model
+        var value = this._getValueFrom(nodeId, attr, node, this.changes) ||
+            this._getValueFrom(nodeId, attr, node, this.currentChanges);
+
+        if (value) {
+            return value;
+        }
+
+        return this.core.getAttribute(node, attr);
+    };
+
+    ExecuteJob.prototype._getValueFrom = function (nodeId, attr, node, changes) {
+        var base;
+        if (changes[nodeId] && changes[nodeId][attr] !== undefined) {
+            // If deleted the attribute, get the default (inherited) value
+            if (changes[nodeId][attr] === null) {
+                base = this.isCreateId(nodeId) ? node : this.core.getBase(node);
+                return this.getAttribute(base, attr);
+            }
+            return changes[nodeId][attr];
+        }
+    };
+
+    ExecuteJob.prototype._applyNodeChanges = function (node, changes) {
+        var attr,
+            value;
+
+        this.logger.info(`About to apply changes for ${this.core.getPath(node)}`);
+        for (var i = changes.length; i--;) {
+            attr = changes[i][0];
+            value = changes[i][1];
+            if (value !== null) {
+                this.logger.info(`Setting ${attr} to ${value} (${this.core.getPath(node)})`);
+                this.core.setAttribute(node, attr, value);
+            } else {
+                this.core.delAttribute(node, attr);
+            }
+        }
+        return node;
+    };
+
+    ExecuteJob.prototype.applyModelChanges = function () {
+        return this.applyCreations()
+            .then(() => this.applyChanges())
+            .then(() => this.applyDeletions());
+    };
+
+    ExecuteJob.prototype.applyChanges = function () {
+        var nodeIds = Object.keys(this.changes),
+            attrs,
+            value,
+            changes,
+            promises = [],
+            changesFor = {},
+            id,
+            promise;
+
+        this.logger.info('Collecting changes to apply in commit');
+
+        for (var i = nodeIds.length; i--;) {
+            changes = [];
+            attrs = Object.keys(this.changes[nodeIds[i]]);
+            for (var a = attrs.length; a--;) {
+                value = this.changes[nodeIds[i]][attrs[a]];
+                changes.push([attrs[a], value]);
+            }
+            changesFor[nodeIds[i]] = changes;
+
+            assert(changes, `changes are invalid for ${nodeIds[i]}: ${changes}`);
+            assert(!this.isCreateId(nodeIds[i]),
+                `Creation id not resolved to actual id: ${nodeIds[i]}`);
+            promise = this.core.loadByPath(this.rootNode, nodeIds[i]);
+            promises.push(promise);
+        }
+
+        this.currentChanges = this.changes;
+        this.changes = {};
+        // Need to differentiate between read/write changes.
+        this.logger.info(`About to apply changes for ${promises.length} nodes`);
+        return Q.all(promises)
+            .then(nodes => {
+                for (var i = nodes.length; i--;) {
+                    id = this.core.getPath(nodes[i]);
+                    assert(nodes[i], `node is ${nodes[i]} (${nodeIds[i]})`);
+                    this._applyNodeChanges(nodes[i], changesFor[id]);
+                }
+
+                // Local model is now up-to-date. No longer need currentChanges
+                this.currentChanges = {};
+            });
+    };
+
+    ExecuteJob.prototype.applyCreations = function () {
+        var nodeIds = Object.keys(this.creations),
+            tiers = this.createCreationTiers(nodeIds),
+            creations = this.creations,
+            newIds = {},
+            promise = Q(),
+            tier;
+
+        this.logger.info('Applying node creations');
+        for (var i = 0; i < tiers.length; i++) {
+            tier = tiers[i];
+            // Chain the promises, loading each tier sequentially
+            promise = promise.then(this.applyCreationTier.bind(this, creations, newIds, tier));
+        }
+
+        this.creations = {};
+        return promise;
+    };
+
+    ExecuteJob.prototype.applyCreationTier = function (creations, newIds, tier) {
+        var promises = [],
+            parentId,
+            node;
+
+        for (var j = tier.length; j--;) {
+            node = creations[tier[j]];
+            assert(node, `Could not find create info for ${tier[j]}`);
+            parentId = newIds[node.parent] || node.parent;
+            promises.push(this.applyCreation(tier[j], node.base, parentId));
+        }
+        return Q.all(promises).then(nodes => {
+            // Record the newIds so they can be used to resolve creation ids
+            // in subsequent tiers
+            for (var i = tier.length; i--;) {
+                newIds[tier[i]] = this.core.getPath(nodes[i]);
+            }
+        });
+    };
+
+    // Figure out the dependencies between nodes to create.
+    // eg, if newId1 is to be created in newId2, then newId2 will
+    // be in an earlier tier than newId1. Essentially a topo-sort
+    // on a tree structure
+    ExecuteJob.prototype.createCreationTiers = function (nodeIds) {
+        var tiers = [],
+            prevTier = {},
+            tier = {},
+            id,
+            prevLen,
+            i;
+
+        // Create first tier (created inside existing nodes)
+        for (i = nodeIds.length; i--;) {
+            id = nodeIds[i];
+            if (!this.isCreateId(this.creations[id].parent)) {
+                tier[id] = true;
+                nodeIds.splice(i, 1);
+            }
+        }
+        prevTier = tier;
+        tiers.push(Object.keys(tier));
+
+        // Now, each tier consists of the nodes to be created inside a
+        // node from the previous tier
+        while (nodeIds.length) {
+            prevLen = nodeIds.length;
+            tier = {};
+            for (i = nodeIds.length; i--;) {
+                id = nodeIds[i];
+                if (prevTier[this.creations[id].parent]) {
+                    tier[id] = true;
+                    nodeIds.splice(i, 1);
+                }
+            }
+            prevTier = tier;
+            tiers.push(Object.keys(tier));
+            // Every iteration should find at least one node
+            assert(prevLen > nodeIds.length,
+                `Created empty create tier! Remaining: ${nodeIds.join(', ')}`);
+        }
+
+        return tiers;
+    };
+
+    ExecuteJob.prototype.applyCreation = function (tmpId, baseType, parentId) {
+        var base = this.META[baseType],
+            nodeId,
+            id;
+
+        this.logger.info(`Applying creation of ${tmpId} (${baseType}) in ${parentId}`);
+
+        assert(!this.isCreateId(parentId),
+            `Did not resolve parent id: ${parentId} for ${tmpId}`);
+        assert(base, `Invalid base type: ${baseType}`);
+        return this.core.loadByPath(this.rootNode, parentId)
+            .then(parent => this.core.createNode({base, parent}))
+            .then(node => {  // Update the _metadata records
+                id = this.createIdToMetadataId[tmpId];
+                delete this.createIdToMetadataId[tmpId];
+                this._metadata[id] = node;
+
+                // Update creations
+                nodeId = this.core.getPath(node);
+                if (this.changes[tmpId]) {
+                    assert(!this.changes[nodeId],
+                        `Newly created node cannot already have changes! (${nodeId})`);
+                    this.changes[nodeId] = this.changes[tmpId];
+                    delete this.changes[tmpId];
+                }
+                return node;
+            });
+    };
+
+    ExecuteJob.prototype.applyDeletions = function () {
+        var deletions = this.deletions;
+
+        this.deletions = [];
+        return Q.all(deletions.map(id => this.core.loadByPath(this.rootNode, id)))
+            .then(nodes => {
+                for (var i = nodes.length; i--;) {
+                    this.core.deleteNode(nodes[i]);
+                }
+            });
+    };
+
+    // Override 'save' to notify the user on fork
+    ExecuteJob.prototype.save = function (msg) {
+        this._currentSave = this._currentSave
+            .then(() => this.updateForkName(this.forkNameBase))
+            .then(() => this.applyModelChanges())
+            .then(() => PluginBase.prototype.save.call(this, msg))
+            .then(result => {
+                this.logger.info(`Save finished w/ status: ${result.status}`);
+                if (result.status === STORAGE_CONSTANTS.FORKED) {
+                    return this.onSaveForked(result.forkName);
+                } else if (result.status === STORAGE_CONSTANTS.MERGED) {
+                    this.logger.debug('Merged changes. About to update plugin nodes');
+                    return this.updateNodes();
+                }
+            });
+
+        return this._currentSave;
+    };
+
+    ExecuteJob.prototype.onSaveForked = function (forkName) {
+        var name = this.getAttribute(this.activeNode, 'name'),
+            msg = `"${name}" execution has forked to "${forkName}"`;
+        this.currentForkName = forkName;
+
+        this.logManager.fork(forkName);
+        this.runningJobHashes.forEach(jobId => this.originManager.fork(jobId, forkName));
+        this.sendNotification(msg);
+    };
+
+    ExecuteJob.prototype.updateNodes = function (hash) {
+        var activeId = this.core.getPath(this.activeNode);
+
+        hash = hash || this.currentHash;
+        return Q.ninvoke(this.project, 'loadObject', hash)
+            .then(commitObject => {
+                return this.core.loadRoot(commitObject.root);
+            })
+            .then(rootObject => {
+                this.rootNode = rootObject;
+                return this.core.loadByPath(rootObject,activeId);
+            })
+            .then(activeObject => this.activeNode = activeObject)
+            .then(() => {
+                var metaNames = Object.keys(this.META);
+
+                return Q.all(metaNames.map(name => this.updateMetaNode(name)));
+            })
+            .then(() => {
+                var mdNodes,
+                    mdIds;
+
+                mdIds = Object.keys(this._metadata)
+                    .filter(id => !this.isCreateId(this._metadata[id]));
+
+                mdNodes = mdIds.map(id => this.core.getPath(this._metadata[id]))
+                    .map(nodeId => this.core.loadByPath(this.rootNode, nodeId));
+
+                return Q.all(mdNodes).then(nodes => {
+                    for (var i = nodes.length; i--;) {
+                        this._metadata[mdIds[i]] = nodes[i];
+                    }
+                });
+            });
+    };
+
+    ExecuteJob.prototype.updateMetaNode = function (name) {
+        var id = this.core.getPath(this.META[name]);
+        return this.core.loadByPath(this.rootNode, id).then(node => this.META[name] = node);
+    };
+
+    return ExecuteJob;
+});

src/plugins/ExecuteJob/templates/deepforge.ejs

@@ -1,6 +1,13 @@
 -- Instantiate the deepforge object
 deepforge = {}
 
+function deepforge.initialize()
+    require 'nn'
+    require 'rnn'
+    require './classes/init'
+    require './custom-layers'
+end
+
 function deepforge.id()
     if __deepforge_id == nil then
         __deepforge_id = 0
@@ -16,7 +23,7 @@ function deepforge._cmd(...)
     for i=1,n do
         cmd = cmd .. ' ' .. tostring(arg[i])
     end
-    print(cmd)
+    print(cmd .. ' ')  -- guarantee ends w/ space
 end
 
 -- Graph support
@@ -46,7 +53,7 @@ function Graph:line(name, opts)
 end
 
 -- Image support
-function deepforge.image(name, tensor)
+local function saveImage(name, tensor)
     require 'image'
     require 'paths'
 
@@ -59,7 +66,32 @@ function deepforge.image(name, tensor)
     end
 
     image.save(path, tensor)
-    deepforge._cmd("<%= IMAGE %>", name)
+end
+
+function deepforge.image(name, tensor)
+    saveImage(name, tensor)
+    deepforge._cmd("<%= IMAGE.BASIC %>", deepforge.id(), name)
+end
+
+Image = torch.class('deepforge.Image')
+function Image:__init(name, tensor)
+    self.id = deepforge.id()
+    self.name = name
+
+    if tensor ~= nil then
+        saveImage(name, tensor)
+        deepforge._cmd('<%= IMAGE.CREATE %>', self.id, self.name)
+    end
+end
+
+function Image:update(tensor)
+    saveImage(self.name, tensor)
+    deepforge._cmd('<%= IMAGE.UPDATE %>', self.id, self.name)
+end
+
+function Image:title(name)
+    self.name = name
+    deepforge._cmd('<%= IMAGE.NAME %>', self.id, self.name)
 end
 
 return deepforge

src/plugins/ExecuteJob/templates/main.ejs

@@ -1,15 +1,12 @@
--- load custom layers
-require './custom-layers'
-
--- load custom class definitions
-require './classes'
+-- load custom layers and classes
+deepforge.initialize()
 
 -- input data<% inputs.forEach(function(pair) { var input = pair[0], isNil = pair[1];%>
-<%= isNil ? 'local ' : ''%><%= input %> = <% if (isNil) { %>nil<% } else { %>require './inputs/<%= input %>'<%}}); %>
+local <%= input %> = <% if (isNil) { %>nil<% } else { %>require './inputs/<%= input %>'<%}}); %>
 
 -- load references<% pointers.forEach(function(pair) { var pointer = pair[0], isNil = pair[1];%>
-<%= isNil ? 'local ' : ''%><%= pointer %> = <% if (isNil) { %>nil<% } else { %>require './pointers/<%= pointer %>'<%}}); %>
-attributes = require './attributes'
+local <%= pointer %> = <% if (isNil) { %>nil<% } else { %>require './pointers/<%= pointer %>'<%}}); %>
+local attributes = require './attributes'
 
 -- main operation code for <%= name %>
 <%= code %>

src/plugins/ExecuteJob/templates/start.ejs

@@ -12,7 +12,7 @@ var spawn = require('child_process').spawn,
 
 // Get the BlobClient...
 var COMMAND_PREFIX = '<%= START_CMD %>',
-    IMAGE = '<%= IMAGE %>',
+    IMAGE = '<%= IMAGE.PREFIX %>',
     requirejs = require('webgme').requirejs,
     remainingImageCount = 0,
     exitCode = null;
@@ -80,7 +80,7 @@ requirejs([
 
     var uploadImage = function(line) {
         var args = line.split(/\s+/),
-            name = args.slice(2).join(' ').replace(/\s+$/, ''),
+            name = args.slice(3).join(' ').replace(/\s+$/, ''),
             filename = 'metadata/' + name + '.png';
 
         // Upload the image from metadata/

src/plugins/ExecutePipeline/ExecutePipeline.js

@@ -6,6 +6,7 @@ define([
     'plugin/ExecuteJob/ExecuteJob/ExecuteJob',
     'common/storage/constants',
     'common/core/constants',
+    'deepforge/Constants',
     'q',
     'text!./metadata.json',
     'underscore'
@@ -13,6 +14,7 @@ define([
     CreateExecution,
     ExecuteJob,
     STORAGE_CONSTANTS,
+    GME_CONSTANTS,
     CONSTANTS,
     Q,
     pluginMetadata,
@@ -31,9 +33,14 @@ define([
     var ExecutePipeline = function () {
         // Call base class' constructor.
         CreateExecution.call(this);
+        ExecuteJob.call(this);
         this.pluginMetadata = pluginMetadata;
 
-        this._currentSave = Q();
+        this.changes = {};
+        this.currentChanges = {};  // read-only changes being applied
+        this.creations = {};
+        this.deletions = [];
+        this.createIdToMetadataId = {};
         this.initRun();
     };
 
@@ -94,17 +101,15 @@ define([
      * @param {function(string, plugin.PluginResult)} callback - the result callback
      */
     ExecutePipeline.prototype.main = function (callback) {
-        // This will probably need to execute the operations, too, because the
-        // inputs for the next operation cannot be created until the inputs have
-        // been generated
+        var startPromise,
+            runId;
 
         this.initRun();
-        var startPromise;
         if (this.core.isTypeOf(this.activeNode, this.META.Pipeline)) {
             // If starting with a pipeline, we will create an Execution first
             startPromise = this.createExecution(this.activeNode)
                 .then(execNode => {
-                    this.logger.debug(`Finished creating execution "${this.core.getAttribute(execNode, 'name')}"`);
+                    this.logger.debug(`Finished creating execution "${this.getAttribute(execNode, 'name')}"`);
                     this.activeNode = execNode;
                 });
         } else if (this.core.isTypeOf(this.activeNode, this.META.Execution)) {
@@ -118,49 +123,133 @@ define([
         this.currentForkName = null;
 
         startPromise
-        .then(() => this.core.loadSubTree(this.activeNode))
-        .then(subtree => {
-            var children = subtree
-                .filter(n => this.core.getParent(n) === this.activeNode);
+            .then(() => this.core.loadSubTree(this.activeNode))
+            .then(subtree => {
+                var children;
 
-            this.pipelineName = this.core.getAttribute(this.activeNode, 'name');
-            this.logger.debug(`Loaded subtree of ${this.pipelineName}. About to build cache`);
-            this.buildCache(subtree);
-            this.logger.debug('Parsing execution for job inter-dependencies');
-            this.parsePipeline(children);  // record deps, etc
+                children = subtree
+                    .filter(n => this.core.getParent(n) === this.activeNode);
 
-            this.logger.debug('Clearing old results');
-            return this.clearResults();
-        })
-        .then(() => this.executePipeline())
-        .fail(e => this.logger.error(e));
+                this.pipelineName = this.getAttribute(this.activeNode, 'name');
+                this.forkNameBase = this.pipelineName;
+                this.logger.debug(`Loaded subtree of ${this.pipelineName}. About to build cache`);
+                this.buildCache(subtree);
+                this.logger.debug('Parsing execution for job inter-dependencies');
+                this.parsePipeline(children);  // record deps, etc
+
+                // Detect if resuming execution
+                runId = this.getAttribute(this.activeNode, 'runId');
+                return this.isResuming().then(resuming => {
+                    if (resuming) {
+                        this.currentRunId = runId;
+                        this.startExecHeartBeat();
+                        return this.resumePipeline();
+                    }
+                    return this.startPipeline();
+                });
+
+            })
+            .fail(e => this.logger.error(e));
     };
 
-    // Override 'save' to prevent race conditions while saving
-    ExecutePipeline.prototype.save = function (msg) {
-        // When 'save'  is called, it should still finish any current save op
-        // before continuing
-        this._currentSave = this._currentSave
-            .then(() => this.updateForkName(this.pipelineName))
-            .then(() => CreateExecution.prototype.save.call(this, msg))
-            .then(result => {
-                var msg;
-                if (result.status === STORAGE_CONSTANTS.FORKED) {
-                    this.currentForkName = result.forkName;
-                    msg = `"${this.pipelineName}" execution has forked to "${result.forkName}"`;
-                    this.sendNotification(msg);
-                } else if (result.status === STORAGE_CONSTANTS.MERGED) {
-                    this.logger.debug('Merged changes. About to update plugin nodes');
-                    return this.updateNodes();
+    ExecutePipeline.prototype.isResuming = function () {
+        var currentlyRunning = this.getAttribute(this.activeNode, 'status') === 'running',
+            runId = this.getAttribute(this.activeNode, 'runId');
+
+        if (runId && currentlyRunning) {
+            // Verify that it is on the correct branch
+            return this.originManager.getOrigin(runId)
+                .then(origin => {
+                    if (origin && origin.branch === this.branchName) {
+                        return this.pulseClient.check(runId)
+                            // If it is dead (not unknown!), then resume
+                            .then(status => status === CONSTANTS.PULSE.DEAD);
+                    } else {
+                        return false;
+                    }
+                });
+        }
+        return Q().then(() => false);
+    };
+
+    ExecutePipeline.prototype.resumePipeline = function () {
+        var nodes = Object.keys(this.nodes).map(id => this.nodes[id]),
+            allJobs = nodes.filter(node => this.core.isTypeOf(node, this.META.Job)),
+            status,
+            jobs = {
+                success: [],
+                failed: [],
+                running: [],
+                pending: []
+            };
+
+        this.logger.info(`Resuming pipeline execution: ${this.currentRunId}`);
+
+        // Get all completed jobs' operations and update records for these
+        for (var i = allJobs.length; i--;) {
+            status = this.getAttribute(allJobs[i], 'status');
+            if (!jobs[status]) {
+                jobs[status] = [];
+            }
+
+            // If any running jobs are missing jobIds, set them to pending
+            if (status === 'running' && !this.canResumeJob(allJobs[i])) {
+                jobs.pending.push(allJobs[i]);
+            } else {
+                jobs[status].push(allJobs[i]);
+            }
+        }
+
+        // Remove finished jobs from incomingCounts
+        jobs.success.concat(jobs.failed, jobs.running)
+            .map(job => this.core.getPath(job))
+            .forEach(id => delete this.incomingCounts[id]);
+
+        return Q.all(allJobs.map(job => this.recordOldMetadata(job, true)))
+            .then(() => Q.all(jobs.success.map(job => this.getOperation(job))))
+            .then(ops => ops.forEach(op => this.updateJobCompletionRecords(op)))
+            .then(() => {
+                
+                if (jobs.running.length) {  // Resume all running jobs
+                    return Q.all(jobs.running.map(job => this.resumeJob(job)));
+                } else if (this.completedCount === this.totalCount) {
+                    return this.onPipelineComplete();
+                } else {
+                    // If none are running, try to start the next ones
+                    return this.executeReadyOperations();
                 }
-
-            });
-
-        return this._currentSave;
+            })
+            .fail(err => this._callback(err));
     };
 
-    ExecutePipeline.prototype.updateNodes = function () {
-        var result = ExecuteJob.prototype.updateNodes.call(this);
+    ExecutePipeline.prototype.startPipeline = function () {
+        var rand = Math.floor(Math.random()*10000),
+            commit = this.commitHash.replace('#', '');
+
+        this.logger.debug('Clearing old results');
+        this.currentRunId = `Pipeline_${commit}_${Date.now()}_${rand}`;
+
+        // Record the execution origin
+        this.originManager.record(this.currentRunId, {
+            nodeId: this.core.getPath(this.activeNode),
+            job: 'N/A',
+            execution: this.getAttribute(this.activeNode, 'name')
+        });
+
+        this.startExecHeartBeat();
+        return this.clearResults()
+            .then(() => this.executePipeline())
+            .fail(e => this.logger.error(e));
+    };
+
+    ExecutePipeline.prototype.onSaveForked = function (forkName) {
+        // Update the origin on fork
+        this.originManager.fork(this.currentRunId, forkName);
+        return ExecuteJob.prototype.onSaveForked.call(this, forkName);
+    };
+
+    ExecutePipeline.prototype.updateNodes = function (hash) {
+        var result = ExecuteJob.prototype.updateNodes.call(this, hash);
         return result.then(() => this.updateCache());
     };
 
@@ -176,6 +265,10 @@ define([
         });
     };
 
+    ExecutePipeline.prototype.isExecutionCanceled = function () {
+        return this.getAttribute(this.activeNode, 'status') === 'canceled';
+    };
+
     ExecutePipeline.prototype.isInputData = function (node) {
         var prnt = this.core.getParent(node);
         return this.core.isTypeOf(prnt, this.META.Inputs);
@@ -195,14 +288,15 @@ define([
         nodes.filter(node => this.core.isTypeOf(node, this.META.Job))
             .forEach(node => {
                 this.recordOldMetadata(node);
-                this.core.setAttribute(node, 'status', 'pending');
+                this.setAttribute(node, 'status', 'pending');
             });
 
         // Set the status of the execution to 'running'
-        this.core.setAttribute(this.activeNode, 'status', 'running');
+        this.setAttribute(this.activeNode, 'status', 'running');
         this.logger.info('Setting all jobs status to "pending"');
         this.logger.debug(`Making a commit from ${this.currentHash}`);
-        this.core.setAttribute(this.activeNode, 'startTime', Date.now());
+        this.setAttribute(this.activeNode, 'startTime', Date.now());
+        this.setAttribute(this.activeNode, 'runId', this.currentRunId);
         this.core.delAttribute(this.activeNode, 'endTime');
         return this.save(`Initializing ${this.pipelineName} for execution`);
     };
@@ -280,10 +374,10 @@ define([
     };
 
     ExecutePipeline.prototype.getSiblingIdContaining = function (nodeId) {
-        var parentId = this.core.getPath(this.activeNode) + CONSTANTS.PATH_SEP,
+        var parentId = this.core.getPath(this.activeNode) + GME_CONSTANTS.PATH_SEP,
             relid = nodeId.replace(parentId, '');
 
-        return parentId + relid.split(CONSTANTS.PATH_SEP).shift();
+        return parentId + relid.split(GME_CONSTANTS.PATH_SEP).shift();
     };
 
     ExecutePipeline.prototype.executePipeline = function() {
@@ -294,24 +388,24 @@ define([
     ExecutePipeline.prototype.onOperationFail = function(node, err) {
         var job = this.core.getParent(node),
             id = this.core.getPath(node),
-            name = this.core.getAttribute(node, 'name');
+            name = this.getAttribute(node, 'name');
 
         this.logger.debug(`Operation ${name} (${id}) failed: ${err}`);
-        this.core.setAttribute(job, 'status', 'fail');
+        this.setAttribute(job, 'status', 'fail');
         this.clearOldMetadata(job);
         this.onPipelineComplete(err);
     };
 
     ExecutePipeline.prototype.onOperationCanceled = function(op) {
         var job = this.core.getParent(op);
-        this.core.setAttribute(job, 'status', 'canceled');
+        this.setAttribute(job, 'status', 'canceled');
         this.runningJobs--;
-        this.logger.debug(`${this.core.getAttribute(job, 'name')} has been canceled`);
+        this.logger.debug(`${this.getAttribute(job, 'name')} has been canceled`);
         this.onPipelineComplete();
     };
 
     ExecutePipeline.prototype.onPipelineComplete = function(err) {
-        var name = this.core.getAttribute(this.activeNode, 'name'),
+        var name = this.getAttribute(this.activeNode, 'name'),
             msg = `"${this.pipelineName}" `;
 
         if (err) {
@@ -341,19 +435,57 @@ define([
             msg += 'finished!';
         }
 
-        this.logger.debug(`Pipeline "${name}" complete!`);
-        this.core.setAttribute(this.activeNode, 'endTime', Date.now());
-        this.core.setAttribute(this.activeNode, 'status',
-            (this.pipelineError ? 'failed' : (this.canceled ? 'canceled' : 'success')));
+        return this.isDeleted().then(isDeleted => {
+            this.stopExecHeartBeat();
+            if (!isDeleted) {
 
-        this._finished = true;
-        this.resultMsg(msg);
-        this.save('Pipeline execution finished')
-            .then(() => {
+                this.logger.debug(`Pipeline "${name}" complete!`);
+                this.setAttribute(this.activeNode, 'endTime', Date.now());
+                this.setAttribute(this.activeNode, 'status',
+                    (this.pipelineError ? 'failed' :
+                    (this.canceled ? 'canceled' : 'success')
+                    )
+                );
+
+                this._finished = true;
+                this.resultMsg(msg);
+                this.save('Pipeline execution finished')
+                    .then(() => {
+                        this.result.setSuccess(!this.pipelineError);
+                        this._callback(this.pipelineError || null, this.result);
+                    })
+                    .fail(e => this.logger.error(e));
+            } else {  // deleted!
+                this.logger.debug('Execution has been deleted!');
                 this.result.setSuccess(!this.pipelineError);
                 this._callback(this.pipelineError || null, this.result);
+            }
+        });
+
+    };
+
+    ExecutePipeline.prototype.isDeleted = function () {
+        var activeId = this.core.getPath(this.activeNode);
+
+        // Check if the current execution has been deleted
+        return this.project.getBranchHash(this.branchName)
+            .then(hash => this.updateNodes(hash))
+            .then(() => this.core.loadByPath(this.rootNode, activeId))
+            .then(node => {
+                var deleted = node === null,
+                    msg = `Verified that execution is ${deleted ? '' : 'not '}deleted`;
+
+                this.logger.debug(msg);
+                return deleted;
             })
-            .fail(e => this.logger.error(e));
+            .fail(err => this.logger.error(err));
+    };
+
+    ExecutePipeline.prototype.onPipelineDeleted = function () {
+        var msg = `${this.pipelineName} has been deleted`;
+        this.resultMsg(msg);
+        this.result.setSuccess(true);
+        this._callback(null, this.result);
     };
 
     ExecutePipeline.prototype.executeReadyOperations = function () {
@@ -386,52 +518,25 @@ define([
     };
 
     ExecutePipeline.prototype.onOperationComplete = function (opNode) {
-        var name = this.core.getAttribute(opNode, 'name'),
-            nextPortIds = this.getOperationOutputIds(opNode),
+        var name = this.getAttribute(opNode, 'name'),
             jNode = this.core.getParent(opNode),
-            resultPorts,
             jobId = this.core.getPath(jNode),
+            counts,
             hasReadyOps;
 
         // Set the operation to 'success'!
         this.clearOldMetadata(jNode);
         this.runningJobs--;
-        this.core.setAttribute(jNode, 'status', 'success');
+        this.setAttribute(jNode, 'status', 'success');
         this.logger.info(`Setting ${jobId} status to "success"`);
         this.logger.info(`There are now ${this.runningJobs} running jobs`);
         this.logger.debug(`Making a commit from ${this.currentHash}`);
         this.save(`Operation "${name}" in ${this.pipelineName} completed successfully`)
             .then(() => {
 
-                // Transport the data from the outputs to any connected inputs
-                //   - Get all the connections from each outputId
-                //   - Get the corresponding dst outputs
-                //   - Use these new ids for checking 'hasReadyOps'
-                resultPorts = nextPortIds.map(id => this.inputPortsFor[id])
-                    .reduce((l1, l2) => l1.concat(l2), []);
+                counts = this.updateJobCompletionRecords(opNode);
+                hasReadyOps = counts.indexOf(0) > -1;
 
-                resultPorts
-                    .map((id, i) => [this.nodes[id], this.nodes[nextPortIds[i]]])
-                    .forEach(pair => {  // [ resultPort, nextPort ]
-                        var result = pair[0],
-                            next = pair[1],
-                            hash = this.core.getAttribute(result, 'data');
-                        
-                        this.logger.info(`forwarding data (${hash}) from ${this.core.getPath(result)} ` +
-                            `to ${this.core.getPath(next)}`);
-                        this.core.setAttribute(next, 'data', hash);
-                        this.logger.info(`Setting ${jobId} data to ${hash}`);
-                    });
-
-                // For all the nextPortIds, decrement the corresponding operation's incoming counts
-                hasReadyOps = nextPortIds.map(id => this.getSiblingIdContaining(id))
-                    .reduce((l1, l2) => l1.concat(l2), [])
-
-                    // decrement the incoming counts for each operation id
-                    .map(opId => --this.incomingCounts[opId])
-                    .indexOf(0) > -1;
-
-                this.completedCount++;
                 this.logger.debug(`Operation "${name}" completed. ` + 
                     `${this.totalCount - this.completedCount} remaining.`);
                 if (hasReadyOps) {
@@ -442,9 +547,47 @@ define([
             });
     };
 
+    ExecutePipeline.prototype.updateJobCompletionRecords = function (opNode) {
+        var nextPortIds = this.getOperationOutputIds(opNode),
+            resultPorts,
+            counts;
+
+        // Transport the data from the outputs to any connected inputs
+        //   - Get all the connections from each outputId
+        //   - Get the corresponding dst outputs
+        //   - Use these new ids for checking 'hasReadyOps'
+
+        resultPorts = nextPortIds.map(id => this.inputPortsFor[id])  // dst -> src port
+            .reduce((l1, l2) => l1.concat(l2), []);
+
+        resultPorts
+            .map((id, i) => [this.nodes[id], this.nodes[nextPortIds[i]]])
+            .forEach(pair => {  // [ resultPort, nextPort ]
+                var result = pair[0],
+                    next = pair[1],
+                    hash = this.getAttribute(result, 'data');
+                
+                this.logger.info(`forwarding data (${hash}) from ${this.core.getPath(result)} ` +
+                    `to ${this.core.getPath(next)}`);
+                this.setAttribute(next, 'data', hash);
+                //this.logger.info(`Setting ${jobId} data to ${hash}`);
+            });
+
+        // For all the nextPortIds, decrement the corresponding operation's incoming counts
+        counts = nextPortIds.map(id => this.getSiblingIdContaining(id))
+            .reduce((l1, l2) => l1.concat(l2), [])
+
+            // decrement the incoming counts for each operation id
+            .map(opId => --this.incomingCounts[opId]);
+
+        this.completedCount++;
+        return counts;
+    };
+
     ExecutePipeline.prototype.getOperationOutputIds = function(node) {
         var jobId = this.getSiblingIdContaining(this.core.getPath(node));
 
+        // Map the job to it's output ports
         return this.outputsOf[jobId] || [];
     };
 

src/plugins/GenerateArchitecture/GenerateArchitecture.js

@@ -6,13 +6,15 @@ define([
     'SimpleNodes/Constants',
     'deepforge/layer-args',
     'deepforge/utils',
+    'deepforge/Constants',
     'underscore',
     'text!./metadata.json'
 ], function (
     PluginBase,
-    Constants,
+    SimpleNodeConstants,
     createLayerDict,
     utils,
+    Constants,
     _,
     metadata
 ) {
@@ -46,7 +48,7 @@ define([
         this.addCustomLayersToMeta();
         this.LayerDict = createLayerDict(this.core, this.META);
         this.uniqueId = 2;
-        this.varnames = {};
+        this.varnames = {net: true};
         return PluginBase.prototype.main.apply(this, arguments);
     };
 
@@ -65,7 +67,7 @@ define([
     };
 
     GenerateArchitecture.prototype.createOutputFiles = function (tree) {
-        var layers = tree[Constants.CHILDREN],
+        var layers = tree[SimpleNodeConstants.CHILDREN],
             result = {},
             code = '';
 
@@ -125,12 +127,57 @@ define([
     };
 
     GenerateArchitecture.prototype.createLayer = function (layer) {
-        var args = this.createArgString(layer);
-        return `nn.${layer.name}${args}`;
+        var args = this.createArgString(layer),
+            def = `nn.${layer.name}${args}`,
+            type = layer.base.base.name,
+            addedIds,
+            node,
+            name,
+            children,
+            id;
+
+        // Check if it is a container and has the 'addLayers' set
+        // If so, it should sort them by their registry 'index' and add
+        // each nested architecture's code to the given container
+        if (type === 'Container') {
+            // Get the members of the 'addLayers' set
+            addedIds = {};
+            id = layer[SimpleNodeConstants.NODE_PATH];
+            node = this._nodeCache[id];
+            this.core.getMemberPaths(node, Constants.CONTAINED_LAYER_SET)
+                .forEach(id => addedIds[id] = true);
+
+            // Get the (sorted) children
+            children = layer[SimpleNodeConstants.CHILDREN]
+                .map(child => {  // get (child, index) tuples
+                    var index;
+
+                    id = child[SimpleNodeConstants.NODE_PATH];
+                    index = this.core.getMemberRegistry(node, Constants.CONTAINED_LAYER_SET, id, Constants.CONTAINED_LAYER_INDEX);
+                    return [child, index];
+                })
+                .filter(pair => pair[1] !== undefined)  // remove non-members
+                .sort((a, b) => a[1] < b[1] ? -1 : 1)  // sort by 'index'
+                .map(pair => pair[0]);
+
+
+            var addedLayerDefs = '',
+                firstLayer;
+            for (var i = 0; i < children.length; i++) {
+                id = children[i][SimpleNodeConstants.NODE_PATH];
+                // Get the children!
+                firstLayer = children[i][SimpleNodeConstants.CHILDREN][0];
+                name = this.getVarName(utils.abbr(layer.name + '_' + i));
+                addedLayerDefs += this.createSequential(firstLayer, name).code;
+                def += `:add(${name})`;
+            }
+            this.hoist(addedLayerDefs);
+        }
+        return def;
     };
 
     GenerateArchitecture.prototype.createSequential = function (layer, name) {
-        var next = layer[Constants.NEXT][0],
+        var next = layer[SimpleNodeConstants.NEXT][0],
             args,
             snippet,
             snippets,
@@ -142,7 +189,7 @@ define([
 
         while (layer) {
             // if there is only one successor, just add the given layer
-            if (layer[Constants.PREV].length > 1) {  // sequential layers are over
+            if (layer[SimpleNodeConstants.PREV].length > 1) {  // sequential layers are over
                 next = layer;  // the given layer will be added by the caller
                 break;
             } else {  // add the given layer
@@ -151,11 +198,11 @@ define([
 
             }
 
-            while (layer && layer[Constants.NEXT].length > 1) {  // concat/parallel
+            while (layer && layer[SimpleNodeConstants.NEXT].length > 1) {  // concat/parallel
                 // if there is a fork, recurse and add a concat layer
 
-                this.logger.debug(`detected fork of size ${layer[Constants.NEXT].length}`);
-                snippets = layer[Constants.NEXT].map(nlayer =>
+                this.logger.debug(`detected fork of size ${layer[SimpleNodeConstants.NEXT].length}`);
+                snippets = layer[SimpleNodeConstants.NEXT].map(nlayer =>
                     this.createSequential(nlayer, this.getVarName('net')));
                 code += '\n' + snippets.map(snippet => snippet.code).join('\n');
 
@@ -183,7 +230,7 @@ define([
                                 `concat_${layer[INDEX]}:add(${snippet.name})`)
                             .join('\n') + `\n\n${name}:add(concat_${layer[INDEX]})`;
                         
-                        next = layer[Constants.NEXT][0];
+                        next = layer[SimpleNodeConstants.NEXT][0];
                     } else {
                         next = null;  // no next layers
                     }
@@ -203,7 +250,7 @@ define([
             }
 
             layer = next;
-            next = layer && layer[Constants.NEXT][0];
+            next = layer && layer[SimpleNodeConstants.NEXT][0];
         }
 
         return {
@@ -218,14 +265,14 @@ define([
         var content = layer[arg];
 
         if (typeof content === 'object') {  // layer as arg
-            if (content[Constants.CHILDREN].length) {
+            if (content[SimpleNodeConstants.CHILDREN].length) {
                 // Generate the code for the children of layer[arg]
                 var name = this.getVarName(utils.abbr(arg)),
                     layers;
 
                 this.logger.debug(`Adding layer arg for ${arg} (${layer.name})`);
                 try {
-                    layers = this.genRawArchCode(layer[arg][Constants.CHILDREN], name);
+                    layers = this.genRawArchCode(layer[arg][SimpleNodeConstants.CHILDREN], name);
                 } catch (e) {
                     this.logger.error(`Layer arg creation failed: ${e}`);
                     return null;
@@ -244,20 +291,30 @@ define([
     GenerateArchitecture.prototype.createArgString = function (layer) {
         var setters = this.LayerDict[layer.name].setters,
             setterNames = Object.keys(this.LayerDict[layer.name].setters),
-            base = layer[Constants.BASE],
+            base = layer[SimpleNodeConstants.BASE],
             desc,
             fn,
-            layerCode;
+            layerCode,
+            args,
+            i;
 
         this.logger.debug(`Creating arg string for ${layer.name}`);
-        layerCode = '(' + this.LayerDict[layer.name].args
-            .map(arg => this.getValue(arg.name, layer))
-            .filter(GenerateArchitecture.isSet)
+        args = this.LayerDict[layer.name].args
+            .map(arg => this.getValue(arg.name, layer));
+
+        for (i = args.length; i--;) {
+            if (GenerateArchitecture.isSet(args[i])) {
+                break;
+            }
+            args.pop();
+        }
+
+        layerCode = '(' + args.map(arg => GenerateArchitecture.isSet(arg) ? arg : 'nil')
             .join(', ') + ')';
 
         // Add any setters
         // For each setter, check if it has been changed (and needs to be set)
-        for (var i = setterNames.length; i--;) {
+        for (i = setterNames.length; i--;) {
             desc = setters[setterNames[i]];
             if (desc.setterType === 'const') {
                 // if the value is not the default, add the given fn

src/plugins/GenerateExecFile/GenerateExecFile.js

@@ -1,27 +1,40 @@
-/*globals define, _*/
+/*globals define */
 /*jshint node:true, browser:true*/
 
-/**
- * Generated by PluginGenerator 1.7.0 from webgme on Sat Jun 04 2016 18:01:54 GMT-0500 (CDT).
- * A plugin that inherits from the PluginBase. To see source code documentation about available
- * properties and methods visit %host%/docs/source/PluginBase.html.
- */
-
 define([
     'text!./metadata.json',
+    'text!./deepforge.ejs',
+    'text!./toboolean.lua',
     'plugin/PluginBase',
     'deepforge/plugin/PtrCodeGen',
+    'deepforge/Constants',
+    'underscore',
     'q'
 ], function (
     pluginMetadata,
+    deepForgeTxt,
+    TOBOOLEAN,
     PluginBase,
     PtrCodeGen,
+    CONSTANTS,
+    _,
     Q
 ) {
     'use strict';
 
     pluginMetadata = JSON.parse(pluginMetadata);
-    var HEADER_LENGTH = 60;
+    var HEADER_LENGTH = 60,
+        SKIP_ATTRS = {
+            lineOffset: true,
+            code: true
+        },
+        RESERVED = /^(and|break|do|else|elseifend|false|for|function|if|in|local|nil|not|orrepeat|return|then|true|until|while|print)$/,
+        INDENT = '   ',
+        INIT_CLASSES_FN = '__initClasses',
+        INIT_LAYERS_FN = '__initLayers',
+        DEEPFORGE_CODE = _.template(deepForgeTxt)({
+            initCode: `${INIT_CLASSES_FN}()\n${INDENT}${INIT_LAYERS_FN}()`
+        });
 
     /**
      * Initializes a new instance of GenerateExecFile.
@@ -33,21 +46,7 @@ define([
     var GenerateExecFile = function () {
         // Call base class' constructor.
         PluginBase.call(this);
-        this.pluginMetadata = pluginMetadata;
-
-        this._srcIdFor = {};  // input path -> output data node path
-
-        this._nameFor = {};  // input path -> opname
-        this._dataNameFor = {};  
-        this._opNames = {};
-
-        // topo sort stuff
-        this._nextOps = {};
-        this._incomingCnts = {};
-
-        this._operations = {};
-        this.activeNodeId = null;
-        this.activeNodeDepth = null;
+        this.initRecords();
     };
 
     /**
@@ -61,6 +60,35 @@ define([
     GenerateExecFile.prototype = Object.create(PluginBase.prototype);
     GenerateExecFile.prototype.constructor = GenerateExecFile;
 
+    GenerateExecFile.prototype.initRecords = function() {
+        this.pluginMetadata = pluginMetadata;
+
+        this._srcIdFor = {};  // input path -> output data node path
+
+        this._nameFor = {};  // input path -> opname
+        this._outputNames = {};
+        this._baseNameFor = {};
+        this._dataNameFor = {};  
+        this._instanceNames = {};
+        this._opBaseNames = {};
+        this._fnNameFor = {};
+        this._functions = {};  // function definitions for the operations
+
+        // topo sort stuff
+        this._nextOps = {};
+        this._incomingCnts = {};
+
+        this._operations = {};
+        this.activeNodeId = null;
+        this.activeNodeDepth = null;
+
+        this.isInputOp = {};
+        this._portCache = {};
+        this.inputNode = {};
+        this.outputDataToOpId = {};
+        this.isOutputOp = {};
+    };
+
     /**
      * Main function for the plugin to execute. This will perform the execution.
      * Notes:
@@ -71,6 +99,10 @@ define([
      * @param {function(string, plugin.PluginResult)} callback - the result callback
      */
     GenerateExecFile.prototype.main = function (callback) {
+        var name = this.core.getAttribute(this.activeNode, 'name');
+
+        this.initRecords();
+
         // Get all the children and call generate exec file
         this.activeNodeId = this.core.getPath(this.activeNode);
         this.activeNodeDepth = this.activeNodeId.split('/').length + 1;
@@ -81,7 +113,7 @@ define([
 
         return this.core.loadChildren(this.activeNode)
             .then(nodes => this.createExecFile(nodes))
-            .then(code => this.blobClient.putFile('init.lua', code))
+            .then(code => this.blobClient.putFile(`${name}.lua`, code))
             .then(hash => {
                 this.result.addArtifact(hash);
                 this.result.setSuccess(true);
@@ -91,6 +123,61 @@ define([
     };
 
     GenerateExecFile.prototype.createExecFile = function (children) {
+        return this.createCodeSections(children)
+            .then(sections => {
+                var classes,
+                    initClassFn,
+                    initLayerFn,
+                    code = [];
+
+                // concat all the sections into a single file
+
+                // wrap the class/layer initialization in a fn
+                // Add the classes ordered wrt their deps
+                classes = Object.keys(sections.classes)
+                    .sort((a, b) => {
+                        // if a depends on b, switch them (return 1)
+                        if (sections.classDependencies[a].includes(b)) {
+                            return 1;
+                        }
+                        return -1;
+                    })
+                    // Create fns from the classes
+                    .map(name => [
+                        `local function init${name}()`,
+                        indent(sections.classes[name]),
+                        'end',
+                        `init${name}()`
+                    ].join('\n'));
+
+                initClassFn = [
+                    `local function ${INIT_CLASSES_FN}()`,
+                    indent(classes.join('\n\n')),
+                    'end'
+                ].join('\n');
+
+                code = code.concat(initClassFn);
+
+                // wrap the layers in a function
+                initLayerFn = [
+                    `local function ${INIT_LAYERS_FN}()`,
+                    indent(_.values(sections.layers).join('\n\n')),
+                    'end'
+                ].join('\n');
+                code = code.concat(initLayerFn);
+                code = code.concat(_.values(sections.operations));
+
+                code = code.concat(_.values(sections.pipelines));
+
+                code.push(DEEPFORGE_CODE);
+                code.push('deepforge.initialize()');
+                code.push(sections.main);
+
+                return code.join('\n\n');
+            });
+    };
+
+    GenerateExecFile.prototype.createCodeSections = function (children) {
         // Convert opNodes' jobs to the nested operations
         var opNodes,
             nodes;
@@ -100,18 +187,37 @@ define([
                 nodes = _nodes;
                 opNodes = nodes
                     .filter(node => this.isMetaTypeOf(node, this.META.Operation));
-                return Q.all(nodes.map(node => this.registerNameAndData(node)));
+
+                // Sort the connections to come first
+                nodes
+                    .map(node => [
+                        node,
+                        this.isMetaTypeOf(node, this.META.Transporter) ? -1 : 1
+                    ])
+                    .sort((a, b) => a[1] < b[1] ? -1 : 1);
+
+                return Q.all(nodes.map(node => this.registerNode(node)));
             })
-            .then(() => Q.all(opNodes.map(node => this.createOperation(node))))
+            .then(() => Q.all(opNodes
+                .filter(n => {
+                    var id = this.core.getPath(n);
+                    return !this.isInputOp[id];
+                })
+                .map(node => this.createOperation(node)))
+            )
             .then(operations => {
-                var nextIds = opNodes.map(n => this.core.getPath(n))
-                        .filter(id => !this._incomingCnts[id]);
+                var opDict = {},
+                    firstOpIds;
 
-                operations.forEach(op => this._operations[op.id] = op);
+                firstOpIds = opNodes.map(n => this.core.getPath(n))
+                    .filter(id => !this._incomingCnts[id]);
 
-                // Toposort and concat!
-                return this.combineOpNodes(nextIds);
+                operations.forEach(op => opDict[op.id] = op);
+
+                // Toposort!
+                return this.sortOperations(opDict, firstOpIds);
             })
+            .then(operations => this.generateCodeSections(operations))
             .fail(err => this.logger.error(err));
     };
 
@@ -129,18 +235,20 @@ define([
         );
     };
 
-    GenerateExecFile.prototype.combineOpNodes = function (opIds) {
+    GenerateExecFile.prototype.sortOperations = function (operationDict, opIds) {
         var nextIds = [],
+            sorted = opIds,
             dstIds,
-            code,
             id;
 
-        // Combine all nodes with incoming cnts of 0
-        code = opIds.map(id => this._operations[id].code).join('\n');
+        if (!opIds.length) {
+            return [];
+        }
 
         // Decrement all next ops
         dstIds = opIds.map(id => this._nextOps[id])
             .reduce((l1, l2) => l1.concat(l2), []);
+
         for (var i = dstIds.length; i--;) {
             id = dstIds[i];
             if (--this._incomingCnts[id] === 0) {
@@ -149,64 +257,447 @@ define([
         }
 
         // append
+        return sorted
+            .map(id => operationDict[id])
+            .filter(op => !!op)
+            .concat(this.sortOperations(operationDict, nextIds));
+    };
+
+    GenerateExecFile.prototype.generateCodeSections = function(sortedOps) {
+        // Create the code sections:
+        //  - operation definitions
+        //  - pipeline definition
+        //  - main
+        var code = {},
+            baseIds = [],
+            outputOps = [],
+            mainOps = [];
+
+        // Define the operation functions...
+        code.operations = {};
+        for (var i = 0; i < sortedOps.length; i++) {
+            if (this.isInputOp[sortedOps[i].id]) {
+                continue;
+            }
+            if (!this.isOutputOp[sortedOps[i].id]) {
+                if (!baseIds.includes(sortedOps[i].baseId)) {  // new definition
+                    code.operations[sortedOps[i].basename] = this.defineOperationFn(sortedOps[i]);
+                    baseIds.push(sortedOps[i].baseId);
+                }
+                mainOps.push(sortedOps[i]);
+            } else {
+                outputOps.push(sortedOps[i]);
+            }
+        }
+
+        // Define the pipeline function
+        code.pipelines = this.definePipelineFn(mainOps, outputOps);
+
+        // Define the main body
+        this.addCodeMain(code);
+
+        // Add custom class definitions
+        this.addCustomClasses(code);
+
+        // Add custom layer definitions
+        this.addCustomLayers(code);
+
+        return code;
+    };
+
+    var indent = function(text) {
+        return text.replace(/^/mg, INDENT);
+    };
+
+    GenerateExecFile.prototype.defineOperationFn = function(operation) {
+        var lines = [],
+            args = operation.inputNames || [];
+
+        // Create the function definition
+        args.unshift('attributes');
+        // Add the refs to the end
+        args = args.concat(operation.refNames);
+
+        args = args.join(', ');
+
+        lines.push(`local function ${operation.basename}(${args})`);
+        lines.push(indent(operation.code));
+        lines.push('end');
+
+        return lines.join('\n');
+    };
+
+    GenerateExecFile.prototype.definePipelineFn = function(sortedOps, outputOps) {
+        var inputArgs = Object.keys(this.isInputOp).map(id => this._nameFor[id]),
+            name = this.core.getAttribute(this.activeNode, 'name'),
+            safename = getUniqueName(name, this._opBaseNames),
+            results = [],
+            result = {},
+            returnStat,
+            fnbody;
+
+        // Call each function in order, with the respective attributes, etc
+        fnbody = sortedOps.map(op => this.getOpInvocation(op)).join('\n');
+
+        // Create the return statement
+        results.push('\n\nresults = {}');
+        outputOps.map(op => this.getOutputPair(op))
+            .forEach(pair => results.push(`results['${pair[0]}'] = ${pair[1]}`));
+        results.push('return results');
+        returnStat = results.join('\n');
+
+        // Merge the fnbody, return statement and the function def
+        result[safename] = `local function ${safename} (${inputArgs.join(', ')})\n` +
+            `${indent(fnbody + returnStat)}\nend`;
+
+        return result;
+    };
+
+    GenerateExecFile.prototype.getOutputPair = function(operation) {
+        var input = operation.inputValues[0].slice(),
+            value;
+
+        // Get the src operation name and data value name
+        input[0] += '_results';
+        value = input.join('.');
+        return [this._nameFor[operation.id], value];
+    };
+
+    GenerateExecFile.prototype.addCodeMain = function(sections) {
+        var pipelineName = Object.keys(sections.pipelines)[0],
+            hasBool = false,
+            code = [],
+            loadNodes = {},
+            args;
+
+        args = Object.keys(this.isInputOp).map((id, index) => {
+            var node = this.inputNode[id],
+                base = this.core.getBase(node),
+                type = this.core.getAttribute(base, 'name'),
+                arg = `arg[${index+1}]`;
+
+            if (type === 'boolean') {
+                hasBool = true;
+                return `toboolean(${arg})`;
+            } else if (type === 'number') {
+                return `tonumber(${arg})`;
+            } else if (type === 'string') {
+                return arg;
+            } else {
+                loadNodes[id] = node;
+                return `load['${this._nameFor[id]}'](${arg})`;
+            }
+        });
+
+        // Handle the arg types
+        if (hasBool) {
+            // add toboolean def
+            code.push(TOBOOLEAN);
+        }
+        // Define the 'saveOutputs' method
+        var saveNodes = {};
+        Object.keys(this.outputDataToOpId).forEach(dataId => {
+            var opId = this.outputDataToOpId[dataId];
+            // The key is used for the output name resolution. The
+            // value is used for the serialization fn look-up. So,
+            // the key is the output operation id and the value is
+            // the data port connected to the output operation
+            saveNodes[opId] = this._portCache[this._srcIdFor[dataId]];
+        });
+
+        // Add dictionary of serializers/deserializers
+        code.push(
+            this.createTorchFnDict('load', loadNodes, 'deserialize', 'path'),
+            this.createTorchFnDict('save', saveNodes, 'serialize', 'path, data')
+        );
+
+        // Add a saveOutputs method for convenience
+        code.push([
+            'local function saveOutputs(data)',
+            indent(Object.keys(this.isOutputOp).map(id => {
+                var name = this._nameFor[id];
+                return `print('saving ${name}...')\nsave['${name}']('${name}', data['${name}'])`;
+            }).join('\n')),
+            'end'
+        ].join('\n'));
+
+        code.push(
+            `local outputs = ${pipelineName}(${args.join(', ')})\n` +
+            'saveOutputs(outputs)',
+            'return outputs'
+        );
+
+        sections.main = code.join('\n\n');
+    };
+
+    GenerateExecFile.prototype.createTorchFnDict = function(name, nodeDict, attr, args) {
         return [
-            code,
-            nextIds.length ? this.combineOpNodes(nextIds) : ''
+            `local ${name} = {}`,
+            Object.keys(nodeDict).map(id => {
+                var node = nodeDict[id];
+                return [
+                    `${name}['${this._nameFor[id]}'] = function(${args})`,
+                    indent(this.core.getAttribute(node, attr)),
+                    'end'
+                ].join('\n');
+            }).join('\n')
         ].join('\n');
     };
 
-    GenerateExecFile.prototype.registerNameAndData = function (node) {
+    GenerateExecFile.prototype.addCustomClasses = function(sections) {
+        var metaDict = this.core.getAllMetaNodes(this.rootNode),
+            isClass,
+            metanodes,
+            classNodes,
+            inheritanceLvl = {};
+
+        this.logger.info('Creating custom layer file...');
+        metanodes = Object.keys(metaDict).map(id => metaDict[id]);
+        isClass = this.getTypeDictFor('Complex', metanodes);
+
+        // Store the dependencies for each class
+        sections.classDependencies = {};
+
+        classNodes = metanodes.filter(node => {
+            var base = this.core.getBase(node),
+                baseId = this.core.getPath(base),
+                deps = [],
+                name,
+                count = 1;
+
+            // Count the sets back to a class node
+            while (base) {
+                deps.push(this.core.getAttribute(base, 'name'));
+                if (isClass[baseId]) {
+                    inheritanceLvl[this.core.getPath(node)] = count;
+                    name = this.core.getAttribute(node, 'name');
+                    sections.classDependencies[name] = deps;
+                    return true;
+                }
+                base = this.core.getBase(base);
+                baseId = this.core.getPath(base);
+                count++;
+            }
+
+            return false;
+        });
+
+        // Get the code definitions for each
+        sections.classes = {};
+        classNodes
+            .sort((a, b) => {
+                var aId = this.core.getPath(a),
+                    bId = this.core.getPath(b);
+
+                return inheritanceLvl[aId] > inheritanceLvl[bId];
+            })
+            .forEach(node => {
+                var name = this.core.getAttribute(node, 'name'),
+                    code = this.core.getAttribute(node, 'code');
+
+                sections.classes[name] = code;
+            });
+    };
+
+    GenerateExecFile.prototype.addCustomLayers = function(sections) {
+        var metaDict = this.core.getAllMetaNodes(this.rootNode),
+            isCustomLayer,
+            metanodes,
+            customLayers;
+
+        this.logger.info('Creating custom layer file...');
+        metanodes = Object.keys(metaDict).map(id => metaDict[id]);
+        isCustomLayer = this.getTypeDictFor('CustomLayer', metanodes);
+
+        customLayers = metanodes.filter(node =>
+            this.core.getMixinPaths(node).some(id => isCustomLayer[id]));
+
+        // Get the code definitions for each
+        sections.layers = {};
+        customLayers
+            .map(layer => [
+                this.core.getAttribute(layer, 'name'),
+                this.core.getAttribute(layer, 'code')
+            ])
+            .forEach(pair => sections.layers[pair[0]] = pair[1]);
+    };
+
+
+    GenerateExecFile.prototype.getTypeDictFor = function (name, metanodes) {
+        var isType = {};
+        // Get all the custom layers
+        for (var i = metanodes.length; i--;) {
+            if (this.core.getAttribute(metanodes[i], 'name') === name) {
+                isType[this.core.getPath(metanodes[i])] = true;
+            }
+        }
+        return isType;
+    };
+
+    var toAttrString = function(attr) {
+        if (/^\d+\.?\d*$/.test(attr) || /^(true|false|nil)$/.test(attr)) {
+            return attr;
+        }
+        return `"${attr}"`;
+    };
+
+    GenerateExecFile.prototype.getOpInvocation = function(op) {
+        var lines = [],
+            attrs,
+            refInits = [],
+            args;
+
+        attrs = '{' +
+            Object.keys(op.attributes).map(key => `${key}=${toAttrString(op.attributes[key])}`)
+            .join(',') +
+        '}';
+
+        lines.push(`local ${op.name}_attrs = ${attrs}`);
+        args = (op.inputValues || [])
+            .map(val => val instanceof Array ? `${val[0]}_results.${val[1]}` : val);
+
+        args.unshift(op.name + '_attrs');
+            
+        // Create the ref init functions
+        refInits = op.refs.map((code, index) => {
+            return [
+                `local function create_${op.refNames[index]}()`,
+                indent(code),
+                'end'
+            ].join('\n');
+        });
+        lines = lines.concat(refInits);
+        args = args.concat(op.refNames.map(name => `create_${name}()`));
+        args = args.join(', ');
+        lines.push(`local ${op.name}_results = ${op.basename}(${args})`);
+
+        return lines.join('\n');
+    };
+
+    GenerateExecFile.prototype.getOutputName = function(node) {
+        var basename = this.core.getAttribute(node, 'saveName');
+
+        return getUniqueName(basename, this._outputNames, true);
+    };
+
+    GenerateExecFile.prototype.getVariableName = function (/*node*/) {
+        var c = Object.keys(this.isInputOp).length;
+
+        if (c !== 1) {
+            return `input${c}`;
+        }
+
+        return 'input';
+    };
+
+    GenerateExecFile.prototype.registerNode = function (node) {
+        if (this.isMetaTypeOf(node, this.META.Operation)) {
+            return this.registerOperation(node);
+        } else if (this.isMetaTypeOf(node, this.META.Transporter)) {
+            return this.registerTransporter(node);
+        }
+    };
+
+    var getUniqueName = function(namebase, takenDict, unsafeAllowed) {
+        var name,
+            i = 2,
+            isUnsafe = function(name) {
+                return !unsafeAllowed && RESERVED.test(name);
+            };
+
+        if (!unsafeAllowed) {
+            namebase = namebase.replace(/[^A-Za-z\d]/g, '_');
+        }
+        name = namebase;
+        // Get a unique operation name
+        while (takenDict[name] || isUnsafe(name)) {
+            name = namebase + '_' + i;
+            i++;
+        }
+        takenDict[name] = true;
+
+        return name;
+    };
+
+    GenerateExecFile.prototype.registerOperation = function (node) {
         var name = this.core.getAttribute(node, 'name'),
             id = this.core.getPath(node),
-            basename = name,
-            i = 2;
+            base = this.core.getBase(node),
+            baseId = this.core.getPath(base),
+            baseName = this.core.getAttribute(base, 'name');
 
-        if (this.isMetaTypeOf(node, this.META.Operation)) {
-
-            // Get a unique operation name
-            while (this._opNames[name]) {
-                name = basename + '_' + i;
-                i++;
-            }
-
-            // register the unique name
-            this._opNames[name] = true;
-            this._nameFor[id] = name;
-
-            // For operations, register all output data node names by path
-            return this.core.loadChildren(node)
-                .then(cntrs => {
-                    var cntr = cntrs.find(n => this.isMetaTypeOf(n, this.META.Outputs));
-                    return this.core.loadChildren(cntr);
-                })
-                .then(outputs => {
-                    outputs.forEach(output => {
-                        var dataId = this.core.getPath(output);
-
-                        name = this.core.getAttribute(output, 'name');
-                        this._dataNameFor[dataId] = name;
-                    });
-                });
-
-        // For each input data node, register the associated output id
-        } else if (this.isMetaTypeOf(node, this.META.Transporter)) {
-            var outputData = this.core.getPointerPath(node, 'src'),
-                inputData = this.core.getPointerPath(node, 'dst'),
-                srcOpId = this.getOpIdFor(outputData),
-                dstOpId = this.getOpIdFor(inputData);
-
-            this._srcIdFor[inputData] = outputData;
-
-            // Store the next operation ids for the op id
-            if (!this._nextOps[srcOpId]) {
-                this._nextOps[srcOpId] = [];
-            }
-            this._nextOps[srcOpId].push(dstOpId);
-
-            // Increment the incoming counts for each dst op
-            this._incomingCnts[dstOpId] = this._incomingCnts[dstOpId] || 0;
-            this._incomingCnts[dstOpId]++;
+        // If it is an Input/Output operation, assign it a variable name
+        if (baseName === CONSTANTS.OP.INPUT) {
+            this.isInputOp[id] = node;
+            name = this.getVariableName(node);
+        } else if (baseName === CONSTANTS.OP.OUTPUT) {
+            this.isOutputOp[id] = node;
+            name = this.getOutputName(node);
+        } else {
+            // get a unique operation instance name
+            name = getUniqueName(name, this._instanceNames);
         }
+
+        this._nameFor[id] = name;
+
+        // get a unique operation base name
+        if (!this._fnNameFor[baseId]) {
+            name = this.core.getAttribute(base, 'name');
+            name = getUniqueName(name, this._opBaseNames);
+            this._fnNameFor[baseId] = name;
+        }
+
+        // For operations, register all output data node names by path
+        return this.core.loadChildren(node)
+            .then(cntrs => {
+                var outputs = cntrs.find(n => this.isMetaTypeOf(n, this.META.Outputs)),
+                    inputs = cntrs.find(n => this.isMetaTypeOf(n, this.META.Inputs));
+
+                return Q.all([inputs, outputs].map(cntr => this.core.loadChildren(cntr)));
+            })
+            .then(data => {
+                var inputs = data[0],
+                    outputs = data[1];
+
+                // Get the input type
+                outputs.forEach(output => {
+                    var dataId = this.core.getPath(output);
+
+                    name = this.core.getAttribute(output, 'name');
+                    this._dataNameFor[dataId] = name;
+
+                    this._portCache[dataId] = output;
+                });
+                inputs.forEach(input => 
+                    this._portCache[this.core.getPath(input)] = input
+                );
+
+                // Extra recording for input/output nodes in the pipeline
+                if (this.isInputOp[id]) {
+                    this.inputNode[id] = outputs[0];
+                } else if (this.isOutputOp[id]) {
+                    this.outputDataToOpId[this.core.getPath(inputs[0])] = id;
+                }
+            });
+    };
+
+    GenerateExecFile.prototype.registerTransporter = function (node) {
+        var outputData = this.core.getPointerPath(node, 'src'),
+            inputData = this.core.getPointerPath(node, 'dst'),
+            srcOpId = this.getOpIdFor(outputData),
+            dstOpId = this.getOpIdFor(inputData);
+
+        this._srcIdFor[inputData] = outputData;
+
+        // Store the next operation ids for the op id
+        if (!this._nextOps[srcOpId]) {
+            this._nextOps[srcOpId] = [];
+        }
+        this._nextOps[srcOpId].push(dstOpId);
+
+        // Increment the incoming counts for each dst op
+        this._incomingCnts[dstOpId] = this._incomingCnts[dstOpId] || 0;
+        this._incomingCnts[dstOpId]++;
     };
 
     GenerateExecFile.prototype.getOpIdFor = function (dataId) {
@@ -226,16 +717,21 @@ define([
     //     - replace the `return <thing>` w/ `<ref-name> = <thing>`
     GenerateExecFile.prototype.createOperation = function (node) {
         var id = this.core.getPath(node),
+            baseId = this.core.getPath(this.core.getBase(node)),
+            attrNames = this.core.getValidAttributeNames(node),
             operation = {};
 
         operation.name = this._nameFor[id];
+        operation.basename = this._fnNameFor[baseId];
+        operation.baseId = baseId;
         operation.id = id;
         operation.code = this.core.getAttribute(node, 'code');
-
-        // Update the 'code' attribute
-        // Change the last return statement to assign the results to a table
-        operation.code = this.assignResultToVar(operation.code,
-            `${operation.name}_results`);
+        operation.attributes = {};
+        for (var i = attrNames.length; i--;) {
+            if (!SKIP_ATTRS[attrNames[i]]) {
+                operation.attributes[attrNames[i]] = this.core.getAttribute(node, attrNames[i]);
+            }
+        }
 
         // Get all the input names (and sources)
         return this.core.loadChildren(node)
@@ -245,7 +741,7 @@ define([
                 inputs = containers
                     .find(cntr => this.isMetaTypeOf(cntr, this.META.Inputs));
 
-                this.logger.info(`${name} has ${containers.length} cntrs`);
+                this.logger.info(`${operation.name} has ${containers.length} cntrs`);
                 return this.core.loadChildren(inputs);
             })
             .then(data => {
@@ -254,13 +750,18 @@ define([
                     ids = data.map(d => this.core.getPath(d)),
                     srcIds = ids.map(id => this._srcIdFor[id]);
 
-                operation.inputs = inputNames.map((name, i) => {
+                operation.inputNames = inputNames || [];
+                operation.inputValues = inputNames.map((name, i) => {
                     var id = srcIds[i],
                         srcDataName = this._dataNameFor[id],
                         srcOpId = this.getOpIdFor(id),
                         srcOpName = this._nameFor[srcOpId];
 
-                    return `local ${name} = ${srcOpName}_results.${srcDataName}`;
+                    if (this.isInputOp[srcOpId]) {
+                        return this._nameFor[srcOpId];
+                    } else {
+                        return [srcOpName, srcDataName];
+                    }
                 });
 
                 return operation;
@@ -269,8 +770,12 @@ define([
             .then(operation => {
 
                 // For each reference, run the plugin and retrieve the generated code
-                operation.refNames = this.core.getPointerNames(node)
-                    .filter(name => name !== 'base');
+                operation.refNames = [];
+
+                if (!this.isInputOp[operation.id]) {
+                    operation.refNames = this.core.getPointerNames(node)
+                        .filter(name => name !== 'base');
+                }
 
                 var refs = operation.refNames
                     .map(ref => [ref, this.core.getPointerPath(node, ref)]);
@@ -281,16 +786,13 @@ define([
             })
             .then(codeFiles => {
                 operation.refs = codeFiles;
-                this.genOperationCode(operation);
                 return operation;
             });
     };
 
     GenerateExecFile.prototype.genPtrSnippet = function (ptrName, pId) {
         return this.getPtrCodeHash(pId)
-            .then(hash => this.blobClient.getObjectAsString(hash))
-            .then(code => this.createHeader(`creating ${ptrName}`, 40) + '\n' +
-                this.assignResultToVar(code, ptrName));
+            .then(hash => this.blobClient.getObjectAsString(hash));
     };
 
     GenerateExecFile.prototype.createHeader = function (title, length) {
@@ -313,29 +815,37 @@ define([
 
     GenerateExecFile.prototype.genOperationCode = function (operation) {
         var header = this.createHeader(`"${operation.name}" Operation`),
-            codeParts = [];
+            codeParts = [],
+            body = [];
 
         codeParts.push(header);
+        codeParts.push(`local ${operation.name}_results`);
+        codeParts.push('do');
 
         if (operation.inputs.length) {
-            codeParts.push(operation.inputs.join('\n'));
+            body.push(operation.inputs.join('\n'));
         }
 
         if (operation.refs.length) {
-            codeParts.push(operation.refs.join('\n'));
+            body.push(operation.refs.join('\n'));
         }
 
-        codeParts.push(operation.code);
+        body.push(operation.code);
+
+        codeParts.push(indent(body.join('\n')));
+        codeParts.push('end');
         codeParts.push('');
+
         operation.code = codeParts.join('\n');
         return operation;
     };
 
     GenerateExecFile.prototype.assignResultToVar = function (code, name) {
         var i = code.lastIndexOf('return');
+
         return code.substring(0, i) +
             code.substring(i)
-                .replace('return', `local ${name} = `);
+                .replace('return', `${name} = `);
     };
 
     _.extend(GenerateExecFile.prototype, PtrCodeGen.prototype);

src/plugins/GenerateExecFile/deepforge.ejs

@@ -0,0 +1,47 @@
+-- Instantiate the deepforge object
+deepforge = {}
+
+function deepforge.initialize()
+    require 'nn'
+    require 'rnn'
+    <%= initCode %>
+end
+
+-- Graph support
+torch.class('deepforge.Graph')
+
+function deepforge.Graph:__init(name)
+    -- nop
+end
+
+torch.class('deepforge._Line')
+
+function deepforge._Line:__init(graphId, name, opts)
+   -- nop
+end
+
+function deepforge._Line:add(x, y)
+   -- nop
+end
+
+function deepforge.Graph:line(name, opts)
+    return deepforge._Line(self.id, name, opts)
+end
+
+-- Image support
+function deepforge.image(name, tensor)
+   -- nop
+end
+
+torch.class('deepforge.Image')
+function deepforge.Image:__init(name, tensor)
+   -- nop
+end
+
+function deepforge.Image:update(tensor)
+   -- nop
+end
+
+function deepforge.Image:title(name)
+   -- nop
+end

src/plugins/GenerateExecFile/toboolean.lua

@@ -0,0 +1,7 @@
+local function toboolean(str)
+    if str == 'true' then
+        return true
+    elseif str == 'false' then
+        return false
+    end
+end

src/plugins/ImportArtifact/ImportArtifact.js

@@ -86,6 +86,7 @@ define([
                 });
 
                 this.core.setAttribute(dataNode, 'data', hash);
+                this.core.setAttribute(dataNode, 'createdAt', Date.now());
                 baseName = this.core.getAttribute(baseType, 'name');
 
                 var getName;

src/plugins/ImportTorch/nn.js

@@ -3,10 +3,12 @@
 define([
     'deepforge/layer-args',
     'common/util/assert',
+    'deepforge/Constants',
     'deepforge/lua'
 ], function(
     createLayerDict,
     assert,
+    Constants,
     lua
 ) {
     'use strict';
@@ -41,6 +43,20 @@ define([
             return '{' + strings.join(', ') + '}';
         };
 
+        var getAttributeString = function(value, layerType) {
+            if (value instanceof lua.types.LuaTable) {
+                if (value.get('_node')) {
+                    throw Error(`Detected unsupported varargs (composed of layers) for ${layerType}`);
+                }
+                return stringify(value);
+            } else if ((typeof value) === 'object') {
+                // special lua.js object
+                value = value.valueOf();
+            }
+
+            return value;
+        };
+
         var allConnectedTo = function(current) {
             var connectedIds = {},
                 node,
@@ -79,6 +95,7 @@ define([
                 connsFrom[id] = [];
             }
             connsFrom[id].push(conn, dst);
+            return conn;
         };
 
         // nn drawing library
@@ -104,7 +121,8 @@ define([
                 cntr,
                 layer,
                 cntrName,
-                value;
+                value,
+                i;
 
             if (this._cachedNode) {
                 // only generate a single node for each layer
@@ -117,7 +135,17 @@ define([
                 parent: parent
             });
 
-            for (var i = this._attrs.length; i--;) {
+            // merge all the last arguments into a single one (ie, assume the last
+            // attribute is varargs
+            if (this._attrs.length < this._values.length) {
+                i = this._attrs.length;
+                value = this._values.splice(i-1)
+                    .map(val => getAttributeString(val, this._base)).join(', ');
+                this._values.push(value);
+            }
+
+            // Add the attributes to the layer
+            for (i = this._attrs.length; i--;) {
                 name = this._attrs[i].name;
                 value = this._values[i];
 
@@ -169,56 +197,131 @@ define([
             return self;
         };
 
-        // Each container will have `inputs` and `outputs`
+        Layer.prototype._getAllNodes = function() {
+            return [this._node()];
+        };
+
         var Container = function() {
+            Layer.apply(this, arguments);
+            this._nestedIndex = 0;
+        };
+
+        Container.prototype = Object.create(Layer.prototype);
+
+        Container.prototype.add = function(self, tlayer) {
+            var layer = tlayer.get('_node'),
+                container = this._node(),
+                children,
+                arch;
+
+            // Add a nested 'Architecture' node
+            arch = core.createNode({
+                parent: container,
+                base: META.Architecture
+            });
+
+            // Add this node to the 'addLayers' set
+            core.addMember(container, Constants.CONTAINED_LAYER_SET, arch);
+            // Assign it an appropriate 'index' value
+            core.setMemberRegistry(
+                container,
+                Constants.CONTAINED_LAYER_SET,
+                core.getPath(arch),
+                Constants.CONTAINED_LAYER_INDEX,
+                this._nestedIndex++
+            );
+
+            // Move the added node(s)/conns to this architecture node
+            children = layer._getAllNodes();
+            for (var i = children.length; i--;) {
+                core.moveNode(children[i], arch);
+            }
+            layer._parent = arch;
+            return self;
+        };
+
+        // Implicit Containers are sequential and concat containers;
+        // these containers are visually implied in deepforge (although
+        // they are explicitly defined in torch)
+        var ImplicitContainer = function() {
             // inputs and outputs are webgme nodes
             this._inputs = [];
             this._outputs = [];
+            this._children = [];
+            this._connections = [];
         };
 
-        Container.prototype.add = function() {
+        // Implicit containers will have to record their 'children'.
+        // When an implicit container is added to an actual container,
+        // the container will set it's '_parent' value. If any additional
+        // layers are added to the implicit container after, they will
+        // need to be moved to the parent of the implicit container
+        ImplicitContainer.prototype.add = function() {
             logger.error('Add is not overridden!');
         };
 
-        var Sequential = function(/*attrs, args*/) {
-            Container.call(this);
+        ImplicitContainer.prototype._getAllNodes = function() {
+            var nodes = this._children.map(layer => layer._getAllNodes())
+                .reduce((l1, l2) => l1.concat(l2), []);
+
+            return this._connections.concat(nodes);
         };
 
-        Sequential.prototype = new Container();
+        var Sequential = function(/*attrs, args*/) {
+            ImplicitContainer.call(this);
+        };
+
+        Sequential.prototype = new ImplicitContainer();
 
         Sequential.prototype.add = function(self, tlayer) {
             var layer = tlayer.get('_node'),
-                nodes = layer._inputs;
+                nodes = layer._inputs,
+                connections = [];
 
             // If this._inputs is empty, add the layer to the inputs list
             if (this._inputs.length === 0) {  // first node
                 this._inputs = this._inputs.concat(nodes);
             } else {
                 // connect all inputs of the added node to the current outputs
+                // add the connection to the list of allNodes
                 this._outputs.forEach(src =>
-                    nodes.forEach(dst => connect(src, dst))
+                    nodes.forEach(dst => connections.push(connect(src, dst)))
                 );
             }
             this._outputs = layer._outputs;
+            this._children.push(layer);
+            this._connections = this._connections.concat(connections);
+
+            // If _parent is set, move the nodes and connection to the _parent node
+            if (this._parent) {
+                nodes = layer._getAllNodes().concat(connections);
+                for (var i = nodes.length; i--;) {
+                    core.moveNode(nodes[i], this._parent);
+                }
+            }
             return self;
         };
 
         var Concat = function(attrs, args) {
-            Container.call(this);
+            ImplicitContainer.call(this);
 
             // Create a concat node and add it to this._outputs
             var concat = new Layer('Concat', attrs, args);
             this._outputs.push(concat._node());
+            this._children.push(concat);
         };
 
-        Concat.prototype = new Container();
+        Concat.prototype = new ImplicitContainer();
 
         Concat.prototype.add = function(self, tlayer) {
             // Connect the tlayer outputs to this._outputs
             var layer = tlayer.get('_node'),
-                concatLayer = this._outputs[0];
+                concatLayer = this._outputs[0],
+                connections = [],
+                nodes;
 
-            layer._outputs.forEach(output => connect(output, concatLayer));
+            layer._outputs.forEach(output =>
+                connections.push(connect(output, concatLayer)));
 
             // Connect the incomingly connected node to tlayer
             // TODO: This might not work if adding layers after this container is
@@ -226,14 +329,23 @@ define([
 
             // Add the layer's inputs to the inputs
             this._inputs = this._inputs.concat(layer._inputs);
+            this._children.push(layer);
+            this._connections = this._connections.concat(connections);
+            if (this._parent) {
+                nodes = layer._getAllNodes().concat(connections);
+                for (var i = nodes.length; i--;) {
+                    core.moveNode(nodes[i], this._parent);
+                }
+            }
             return self;
         };
 
         // Special layers (with special functions - like 'add')
-        var LAYERS = {
-            Concat: Concat,
-            Sequential: Sequential
-        };
+        var CONTAINERS,
+            LAYERS = {
+                Concat: Concat,
+                Sequential: Sequential
+            };
 
         var getValue = function(txt) {
             if (txt === 'true') {
@@ -286,6 +398,9 @@ define([
 
             if (LAYERS[type]) {
                 node = new LAYERS[type](args, attrs);
+            } else if (CONTAINERS[type]) {
+                node = new Container(type, args, attrs);
+                res.set('add', node.add.bind(node)); // add the 'add' method
             } else {  // Call generic Layer with type name
                 node = new Layer(type, args, attrs);
             }
@@ -327,11 +442,23 @@ define([
                 return;
             }
 
-            // TODO: Create the nn object
+            // Mocking the nn layers (as defined in the metamodel)
             var nn = lua.newContext()._G,
-                names = Object.keys(LayerDict);
+                names = Object.keys(LayerDict),
+                base,
+                baseName;
+
+            // For each layer, check the name of the base type. If it is 'Container',
+            // then it should be added to the CONTAINERS dictionary. This will change how
+            // it is handled in 'CreateLayer'
+            CONTAINERS = {};
 
             for (var i = names.length; i--;) {
+                base = core.getBase(META[names[i]]);
+                baseName = core.getAttribute(base, 'name');
+                if (baseName === 'Container') {
+                    CONTAINERS[names[i]] = true;
+                }
                 nn.set(names[i], CreateLayer.bind(null, names[i]));
             }
 

src/plugins/UpdateLibrarySeed/UpdateLibrarySeed.js

@@ -0,0 +1,151 @@
+/*globals define*/
+/*jshint node:true, browser:true*/
+
+define([
+    'text!./metadata.json',
+    'child_process',
+    'path',
+    'q',
+    'fs',
+    'module',
+    'plugin/PluginBase'
+], function (
+    pluginMetadata,
+    childProcess,
+    path,
+    Q,
+    fs,
+    module,
+    PluginBase
+) {
+    'use strict';
+
+    pluginMetadata = JSON.parse(pluginMetadata);
+    var SEEDS_DIR = path.join(path.dirname(module.uri), '..', '..', 'seeds');
+
+    /**
+     * Initializes a new instance of UpdateLibrarySeed.
+     * @class
+     * @augments {PluginBase}
+     * @classdesc This class represents the plugin UpdateLibrarySeed.
+     * @constructor
+     */
+    var UpdateLibrarySeed = function () {
+        // Call base class' constructor.
+        PluginBase.call(this);
+        this.pluginMetadata = pluginMetadata;
+    };
+
+    /**
+     * Metadata associated with the plugin. Contains id, name, version, description, icon, configStructue etc.
+     * This is also available at the instance at this.pluginMetadata.
+     * @type {object}
+     */
+    UpdateLibrarySeed.metadata = pluginMetadata;
+
+    // Prototypical inheritance from PluginBase.
+    UpdateLibrarySeed.prototype = Object.create(PluginBase.prototype);
+    UpdateLibrarySeed.prototype.constructor = UpdateLibrarySeed;
+
+    /**
+     * Main function for the plugin to execute. This will perform the execution.
+     * Notes:
+     * - Always log with the provided logger.[error,warning,info,debug].
+     * - Do NOT put any user interaction logic UI, etc. inside this method.
+     * - callback always has to be called even if error happened.
+     *
+     * @param {function(string, plugin.PluginResult)} callback - the result callback
+     */
+    UpdateLibrarySeed.prototype.main = function (callback) {
+        // get the root hash
+        var name = this.projectName,
+            version;
+
+        // get the name and validate
+        return this.getLibraryVersion()
+            .then(vers => {
+                version = vers;
+                return this.checkForLibName(name);
+            })
+            .then(valid => {
+                if (!valid) {
+                    var err = `Invalid library name "${name}"`;
+                    this.logger.error(err);
+                    return callback(err, this.result);
+                }
+                return this.updateSeed(name);
+            })
+            .then(() => this.recordVersion(name, version))
+            .then(() => {
+                this.logger.info(`Finished updating library seed for ${name}`);
+                this.result.setSuccess(true);
+                callback(null, this.result);
+            })
+            .fail(err => callback(err, this.result));
+    };
+
+    UpdateLibrarySeed.prototype.bumpVersion = function (rawVersion, releaseType) {
+        var vnames = ['major', 'minor', 'patch'],
+            bumpIndex = vnames.indexOf(releaseType),
+            version = rawVersion.split('.').map(num => parseInt(num));
+
+        version[bumpIndex]++;
+        // zero out the smaller numbers
+        while (++bumpIndex < version.length) {
+            version[bumpIndex] = 0;
+        }
+        return version.join('.');
+    };
+
+    UpdateLibrarySeed.prototype.getLibraryVersion = function () {
+        var version,
+            config = this.getCurrentConfig(),
+            newVersion;
+
+        version = (this.core.getAttribute(this.rootNode, 'version') || '0.0.0');
+        newVersion = this.bumpVersion(version, config.releaseType);
+
+        this.core.setAttribute(this.rootNode, 'version', newVersion);
+        return this.save(`Bumped version to ${newVersion}`).then(() => newVersion);
+    };
+
+    UpdateLibrarySeed.prototype.checkForLibName = function (name) {
+        // check for the library name from the fs
+        return Q.nfcall(fs.readdir, SEEDS_DIR).then(seeds => seeds.indexOf(name) !== -1);
+    };
+
+    UpdateLibrarySeed.prototype.updateSeed = function (seedName) {
+        var deferred = Q.defer(),
+            err,
+            job = childProcess.spawn('webgme', ['new', 'seed', seedName], {
+                cwd: path.dirname(module.uri)
+            });
+
+        this.logger.info(`Updating ${seedName} seed`);
+        job.on('error', _err => {
+            err = _err;
+            if (err.code === 'ENOENT') {
+                return deferred.reject('"webgme" command not found. Is webgme-cli installed on the server?');
+            }
+            return deferred.reject(err);
+        });
+
+        job.on('exit', code => {
+            if (!code) {
+                deferred.resolve();
+            } else {
+                deferred.reject(err || code);
+            }
+        });
+
+        return deferred.promise;
+    };
+
+    UpdateLibrarySeed.prototype.recordVersion = function (seed, version) {
+        var versionPath = path.join(SEEDS_DIR, seed, 'version.txt');
+        this.logger.info(`Updating ${seed} version (${version})`);
+        return Q.nfcall(fs.writeFile, versionPath, version);
+    };
+
+    return UpdateLibrarySeed;
+});

src/plugins/UpdateLibrarySeed/metadata.json

@@ -0,0 +1,28 @@
+{
+  "id": "UpdateLibrarySeed",
+  "name": "UpdateLibrarySeed",
+  "version": "0.1.0",
+  "description": "",
+  "icon": {
+    "class": "glyphicon glyphicon-cog",
+    "src": ""
+  },
+  "disableServerSideExecution": false,
+  "disableBrowserSideExecution": true,
+  "writeAccessRequired": false,
+  "configStructure": [
+      {
+          "name": "releaseType",
+          "displayName": "Release Type",
+          "description": "Specify major, minor or patch release",
+          "value": "minor",
+          "valueItems": [
+              "major",
+              "minor",
+              "patch"
+          ],
+          "valueType": "string",
+          "readOnly": false
+      }
+  ]
+}

src/routers/ExecPulse/ExecPulse.js

@@ -0,0 +1,131 @@
+/*jshint node:true*/
+
+// This is a REST endpoint keeping track of the heartbeats of each execution. This
+// allows detection of "disconnected" executions (enabling the reconnection of the
+// executions - issue #821)
+'use strict';
+
+var express = require('express'),
+    MONGO_COLLECTION = 'ExecPulse',
+    CONSTANTS = require('../../common/Constants').PULSE,
+    mongo,
+    storage,
+    router = express.Router();
+
+/**
+ * Called when the server is created but before it starts to listening to incoming requests.
+ * N.B. gmeAuth, safeStorage and workerManager are not ready to use until the start function is called.
+ * (However inside an incoming request they are all ensured to have been initialized.)
+ *
+ * @param {object} middlewareOpts - Passed by the webgme server.
+ * @param {GmeConfig} middlewareOpts.gmeConfig - GME config parameters.
+ * @param {GmeLogger} middlewareOpts.logger - logger
+ * @param {function} middlewareOpts.ensureAuthenticated - Ensures the user is authenticated.
+ * @param {function} middlewareOpts.getUserId - If authenticated retrieves the userId from the request.
+ * @param {object} middlewareOpts.gmeAuth - Authorization module.
+ * @param {object} middlewareOpts.safeStorage - Accesses the storage and emits events (PROJECT_CREATED, COMMIT..).
+ * @param {object} middlewareOpts.workerManager - Spawns and keeps track of "worker" sub-processes.
+ */
+function initialize(middlewareOpts) {
+    var logger = middlewareOpts.logger.fork('ExecPulse'),
+        ensureAuthenticated = middlewareOpts.ensureAuthenticated,
+        STALE_THRESHOLD = 7500;
+
+    storage = require('../storage')(logger, middlewareOpts.gmeConfig);
+    logger.debug('initializing ...');
+
+    // Ensure authenticated can be used only after this rule.
+    router.use('*', function (req, res, next) {
+        res.setHeader('X-WebGME-Media-Type', 'webgme.v1');
+        next();
+    });
+
+    // Use ensureAuthenticated if the routes require authentication. (Can be set explicitly for each route.)
+    router.use('*', ensureAuthenticated);
+
+    router.get('/', function (req, res) {
+        mongo.find().toArray((err, all) => {
+            if (err) {
+                return res.status(500).send(err);
+            }
+            res.json(all.map(entry => {
+                delete entry._id;
+                return entry;
+            }));
+        });
+    });
+
+    router.get('/:hash', function (req, res) {
+        // Check if the given job is alive (has a valid heartbeat).
+        // If the data doesn't exist, then it is considered alive
+        if (!req.params.hash) {
+            return res.status(400).send('Missing hash');
+        }
+
+        logger.debug('getting pulse of ', req.params.hash);
+        mongo.findOne({hash: req.params.hash})
+            .then(job => {
+                var current = Date.now(),
+                    result = CONSTANTS.DOESNT_EXIST;
+
+                if (job) {
+                    result = (current - job.timestamp) < STALE_THRESHOLD ?
+                        CONSTANTS.ALIVE : CONSTANTS.DEAD;
+                }
+                return res.status(200).send(result.toString());
+            });
+    });
+
+    router.post('/:hash', function (req, res) {
+        var timestamp = Date.now(),
+            job = {
+                hash: req.params.hash,
+                timestamp: timestamp
+            };
+
+        // Validate the input
+        logger.debug('Received heartbeat for ', job.hash);
+        if (!job.hash) {
+            return res.status(400).send('Missing hash');
+        }
+
+        // Delete the given job from the database
+        mongo.update({hash: job.hash}, job, {upsert: true})
+            .then(() => res.sendStatus(201));
+    });
+
+    router.delete('/:hash', function (req, res) {
+        // Delete the given job from the database
+        return mongo.findOneAndDelete({hash: req.params.hash})
+            .then(() => res.sendStatus(204));
+    });
+
+    logger.debug('ready');
+}
+
+/**
+ * Called before the server starts listening.
+ * @param {function} callback
+ */
+function start(callback) {
+    storage.then(db => {
+        mongo = db.collection(MONGO_COLLECTION);
+        callback();
+    });
+}
+
+/**
+ * Called after the server stopped listening.
+ * @param {function} callback
+ */
+function stop(callback) {
+    callback();
+}
+
+
+module.exports = {
+    initialize: initialize,
+    router: router,
+    start: start,
+    stop: stop
+};

src/routers/JobLogsAPI/JobLogManager.js

@@ -0,0 +1,149 @@
+var path = require('path'),
+    Q = require('q'),
+    fs = require('fs'),
+    exists = require('exists-file'),
+    utils = require('../../common/utils'),
+    NO_LOG_FOUND = '';
+
+var JobLogManager = function(logger, config) {
+    this.rootDir = path.join(config.blob.fsDir, 'log-storage');
+    this.logger = logger.fork('JobLogManager');
+    this._onCopyFinished = {};
+};
+
+JobLogManager.prototype._getFilePath = function(jInfo) {
+    this.logger.debug(`getting file path for ${jInfo.job} in ${jInfo.project} on ${jInfo.branch}`);
+    var jobId = jInfo.job.replace(/\//g, '_'),
+        filename = `${jobId}.txt`;
+
+    return path.join(this.rootDir, jInfo.project, jInfo.branch, filename);
+};
+
+JobLogManager.prototype.exists = function(jobInfo) {
+    var filename = this._getFilePath(jobInfo);
+    return Q.nfcall(exists, filename);
+};
+
+JobLogManager.prototype.mkdirIfNeeded = function(dir) {
+    return Q.nfcall(exists, dir).then(exist => {
+        if (!exist) {
+            this.logger.debug('making dir:', dir);
+            return Q.nfcall(fs.mkdir, dir)
+                .catch(() => this.logger.debug(`dir already created: ${dir}`));
+        }
+    });
+};
+
+JobLogManager.prototype._copyFile = function(src, dst) {
+    return Q.nfcall(exists, src).then(exists => {
+        if (!exists) {
+            this.logger.warn(`Cannot copy file from ${src}. File doesn't exist!`);
+            return;
+        }
+
+        return this.mkdirIfNeeded(path.dirname(dst)).then(() => {
+            var deferred = Q.defer(),
+                stream = fs.createReadStream(src).pipe(fs.createWriteStream(dst));
+
+            stream.on('error', deferred.reject);
+            stream.on('finish', deferred.resolve);
+
+            return deferred.promise;
+        });
+    });
+};
+
+// Copy one branch info to the next
+// Could optimize this to symlink until data appended...
+JobLogManager.prototype.migrate = function(migrationInfo, jobIds) {
+    // Recursively copy the srcBranch dir to the dstBranch dir
+    // Should probably use streams...
+    // Need to block appends to the given files so they are not written
+    // to until they have finished copying...
+    // TODO
+    var jobs,
+        src,
+        dst,
+        i;
+
+    for (i = jobIds.length; i--;) {
+        this._onCopyFinished[jobIds[i]] = [];
+    }
+
+    // Copy the job files and evaluate each of the finish functions
+    this.logger.debug('migrating from ' + migrationInfo.srcBranch + ' to '+ migrationInfo.dstBranch);
+    return Q.all(jobIds.map(jobId => {
+        src = this._getFilePath({
+            project: migrationInfo.project,
+            branch: migrationInfo.srcBranch,
+            job: jobId
+        });
+        dst = this._getFilePath({
+            project: migrationInfo.project,
+            branch: migrationInfo.dstBranch,
+            job: jobId
+        });
+        return this._copyFile(src, dst).then(() => {
+            jobs = this._onCopyFinished[jobId];
+            for (var j = jobs.length; j--;) {
+                jobs[j]();
+            }
+        });
+    }));
+};
+
+JobLogManager.prototype._appendTo = function(filename, logs) {
+    return Q.nfcall(exists, filename).then(exists => {
+        var promise = Q().then(() => '');
+        if (exists) {
+            promise = Q.nfcall(fs.readFile, filename, 'utf8');
+        }
+
+        return promise.then(content => {
+            // This could be optimized to not re-read/write the whole file each time...
+            var lines = utils.resolveCarriageReturns(content + logs);
+            return Q.nfcall(fs.writeFile, filename, lines.join('\n'));
+        });
+    });
+};
+
+JobLogManager.prototype.appendTo = function(jobInfo, logs) {
+    var filename = this._getFilePath(jobInfo),
+        branchDirname = path.dirname(filename),
+        projDirname = path.dirname(branchDirname);
+
+    this.logger.debug(`Appending content to ${filename}`);
+    // Make directory if needed
+    return this.mkdirIfNeeded(this.rootDir)
+        .then(() => this.mkdirIfNeeded(projDirname))
+        .then(() => this.mkdirIfNeeded(branchDirname))
+        .then(() => this._appendTo(filename, logs));
+};
+
+JobLogManager.prototype.getLog = function(jobInfo) {
+    var filename = this._getFilePath(jobInfo);
+
+    this.logger.info(`Getting log content from ${filename}`);
+    return this.exists(jobInfo)
+        .then(exists => {
+            if (exists) {
+                return Q.nfcall(fs.readFile, filename);
+            }
+            return NO_LOG_FOUND;
+        });
+};
+
+JobLogManager.prototype.delete = function(jobInfo) {
+    var filename = this._getFilePath(jobInfo);
+
+    return this.exists(jobInfo)
+        .then(exists => {
+            if (exists) {
+                this.logger.debug(`Removing file ${filename}`);
+                return Q.nfcall(fs.unlink, filename);
+            }
+            this.logger.debug(`${filename} doesn't exist. No need to delete...`);
+        });
+};
+
+module.exports = JobLogManager;

src/routers/JobLogsAPI/JobLogsAPI.js

@@ -0,0 +1,144 @@
+/*jshint node:true*/
+
+'use strict';
+
+var express = require('express'),
+    JobLogManager = require('./JobLogManager'),
+    MONGO_COLLECTION = 'JobLogsMetadata',
+    mongo,
+    router = express.Router(),
+    storage;
+
+/**
+ * Called when the server is created but before it starts to listening to incoming requests.
+ * N.B. gmeAuth, safeStorage and workerManager are not ready to use until the start function is called.
+ * (However inside an incoming request they are all ensured to have been initialized.)
+ *
+ * @param {object} middlewareOpts - Passed by the webgme server.
+ * @param {GmeConfig} middlewareOpts.gmeConfig - GME config parameters.
+ * @param {GmeLogger} middlewareOpts.logger - logger
+ * @param {function} middlewareOpts.ensureAuthenticated - Ensures the user is authenticated.
+ * @param {function} middlewareOpts.getUserId - If authenticated retrieves the userId from the request.
+ * @param {object} middlewareOpts.gmeAuth - Authorization module.
+ * @param {object} middlewareOpts.safeStorage - Accesses the storage and emits events (PROJECT_CREATED, COMMIT..).
+ * @param {object} middlewareOpts.workerManager - Spawns and keeps track of "worker" sub-processes.
+ */
+function initialize(middlewareOpts) {
+    var logger = middlewareOpts.logger.fork('JobLogsAPI'),
+        ensureAuthenticated = middlewareOpts.ensureAuthenticated,
+        gmeConfig = middlewareOpts.gmeConfig,
+        logManager = new JobLogManager(logger, gmeConfig);
+
+    logger.debug('initializing ...');
+    storage = require('../storage')(logger, gmeConfig);
+
+    // Ensure authenticated can be used only after this rule.
+    router.use('*', function (req, res, next) {
+        // This header ensures that any failures with authentication won't redirect.
+        res.setHeader('X-WebGME-Media-Type', 'webgme.v1');
+        next();
+    });
+
+    // Use ensureAuthenticated if the routes require authentication. (Can be set explicitly for each route.)
+    router.use('*', ensureAuthenticated);
+
+    router.get('/:project/:branch/:job', function (req, res/*, next*/) {
+        // Retrieve the job logs for the given job
+        logger.info(`Requested logs for ${req.params.job} in ${req.params.project}`);
+        logManager.getLog(req.params)
+            .then(log => {
+                res.set('Content-Type', 'text/plain');
+                res.send(log);
+            })
+            .catch(err => logger.error(`Log retrieval failed: ${err}`));
+    });
+
+    router.get('/metadata/:project/:branch/:job', function (req, res/*, next*/) {
+        logger.info(`Requested metadata for ${req.params.job} in ${req.params.project}`);
+        return mongo.findOne(req.params)
+            .then(info => {
+                var lineCount = info ? info.lineCount : -1;
+
+                return res.json({
+                    lineCount: lineCount
+                });
+            })
+            .catch(err => logger.error(`Metadata retrieval failed: ${err}`));
+    });
+
+    router.patch('/:project/:branch/:job', function (req, res/*, next*/) {
+        var logs = req.body.patch;
+        logger.info(`Received append request for ${req.params.job} in ${req.params.project}`);
+        return logManager.appendTo(req.params, logs)
+            .then(() => {
+                if (req.body.lineCount || req.body.cmdCount || req.body.createdIds) {
+                    var info = {
+                        project: req.params.project,
+                        branch: req.params.branch,
+                        job: req.params.job,
+                        lineCount: req.body.lineCount || -1,
+                        createdIds: req.body.createdIds || [],
+                        cmdCount: req.body.cmdCount || 0
+                    };
+                    logger.debug('metadata is', info);
+                    return mongo.update(req.params, info, {upsert: true})
+                        .then(() => res.send('Append successful'));
+                } else {
+                    res.send('Append successful');
+                }
+            })
+            .catch(err => logger.error(`Append failed: ${err}`));
+    });
+
+    router.delete('/:project/:branch/:job', function (req, res/*, next*/) {
+        logger.info(`Request to delete logs for ${req.params.job} in ${req.params.project}`);
+        logManager.delete(req.params)
+            .then(() => mongo.findOneAndDelete(req.params))
+            .then(() => {
+                logger.info('Job log deletion successful!');
+                res.status(204).send('delete successful');
+            })
+            .catch(err => logger.error(`Job log deletion failed: ${err}`));
+    });
+
+    router.post('/migrate/:project/:srcBranch/:dstBranch', function (req, res/*, next*/) {
+        var jobs = req.body.jobs;
+        logger.info(`Migrating logs from ${req.params.srcBranch} to ${req.params.dstBranch} in ${req.params.project}`);
+        logManager.migrate(req.params, jobs)
+            .then(() => {
+                logger.info('Log migration successful!');
+                res.send('migration successful');
+            })
+            .fail(err => logger.error(`migration failed: ${err}`));
+    });
+
+    logger.debug('ready');
+}
+
+/**
+ * Called before the server starts listening.
+ * @param {function} callback
+ */
+function start(callback) {
+    storage.then(db => {
+        mongo = db.collection(MONGO_COLLECTION);
+        callback();
+    });
+
+}
+
+/**
+ * Called after the server stopped listening.
+ * @param {function} callback
+ */
+function stop(callback) {
+    callback();
+}
+
+
+module.exports = {
+    initialize: initialize,
+    router: router,
+    start: start,
+    stop: stop
+};

src/routers/JobOriginAPI/JobOriginAPI.js

@@ -0,0 +1,162 @@
+/*jshint node:true*/
+
+'use strict';
+
+var express = require('express'),
+    MONGO_COLLECTION = 'JobOrigins',
+    utils = require('../utils'),
+    mongo,
+    router = express.Router(),
+    storage;
+
+/**
+ * Called when the server is created but before it starts to listening to incoming requests.
+ * N.B. gmeAuth, safeStorage and workerManager are not ready to use until the start function is called.
+ * (However inside an incoming request they are all ensured to have been initialized.)
+ *
+ * @param {object} middlewareOpts - Passed by the webgme server.
+ * @param {GmeConfig} middlewareOpts.gmeConfig - GME config parameters.
+ * @param {GmeLogger} middlewareOpts.logger - logger
+ * @param {function} middlewareOpts.ensureAuthenticated - Ensures the user is authenticated.
+ * @param {function} middlewareOpts.getUserId - If authenticated retrieves the userId from the request.
+ * @param {object} middlewareOpts.gmeAuth - Authorization module.
+ * @param {object} middlewareOpts.safeStorage - Accesses the storage and emits events (PROJECT_CREATED, COMMIT..).
+ * @param {object} middlewareOpts.workerManager - Spawns and keeps track of "worker" sub-processes.
+ */
+// When testing, use in memory storage...
+function initialize(middlewareOpts) {
+    var logger = middlewareOpts.logger.fork('JobOriginAPI'),
+        gmeConfig = middlewareOpts.gmeConfig,
+        ensureAuthenticated = middlewareOpts.ensureAuthenticated,
+        REQUIRED_FIELDS = ['hash', 'project', 'execution', 'job', 'nodeId', 'branch'];
+
+    storage = require('../storage')(logger, gmeConfig);
+
+    logger.debug('initializing ...');
+    // Ensure authenticated can be used only after this rule.
+    router.use('*', function (req, res, next) {
+        // This header ensures that any failures with authentication won't redirect.
+        res.setHeader('X-WebGME-Media-Type', 'webgme.v1');
+        next();
+    });
+
+    // Use ensureAuthenticated if the routes require authentication. (Can be set explicitly for each route.)
+    router.use('*', ensureAuthenticated);
+
+    // Connect to mongo...
+
+    router.get('/', function (req, res) {
+        mongo.find().toArray((err, all) => {
+            if (err) {
+                return res.status(500).send(err);
+            }
+            res.json(all.map(entry => {
+                delete entry._id;
+                return entry;
+            }));
+        });
+    });
+
+    router.get('/:jobHash', function (req, res/*, next*/) {
+        var hash = req.params.jobHash,
+            jobInfo = {};
+
+        mongo.findOne({hash: hash})
+            .then(result => {
+                if (result) {
+                    // Filter the result object
+                    for (var i = REQUIRED_FIELDS.length; i--;) {
+                        jobInfo[REQUIRED_FIELDS[i]] = result[REQUIRED_FIELDS[i]];
+                    }
+                    return res.json(jobInfo);
+                }
+                res.sendStatus(404);
+            })
+            .catch(err => {
+                logger.error(`Storing job info failed: ${err}`);
+                res.status(500).send(err);
+            });
+    });
+
+    router.post('/:jobHash', function (req, res/*, next*/) {
+        var hash = req.params.jobHash,
+            jobInfo = {
+                hash: hash,
+                project: req.body.project,
+                execution: req.body.execution,
+                branch: req.body.branch,
+                job: req.body.job,  // job name
+                nodeId: req.body.nodeId
+            };
+
+        // Check that none of the fields are undefined
+        var missing = utils.getMissingField(jobInfo, REQUIRED_FIELDS);
+        if (missing) {
+            return res.status(400).send(`Missing required field: ${missing}`);
+        }
+
+        logger.debug(`Storing job info for ${hash}`);
+        return mongo.insertOne(jobInfo)
+            .then(() => res.sendStatus(201))
+            .catch(err => {
+                logger.error(`Storing job info failed: ${err}`);
+                res.status(500).send(err.toString());
+            });
+    });
+
+    router.delete('/:jobHash', function (req, res/*, next*/) {
+        var hash = req.params.jobHash;
+
+        mongo.findOneAndDelete({hash: hash})
+            .then(() => res.sendStatus(204));
+    });
+
+    // on fork
+    router.patch('/:jobHash', function (req, res) {
+        var hash = req.params.jobHash;
+
+        if (!req.body.branch) {
+            return res.status(400).send('Missing "branch" field');
+        }
+
+        return mongo.findOneAndUpdate({hash: hash}, {$set: {branch: req.body.branch}})
+            .then(() => {
+                logger.debug('Finished updateOne!');
+                res.sendStatus(200);
+            })
+            .catch(err => {
+                logger.error(`Job update failed: ${err}`);
+                res.status(500).send(err);
+            });
+    });
+
+    logger.debug('ready');
+}
+
+/**
+ * Called before the server starts listening.
+ * @param {function} callback
+ */
+function start(callback) {
+    storage.then(db => {
+        mongo = db.collection(MONGO_COLLECTION);
+        callback();
+    });
+
+}
+
+/**
+ * Called after the server stopped listening.
+ * @param {function} callback
+ */
+function stop(callback) {
+    callback();
+}
+
+
+module.exports = {
+    initialize: initialize,
+    router: router,
+    start: start,
+    stop: stop
+};

src/routers/storage.js

@@ -0,0 +1,19 @@
+// Get a mongodb connection
+
+var mongodb = require('mongodb'),
+    connection;
+
+module.exports = function(logger, gmeConfig) {
+    if (!connection) {
+        connection = mongodb.MongoClient.connect(gmeConfig.mongo.uri, gmeConfig.mongo.options)
+            .then(db => {
+                logger.debug('Connected to mongo!');
+                return db;
+            })
+            .catch(err => {
+                logger.error(`Could not connect to mongo: ${err}`);
+                throw err;
+            });
+    }
+    return connection;
+};

src/routers/utils.js

@@ -0,0 +1,10 @@
+module.exports = {
+    getMissingField: function(array, fields) {
+        for (var i = fields.length; i--;) {
+            if (!array[fields[i]]) {
+                return fields[i];
+            }
+        }
+        return null;
+    }
+};

src/seeds/nn/version.txt

@@ -0,0 +1 @@
+0.5.0

src/seeds/pipeline/version.txt

@@ -0,0 +1 @@
+0.4.1

src/visualizers/Visualizers.json

@@ -31,8 +31,8 @@
   },
   {
     "id": "RootViz",
-    "title": "MainView",
-    "panel": "panels/MainView/MainViewPanel",
+    "title": "ForwardViz",
+    "panel": "panels/ForwardViz/ForwardVizPanel",
     "DEBUG_ONLY": false
   },
   {
@@ -71,6 +71,30 @@
     "panel": "panels/PipelineIndex/PipelineIndexPanel",
     "DEBUG_ONLY": false
   },
+  {
+    "id": "ArchitectureIndex",
+    "title": "ArchitectureIndex",
+    "panel": "panels/ArchIndex/ArchIndexPanel",
+    "DEBUG_ONLY": false
+  },
+  {
+    "id": "OperationIndex",
+    "title": "OperationIndex",
+    "panel": "js/Panels/ModelEditor/ModelEditorPanel",
+    "DEBUG_ONLY": false
+  },
+  {
+    "id": "LayerIndex",
+    "title": "LayerIndex",
+    "panel": "js/Panels/ModelEditor/ModelEditorPanel",
+    "DEBUG_ONLY": false
+  },
+  {
+    "id": "DataTypeIndex",
+    "title": "DataTypeIndex",
+    "panel": "js/Panels/ModelEditor/ModelEditorPanel",
+    "DEBUG_ONLY": false
+  },
   {
     "id": "JobEditor",
     "title": "JobEditor",
@@ -100,5 +124,29 @@
     "title": "ExecutionIndex",
     "panel": "panels/ExecutionIndex/ExecutionIndexPanel",
     "DEBUG_ONLY": false
+  },
+  {
+    "id": "WorkerHeader",
+    "title": "WorkerHeader",
+    "panel": "panels/WorkerHeader/WorkerHeaderPanel",
+    "DEBUG_ONLY": false
+  },
+  {
+    "id": "ArtifactIndex",
+    "title": "ArtifactIndex",
+    "panel": "panels/ArtifactIndex/ArtifactIndexPanel",
+    "DEBUG_ONLY": false
+  },
+  {
+    "id": "ArchIndex",
+    "title": "ArchIndex",
+    "panel": "panels/ArchIndex/ArchIndexPanel",
+    "DEBUG_ONLY": false
+  },
+  {
+    "id": "ForwardViz",
+    "title": "ForwardViz",
+    "panel": "panels/ForwardViz/ForwardVizPanel",
+    "DEBUG_ONLY": false
   }
-]
+]

src/visualizers/panels/ArchEditor/ArchEditorControl.js

@@ -4,14 +4,14 @@
 define([
     'deepforge/Constants',
     'deepforge/globals',
-    'panels/EasyDAG/EasyDAGControl',
+    'deepforge/viz/panels/ThumbnailControl',
     'js/NodePropertyNames',
     'js/Utils/ComponentSettings',
     'underscore'
 ], function (
     Constants,
     DeepForge,
-    EasyDAGControl,
+    ThumbnailControl,
     nodePropertyNames,
     ComponentSettings,
     _
@@ -21,10 +21,11 @@ define([
 
     var ArchEditorControl,
         DEFAULT_CONFIG = {
-            DefaultColor: '#ffb74d',
+            DefaultColor: '#80cbc4',
             LayerColors: {
-                Containers: '#ffb74d',
-                Convolution: '#2196f3',
+                Container: '#ffb74d',
+                NestedContainer: '#ffe0b2',
+                Convolution: '#42a5f5',
                 Simple: '#ff9100',
                 Transfer: '#80deea',
                 Misc: '#ce93d8'
@@ -32,12 +33,12 @@ define([
         };
 
     ArchEditorControl = function (options) {
-        EasyDAGControl.call(this, options);
+        ThumbnailControl.call(this, options);
         this._config = DEFAULT_CONFIG;
         ComponentSettings.resolveWithWebGMEGlobal(this._config, this.getComponentId());
     };
 
-    _.extend(ArchEditorControl.prototype, EasyDAGControl.prototype);
+    _.extend(ArchEditorControl.prototype, ThumbnailControl.prototype);
 
     ArchEditorControl.prototype.TERRITORY_RULE = {children: 1};
     ArchEditorControl.prototype.DEFAULT_DECORATOR = 'LayerDecorator';
@@ -46,7 +47,9 @@ define([
     };
 
     ArchEditorControl.prototype.selectedObjectChanged = function(id) {
-        EasyDAGControl.prototype.selectedObjectChanged.call(this, id);
+        this.nestedLevel = typeof id === 'string' ?
+            Math.floor(id.split('/').length/2) % 2 : 0;
+        ThumbnailControl.prototype.selectedObjectChanged.call(this, id);
 
         DeepForge.last.Architecture = id;
         if (typeof id === 'string') {
@@ -56,7 +59,8 @@ define([
     };
 
     ArchEditorControl.prototype._getObjectDescriptor = function(id) {
-        var desc = EasyDAGControl.prototype._getObjectDescriptor.call(this, id);
+        var node = this._client.getNode(id),
+            desc = ThumbnailControl.prototype._getObjectDescriptor.call(this, id);
 
         // Filter attributes
         if (!desc.isConnection) {
@@ -78,7 +82,7 @@ define([
 
             for (i = names.length; i--;) {
                 // check if it is a setter
-                schema = this._client.getAttributeSchema(id, names[i]);
+                schema = node.getAttributeMeta(names[i]);
                 if (names[i] === 'name' || schema.setterType) {
                     desc.attributes[names[i]] = allAttrs[names[i]];
                 }
@@ -87,8 +91,7 @@ define([
             // Add layer type (base class's base class)
             desc.layerType = null;
             if (desc.baseName) {
-                var node = this._client.getNode(id),
-                    base = this._client.getNode(node.getMetaTypeId()),
+                var base = this._client.getNode(node.getMetaTypeId()),
                     layerType = this._client.getNode(base.getBaseId()),
                     color;
 
@@ -97,11 +100,32 @@ define([
                     desc.layerType = layerType.getAttribute(nodePropertyNames.Attributes.name);
 
                     color = this._config.LayerColors[desc.layerType];
+                    if (desc.layerType === 'Container' && this.nestedLevel) {
+                        color = this._config.LayerColors.NestedContainer;
+                    }
                     if (!color) {
                         this._logger.warn(`No color found for ${desc.layerType}`);
                         color = this._config.DefaultColor;
                     }
                     desc.color = color;
+
+                    if (desc.layerType === 'Container') {
+                        desc.containedLayers = node.getMemberIds(Constants.CONTAINED_LAYER_SET)
+                            .map(layerId => {
+                                var index = node.getMemberRegistry(
+                                    Constants.CONTAINED_LAYER_SET,
+                                    layerId,
+                                    Constants.CONTAINED_LAYER_INDEX
+                                );
+                                return [layerId, index];
+                            })
+                            .sort((a, b) => a[1] < b[1] ? -1 : 1)
+                            .map(tuple => tuple[0]);
+
+                        // Set the decorator to ContainerLayerDecorator
+                        desc.Decorator = this._client.decoratorManager
+                            .getDecoratorForWidget('ContainerLayerDecorator', 'EasyDAG');
+                    }
                 }
             }
         }
@@ -109,30 +133,19 @@ define([
     };
 
     ////////////////////////// Layer Selection Logic //////////////////////////
-    ArchEditorControl.prototype._getValidInitialNodes = function() {
-        return this._client.getChildrenMeta(this._currentNodeId).items
-            // For now, anything is possible!
-            // FIXME
-            .map(info => this._getAllDescendentIds(info.id))
-            .reduce((prev, curr) => prev.concat(curr))
-            // Filter all abstract nodes
-            .filter(nodeId => {
-                return !this._client.getNode(nodeId).isAbstract();
-            })
-            .map(id => this._getObjectDescriptor(id))
-            .filter(obj => !obj.isConnection && obj.name !== 'Connection')
-            .filter(layer => layer.layerType !== 'Criterion');
-    };
-
-    ArchEditorControl.prototype._getValidSuccessorNodes =
+    ArchEditorControl.prototype.getValidSuccessors =
     ArchEditorControl.prototype._getValidInitialNodes =
     ArchEditorControl.prototype.getNonCriterionLayers = function() {
         // Return all (non-criterion) layer types
         var metanodes = this._client.getAllMetaNodes(),
             layerId,
+            connId,
+            conn,
             criterionId,
-            allLayerIds = [],
+            allLayers = [],
             layers = [],
+            tgts,
+            j,
             i;
 
         for (i = metanodes.length; i--;) {
@@ -142,24 +155,40 @@ define([
             }
         }
 
+        // Remove all criterion layers and abstract layers
         for (i = metanodes.length; i--;) {
             if (layerId) {
-                if (!metanodes[i].isAbstract() &&
-                    this._client.isTypeOf(metanodes[i].getId(), layerId)) {
+                if (!metanodes[i].isAbstract() && metanodes[i].isTypeOf(layerId)) {
 
                     if (metanodes[i].getAttribute('name') === 'Criterion') {
                         criterionId = metanodes[i].getId();
                     } else {
-                        allLayerIds.push(metanodes[i].getId());
+                        allLayers.push(metanodes[i]);
+                    }
+                } else if (!connId && metanodes[i].getAttribute('name') === 'Connection') {  // Detect the layer connection type...
+                    tgts = this._client.getPointerMeta(metanodes[i].getId(), 'src').items;
+                    for (j = tgts.length; j--;) {
+                        if (tgts[j].id === layerId) {
+                            connId = metanodes[i].getId();
+                        }
                     }
                 }
             }
         }
 
+        if (!connId) {
+            this._logger.warn('Could not find a layer connector');
+            return [];
+        }
+        // Convert the layers into the correct format
+        conn = this._getObjectDescriptor(connId);
         // Remove all criterion layers and abstract layers
-        for (i = allLayerIds.length; i--;) {
-            if (!this._client.isTypeOf(allLayerIds[i], criterionId)) {
-                layers.push({node: this._getObjectDescriptor(allLayerIds[i])});
+        for (i = allLayers.length; i--;) {
+            if (!allLayers[i].isTypeOf(criterionId)) {
+                layers.push({
+                    node: this._getObjectDescriptor(allLayers[i].getId()),
+                    conn: conn
+                });
             }
         }
 
@@ -172,8 +201,9 @@ define([
 
     // Widget extensions
     ArchEditorControl.prototype._initWidgetEventHandlers = function() {
-        EasyDAGControl.prototype._initWidgetEventHandlers.call(this);
+        ThumbnailControl.prototype._initWidgetEventHandlers.call(this);
         this._widget.getCreateNewDecorator = this.getCreateNewDecorator.bind(this);
+        this._widget.insertLayer = this.insertLayer.bind(this);
     };
 
     ArchEditorControl.prototype.getCreateNewDecorator = function() {
@@ -183,5 +213,40 @@ define([
         );
     };
 
+    ArchEditorControl.prototype.insertLayer = function(layerBaseId, connId) {
+        var conn = this._client.getNode(connId),
+            parentId = conn.getParentId(),
+            layerId,
+            nextLayerId = conn.getPointer('dst').to,
+            connBaseId = conn.getBaseId(),
+            newConnId,
+
+            baseName = this._client.getNode(layerBaseId).getAttribute('name'),
+            prevLayerId = conn.getPointer('src').to,
+            srcName = this._client.getNode(prevLayerId).getAttribute('name'),
+            dstName = this._client.getNode(nextLayerId).getAttribute('name'),
+            msg = `Inserting ${baseName} layer between ${srcName} and ${dstName}`;
+
+        this._client.startTransaction(msg);
+        // Create the new layer
+        layerId = this._client.createNode({
+            parentId: parentId,
+            baseId: layerBaseId
+        });
+
+        // Connect the new layer to the previous dst of 'connId'
+        newConnId = this._client.createNode({
+            parentId: parentId,
+            baseId: connBaseId
+        });
+        this._client.setPointer(newConnId, 'src', layerId);
+        this._client.setPointer(newConnId, 'dst', nextLayerId);
+
+        // Change the dst of 'connId' to the new layer
+        this._client.setPointer(connId, 'dst', layerId);
+
+        this._client.completeTransaction();
+    };
+
     return ArchEditorControl;
 });

src/visualizers/panels/ArchIndex/ArchIndexPanel.js

@@ -0,0 +1,100 @@
+/*globals define, _, WebGMEGlobal*/
+/*jshint browser: true*/
+
+define([
+    'js/PanelBase/PanelBaseWithHeader',
+    'js/PanelManager/IActivePanel',
+    'widgets/ArchIndex/ArchIndexWidget',
+    'panels/PipelineIndex/PipelineIndexControl'
+], function (
+    PanelBaseWithHeader,
+    IActivePanel,
+    ArchIndexWidget,
+    ArchIndexControl
+) {
+    'use strict';
+
+    var ArchIndexPanel;
+
+    ArchIndexPanel = function (layoutManager, params) {
+        var options = {};
+        //set properties from options
+        options[PanelBaseWithHeader.OPTIONS.LOGGER_INSTANCE_NAME] = 'ArchIndexPanel';
+        options[PanelBaseWithHeader.OPTIONS.FLOATING_TITLE] = true;
+
+        //call parent's constructor
+        PanelBaseWithHeader.apply(this, [options, layoutManager]);
+
+        this._client = params.client;
+        this._embedded = params.embedded;
+
+        //initialize UI
+        this._initialize();
+
+        this.logger.debug('ctor finished');
+    };
+
+    //inherit from PanelBaseWithHeader
+    _.extend(ArchIndexPanel.prototype, PanelBaseWithHeader.prototype);
+    _.extend(ArchIndexPanel.prototype, IActivePanel.prototype);
+
+    ArchIndexPanel.prototype._initialize = function () {
+        var self = this;
+
+        //set Widget title
+        this.setTitle('');
+
+        this.widget = new ArchIndexWidget(this.logger, this.$el);
+
+        this.widget.setTitle = function (title) {
+            self.setTitle(title);
+        };
+
+        this.control = new ArchIndexControl({
+            logger: this.logger,
+            client: this._client,
+            embedded: this._embedded,
+            widget: this.widget
+        });
+
+        this.onActivate();
+    };
+
+    /* OVERRIDE FROM WIDGET-WITH-HEADER */
+    /* METHOD CALLED WHEN THE WIDGET'S READ-ONLY PROPERTY CHANGES */
+    ArchIndexPanel.prototype.onReadOnlyChanged = function (isReadOnly) {
+        //apply parent's onReadOnlyChanged
+        PanelBaseWithHeader.prototype.onReadOnlyChanged.call(this, isReadOnly);
+
+    };
+
+    ArchIndexPanel.prototype.onResize = function (width, height) {
+        this.logger.debug('onResize --> width: ' + width + ', height: ' + height);
+    };
+
+    /* * * * * * * * Visualizer life cycle callbacks * * * * * * * */
+    ArchIndexPanel.prototype.destroy = function () {
+        this.control.destroy();
+        this.widget.destroy();
+
+        PanelBaseWithHeader.prototype.destroy.call(this);
+        WebGMEGlobal.KeyboardManager.setListener(undefined);
+        WebGMEGlobal.Toolbar.refresh();
+    };
+
+    ArchIndexPanel.prototype.onActivate = function () {
+        this.widget.onActivate();
+        this.control.onActivate();
+        WebGMEGlobal.KeyboardManager.setListener(this.widget);
+        WebGMEGlobal.Toolbar.refresh();
+    };
+
+    ArchIndexPanel.prototype.onDeactivate = function () {
+        this.widget.onDeactivate();
+        this.control.onDeactivate();
+        WebGMEGlobal.KeyboardManager.setListener(undefined);
+        WebGMEGlobal.Toolbar.refresh();
+    };
+
+    return ArchIndexPanel;
+});

src/visualizers/panels/ArtifactIndex/ArtifactIndexControl.js

@@ -0,0 +1,203 @@
+/*globals define, WebGMEGlobal*/
+/*jshint browser: true*/
+
+define([
+    'blob/BlobClient',
+    'js/Constants'
+], function (
+    BlobClient,
+    CONSTANTS
+) {
+
+    'use strict';
+
+    var ArtifactIndexControl;
+
+    ArtifactIndexControl = function (options) {
+
+        this._logger = options.logger.fork('Control');
+        this.blobClient = new BlobClient({
+            logger: this._logger.fork('BlobClient')
+        });
+
+        this._client = options.client;
+
+        // Initialize core collections and variables
+        this._widget = options.widget;
+
+        this._currentNodeId = null;
+        this._initWidgetEventHandlers();
+
+        this._logger.debug('ctor finished');
+    };
+
+    ArtifactIndexControl.prototype._initWidgetEventHandlers = function () {
+        this._widget.onNodeClick = (/*id*/) => {
+            // Change the current active object
+            // This is currently disabled as there are not any good
+            // visualizers for the data types
+            // WebGMEGlobal.State.registerActiveObject(id);
+        };
+
+        this._widget.onNodeDeleteClicked = id => {
+            var name = this._client.getNode(id).getAttribute('name'),
+                msg = `Deleted "${name}" artifact (${id}) --`;
+
+            this._client.startTransaction(msg);
+            this._client.deleteNode(id);
+            this._client.completeTransaction();
+        };
+    };
+
+    /* * * * * * * * Visualizer content update callbacks * * * * * * * */
+    // One major concept here is with managing the territory. The territory
+    // defines the parts of the project that the visualizer is interested in
+    // (this allows the browser to then only load those relevant parts).
+    ArtifactIndexControl.prototype.selectedObjectChanged = function (nodeId) {
+        this._logger.debug('activeObject nodeId \'' + nodeId + '\'');
+
+        // Remove current territory patterns
+        if (this._currentNodeId) {
+            this._client.removeUI(this._territoryId);
+        }
+
+        this._currentNodeId = nodeId;
+
+        if (typeof this._currentNodeId === 'string') {
+            // Put new node's info into territory rules
+            this._widget.currentNode = this._currentNodeId;
+            this._selfPatterns = {};
+
+            this._territoryId = this._client.addUI(this, events => {
+                this._eventCallback(events);
+            });
+
+            this._selfPatterns[nodeId] = {children: 1};
+            this._client.updateTerritory(this._territoryId, this._selfPatterns);
+        }
+    };
+
+    // This next function retrieves the relevant node information for the widget
+    ArtifactIndexControl.prototype._getObjectDescriptor = function (nodeId) {
+        var node = this._client.getNode(nodeId),
+            base,
+            hash,
+            objDescriptor;
+
+        if (node) {
+            base = this._client.getNode(node.getBaseId());
+            hash = node.getAttribute('data');
+            objDescriptor = {
+                id: node.getId(),
+                type: base ? base.getAttribute('name') : 'n/a',
+                name: node.getAttribute('name'),
+                createdAt: node.getAttribute('createdAt'),
+                dataURL: this.blobClient.getDownloadURL(hash),
+                parentId: node.getParentId()
+            };
+        }
+
+        return this.blobClient.getMetadata(hash)
+            .then(metadata => {
+                objDescriptor.size = this._humanFileSize(metadata.size);
+                return objDescriptor;
+            });
+    };
+
+    ArtifactIndexControl.prototype._humanFileSize = function (bytes, si) {
+        var thresh = si ? 1000 : 1024,
+            units = si ?
+                ['kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'] :
+                ['KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'],
+            u = -1;
+
+        if (bytes < thresh) {
+            return bytes + ' B';
+        }
+
+        do {
+            bytes = bytes / thresh;
+            u += 1;
+        } while (bytes >= thresh);
+
+        return bytes.toFixed(1) + ' ' + units[u];
+    };
+
+    /* * * * * * * * Node Event Handling * * * * * * * */
+    ArtifactIndexControl.prototype._eventCallback = function (events) {
+        var i = events ? events.length : 0,
+            event;
+
+        this._logger.debug('_eventCallback \'' + i + '\' items');
+
+        while (i--) {
+            event = events[i];
+            switch (event.etype) {
+
+            case CONSTANTS.TERRITORY_EVENT_LOAD:
+                this._onLoad(event.eid);
+                break;
+            case CONSTANTS.TERRITORY_EVENT_UPDATE:
+                this._onUpdate(event.eid);
+                break;
+            case CONSTANTS.TERRITORY_EVENT_UNLOAD:
+                this._onUnload(event.eid);
+                break;
+            default:
+                break;
+            }
+        }
+
+        this._logger.debug('_eventCallback \'' + events.length + '\' items - DONE');
+    };
+
+    ArtifactIndexControl.prototype._onLoad = function (gmeId) {
+        this._getObjectDescriptor(gmeId).then(desc => this._widget.addNode(desc));
+    };
+
+    ArtifactIndexControl.prototype._onUpdate = function (gmeId) {
+        this._getObjectDescriptor(gmeId).then(desc => this._widget.updateNode(desc));
+    };
+
+    ArtifactIndexControl.prototype._onUnload = function (gmeId) {
+        this._widget.removeNode(gmeId);
+    };
+
+    ArtifactIndexControl.prototype._stateActiveObjectChanged = function (model, activeObjectId) {
+        if (this._currentNodeId === activeObjectId) {
+            // The same node selected as before - do not trigger
+        } else {
+            this.selectedObjectChanged(activeObjectId);
+        }
+    };
+
+    /* * * * * * * * Visualizer life cycle callbacks * * * * * * * */
+    ArtifactIndexControl.prototype.destroy = function () {
+        this._detachClientEventListeners();
+    };
+
+    ArtifactIndexControl.prototype._attachClientEventListeners = function () {
+        this._detachClientEventListeners();
+        WebGMEGlobal.State.on('change:' + CONSTANTS.STATE_ACTIVE_OBJECT, this._stateActiveObjectChanged, this);
+    };
+
+    ArtifactIndexControl.prototype._detachClientEventListeners = function () {
+        WebGMEGlobal.State.off('change:' + CONSTANTS.STATE_ACTIVE_OBJECT, this._stateActiveObjectChanged);
+    };
+
+    ArtifactIndexControl.prototype.onActivate = function () {
+        this._attachClientEventListeners();
+
+        if (typeof this._currentNodeId === 'string') {
+            WebGMEGlobal.State.registerSuppressVisualizerFromNode(true);
+            WebGMEGlobal.State.registerActiveObject(this._currentNodeId);
+            WebGMEGlobal.State.registerSuppressVisualizerFromNode(false);
+        }
+    };
+
+    ArtifactIndexControl.prototype.onDeactivate = function () {
+        this._detachClientEventListeners();
+    };
+
+    return ArtifactIndexControl;
+});

src/visualizers/panels/ArtifactIndex/ArtifactIndexPanel.js

@@ -0,0 +1,99 @@
+/*globals define, _, WebGMEGlobal*/
+/*jshint browser: true*/
+
+define([
+    'js/PanelBase/PanelBaseWithHeader',
+    'js/PanelManager/IActivePanel',
+    'widgets/ArtifactIndex/ArtifactIndexWidget',
+    './ArtifactIndexControl'
+], function (
+    PanelBaseWithHeader,
+    IActivePanel,
+    ArtifactIndexWidget,
+    ArtifactIndexControl
+) {
+    'use strict';
+
+    var ArtifactIndexPanel;
+
+    ArtifactIndexPanel = function (layoutManager, params) {
+        var options = {};
+        //set properties from options
+        options[PanelBaseWithHeader.OPTIONS.LOGGER_INSTANCE_NAME] = 'ArtifactIndexPanel';
+        options[PanelBaseWithHeader.OPTIONS.FLOATING_TITLE] = true;
+
+        //call parent's constructor
+        PanelBaseWithHeader.apply(this, [options, layoutManager]);
+
+        this._client = params.client;
+
+        //initialize UI
+        this._initialize();
+
+        this.logger.debug('ctor finished');
+    };
+
+    //inherit from PanelBaseWithHeader
+    _.extend(ArtifactIndexPanel.prototype, PanelBaseWithHeader.prototype);
+    _.extend(ArtifactIndexPanel.prototype, IActivePanel.prototype);
+
+    ArtifactIndexPanel.prototype._initialize = function () {
+        var self = this;
+
+        //set Widget title
+        this.setTitle('');
+
+        this.widget = new ArtifactIndexWidget(this.logger, this.$el);
+
+        this.widget.setTitle = function (title) {
+            self.setTitle(title);
+        };
+
+        this.control = new ArtifactIndexControl({
+            logger: this.logger,
+            client: this._client,
+            widget: this.widget
+        });
+
+        this.onActivate();
+    };
+
+    /* OVERRIDE FROM WIDGET-WITH-HEADER */
+    /* METHOD CALLED WHEN THE WIDGET'S READ-ONLY PROPERTY CHANGES */
+    ArtifactIndexPanel.prototype.onReadOnlyChanged = function (isReadOnly) {
+        //apply parent's onReadOnlyChanged
+        PanelBaseWithHeader.prototype.onReadOnlyChanged.call(this, isReadOnly);
+
+    };
+
+    ArtifactIndexPanel.prototype.onResize = function (width, height) {
+        this.logger.debug('onResize --> width: ' + width + ', height: ' + height);
+        this.widget.onWidgetContainerResize(width, height);
+    };
+
+    /* * * * * * * * Visualizer life cycle callbacks * * * * * * * */
+    ArtifactIndexPanel.prototype.destroy = function () {
+        this.control.destroy();
+        this.widget.destroy();
+
+        PanelBaseWithHeader.prototype.destroy.call(this);
+        WebGMEGlobal.KeyboardManager.setListener(undefined);
+        WebGMEGlobal.Toolbar.refresh();
+    };
+
+    ArtifactIndexPanel.prototype.onActivate = function () {
+        this.widget.onActivate();
+        this.control.onActivate();
+        WebGMEGlobal.KeyboardManager.setListener(this.widget);
+        WebGMEGlobal.Toolbar.refresh();
+    };
+
+    ArtifactIndexPanel.prototype.onDeactivate = function () {
+        this.widget.onDeactivate();
+        this.control.onDeactivate();
+        WebGMEGlobal.KeyboardManager.setListener(undefined);
+        WebGMEGlobal.Toolbar.refresh();
+    };
+
+    return ArtifactIndexPanel;
+});

src/visualizers/panels/ClassCodeEditor/ClassCodeEditorControl.js

@@ -100,7 +100,7 @@ define([
 
         this._client.startTransaction(`Updating class "${name || nodeName}"`);
         if (name) {
-            this._client.setAttributes(id, 'name', name);
+            this._client.setAttribute(id, 'name', name);
         }
         if (basePath) {
             this._client.setBase(id, basePath);
@@ -117,7 +117,7 @@ define([
             }
         }
 
-        TextEditorControl.prototype.saveTextFor.call(this, id, text);
+        TextEditorControl.prototype.saveTextFor.call(this, id, text, true);
         this._client.completeTransaction();
     };
 
@@ -137,7 +137,7 @@ define([
         }
 
         for (i = nameMatches.length; i--;) {
-            if (this._client.isTypeOf(nameMatches[i].getId(), classNode.getId())) {
+            if (nameMatches[i].isTypeOf(classNode.getId())) {
                 return nameMatches[i].getId();
             }
         }

src/visualizers/panels/ExecutionIndex/ExecutionIndexControl.js

@@ -56,7 +56,7 @@ define([
             // Refresh the other lines visible
             otherLines = Object.keys(this.displayedExecutions)
                 .filter(eId => this.displayedExecutions[eId] && (eId !== id))
-                .map(id => this._linesForExecution[id])
+                .map(id => this._linesForExecution[id] || [])
                 .reduce((l1, l2) => l1.concat(l2), []);
 
             this._updateLines(otherLines, false);
@@ -70,7 +70,8 @@ define([
         var action = added ? 'addNode' : 'removeNode';
 
         // If removing, just get the ids
-        lines = !added ? lines : lines.map(line => this._getObjectDescriptor(line));
+        lines = !added ? lines : lines.map(line => this._getObjectDescriptor(line))
+            .filter(line => !!line);
 
         // update the given lines
         for (var i = lines.length; i--;) {
@@ -160,7 +161,9 @@ define([
                 desc.abbr = this.getUniqAbbreviation(desc);
 
                 // Create a territory for this origin and update it!
-                this._selfPatterns[desc.originId] = {children: 0};
+                if (desc.originId) {
+                    this._selfPatterns[desc.originId] = {children: 0};
+                }
                 setTimeout(() => this._client.updateTerritory(this._territoryId, this._selfPatterns), 0);
             } else if (type === 'Line') {
                 desc = this.getLineDesc(node);

src/visualizers/panels/ExecutionView/ExecutionViewControl.js

@@ -6,12 +6,14 @@ define([
     'deepforge/Constants',
     'panels/EasyDAG/EasyDAGControl',
     'deepforge/viz/PipelineControl',
+    'deepforge/viz/Execute',
     'underscore'
 ], function (
     GME_CONSTANTS,
     CONSTANTS,
     EasyDAGControl,
     PipelineControl,
+    Execute,
     _
 ) {
 
@@ -21,15 +23,18 @@ define([
 
     ExecutionViewControl = function (options) {
         EasyDAGControl.call(this, options);
+        Execute.call(this, this._client, this._logger);
         this.addedNodes = {};
         this.originTerritory = {};
         this.originTerritoryId = null;
+        this.readOnly = false;
     };
 
     _.extend(
         ExecutionViewControl.prototype,
         EasyDAGControl.prototype,
-        PipelineControl.prototype
+        PipelineControl.prototype,
+        Execute.prototype
     );
 
     /* * * * * * * * Visualizer content update callbacks * * * * * * * */
@@ -76,6 +81,9 @@ define([
                     this._client.removeUI(this.originTerritoryId);
                 }
             }
+            if (!this.readOnly) {
+                this.checkPipelineExecution(this._client.getNode(id));
+            }
         }
     };
 

src/visualizers/panels/ExecutionView/ExecutionViewPanel.js

@@ -73,7 +73,7 @@ define([
     ExecutionViewPanel.prototype.onReadOnlyChanged = function (isReadOnly) {
         //apply parent's onReadOnlyChanged
         PanelBaseWithHeader.prototype.onReadOnlyChanged.call(this, isReadOnly);
-
+        this.control.readOnly = isReadOnly;
     };
 
     ExecutionViewPanel.prototype.onResize = function (width, height) {

src/visualizers/panels/Footer/FilteredNotificationWidget.js

@@ -0,0 +1,28 @@
+/* globals define */
+// A notification widget which filters out stdout update notifications
+define([
+    'deepforge/Constants',
+    'js/Widgets/Notification/NotificationWidget'
+], function(
+    CONSTANTS,
+    GmeNotificationWidget
+) {
+    var NotificationWidget = function() {
+        GmeNotificationWidget.apply(this, arguments);
+    };
+
+    NotificationWidget.prototype = Object.create(GmeNotificationWidget.prototype);
+
+    NotificationWidget.prototype.isUserNotication = function(data) {
+        return data.message.indexOf(CONSTANTS.STDOUT_UPDATE) === -1;
+    };
+
+    NotificationWidget.prototype._refreshNotifications = function(eventData) {
+
+        if (this.isUserNotication(eventData)) {
+            GmeNotificationWidget.prototype._refreshNotifications.call(this, eventData);
+        }
+    };
+
+    return NotificationWidget;
+});

src/visualizers/panels/Footer/FooterPanel.js

@@ -1,22 +1,17 @@
-/*globals define, _, WebGMEGlobal, $ */
+/*globals define, _, $ */
 /*jshint browser: true*/
-/**
- * @author rkereskenyi / https://github.com/rkereskenyi
- */
 
 define([
     'js/PanelBase/PanelBase',
     'js/Widgets/NetworkStatus/NetworkStatusWidget',
     'js/Widgets/BranchStatus/BranchStatusWidget',
-    'js/Widgets/BranchSelector/BranchSelectorWidget',
-    'js/Widgets/KeyboardManager/KeyboardManagerWidget',
-    'js/Widgets/Notification/NotificationWidget'
-], function (PanelBase,
-             NetworkStatusWidget,
-             BranchStatusWidget,
-             BranchSelectorWidget,
-             KeyboardManagerWidget,
-             NotificationWidget) {
+    './FilteredNotificationWidget'
+], function (
+    PanelBase,
+    NetworkStatusWidget,
+    BranchStatusWidget,
+    NotificationWidget
+) {
 
     'use strict';
 
@@ -48,7 +43,6 @@ define([
             navBarInner = $('<div/>', {class: 'navbar-inner'}),
             separator = $('<div class="spacer pull-right"></div>'),
             widgetPlaceHolder = $('<div class="pull-right"></div>'),
-            keyBoardManagerEl,
             networkStatusEl,
             branchStatusEl,
             notificationEl;
@@ -59,13 +53,6 @@ define([
         //padding from screen right edge
         navBarInner.append(separator.clone());
 
-        //keyboard enable/disbale widget (NOTE: only on non touch device)
-        if (WebGMEGlobal.SUPPORTS_TOUCH !== true) {
-            keyBoardManagerEl = widgetPlaceHolder.clone();
-            new KeyboardManagerWidget(keyBoardManagerEl);
-            navBarInner.append(keyBoardManagerEl).append(separator.clone());
-        }
-
         networkStatusEl = widgetPlaceHolder.clone();
         new NetworkStatusWidget(networkStatusEl, this._client);
         navBarInner.append(networkStatusEl).append(separator.clone());

src/visualizers/panels/ForgeActionButton/Actions.js

@@ -252,18 +252,7 @@ define([
                     return this.isRunning();
                 },
                 action: function() {
-                    // Stop every running job
-                    var execNode = this.client.getNode(this._currentNodeId),
-                        jobIds = execNode.getChildrenIds(),
-                        msg = `Canceling ${execNode.getAttribute('name')} execution`;
-
-                    this.client.startTransaction(msg);
-                    jobIds.map(id => this.client.getNode(id))
-                        .filter(job => this.isRunning(job))  // get running jobs
-                        .forEach(job => this.stopJob(job));  // stop them
-
-                    this.client.setAttributes(execNode.getId(), 'status', 'canceled');
-                    this.client.completeTransaction();
+                    this.stopExecution();
                 }
             }
         ],
@@ -275,6 +264,32 @@ define([
                 action: function() {
                     this.addOperation();
                 }
+            },
+            {
+                name: 'Export for local execution',
+                icon: 'play_for_work',
+                priority: -1,
+                action: function() {
+                    var pluginId = 'GenerateExecFile',
+                        context = this.client.getCurrentPluginContext(pluginId);
+
+                    // Run the plugin in the browser (set namespace)
+                    context.managerConfig.namespace = 'pipeline';
+                    context.pluginConfig = {};
+                    Q.ninvoke(this.client, 'runBrowserPlugin', pluginId, context)
+                        .then(res => {
+                            var id = this._currentNodeId,
+                                node = this.client.getNode(id),
+                                base = this.client.getNode(node.getBaseId()),
+                                type = base.getAttribute('name'),
+                                name = node.getAttribute('name');
+
+                            // Get the file and download it
+                            this.downloadFromBlob(res.artifacts[0]);
+                            Materialize.toast(`Exported ${name} ${type}!`, 2000);
+                        })
+                        .fail(err => Materialize.toast(`Export failed: ${err}`, 2000));
+                }
             }
         ],
         Architecture: [

src/visualizers/panels/ForgeActionButton/ForgeActionButton.js

@@ -3,6 +3,7 @@
 
 define([
     'blob/BlobClient',
+    'js/Utils/SaveToDisk',
     'js/Constants',
     'panel/FloatingActionButton/FloatingActionButton',
     'deepforge/viz/PipelineControl',
@@ -13,10 +14,10 @@ define([
     'js/RegistryKeys',
     'js/Panels/MetaEditor/MetaEditorConstants',
     'q',
-    'text!./PluginConfig.json',
     'deepforge/globals'
 ], function (
     BlobClient,
+    SaveToDisk,
     CONSTANTS,
     PluginButton,
     PipelineControl,
@@ -27,7 +28,6 @@ define([
     REGISTRY_KEYS,
     META_CONSTANTS,
     Q,
-    PluginConfig,
     DeepForge
 ) {
     'use strict';
@@ -35,7 +35,6 @@ define([
     var NEW_OPERATION_ID = '__NEW_OPERATION__';
     var ForgeActionButton= function (layoutManager, params) {
         PluginButton.call(this, layoutManager, params);
-        this._pluginConfig = JSON.parse(PluginConfig);
         this._client = this.client;
         this._actions = [];
         this._blobClient = new BlobClient({
@@ -181,7 +180,7 @@ define([
 
     ForgeActionButton.prototype.createNamedNode = function(baseId, isMeta) {
         var parentId = this._currentNodeId,
-            newId = this.client.createChild({parentId, baseId}),
+            newId = this.client.createNode({parentId, baseId}),
             basename = 'New' + this.client.getNode(baseId).getAttribute('name'),
             newName = this.getUniqueName(parentId, basename);
 
@@ -189,7 +188,7 @@ define([
         if (!isMeta) {
             newName = newName.substring(0, 1).toLowerCase() + newName.substring(1);
         }
-        this.client.setAttributes(newId, 'name', newName);
+        this.client.setAttribute(newId, 'name', newName);
         return newId;
     };
 
@@ -358,10 +357,24 @@ define([
         var nodeId = this._currentNodeId;
         if (nodeId) {
             this.client.startTransaction(msg);
-            this.client.delMoreNodes([nodeId]);
-            this.client.completeTransaction(msg);
+            this.client.deleteNode(nodeId);
+            this.client.completeTransaction();
         }
     };
 
+    ForgeActionButton.prototype.downloadFromBlob = function(hash) {
+        var name;
+
+        this._blobClient.getMetadata(hash)
+            .then(metadata => {
+                name = metadata.name;
+                return this._blobClient.getObjectAsString(hash);
+            })
+            .then(text => {
+                SaveToDisk.downloadTextAsFile(name, text);
+            })
+            .fail(err => this.logger.error(`Blob download failed: ${err}`));
+    };
+
     return ForgeActionButton;
 });

src/visualizers/panels/ForgeActionButton/PluginConfig.json

@@ -1,21 +0,0 @@
-{
-    "GenerateArchitecture": {
-        "icon": "description",
-        "hotkey": "shift enter",
-        "priority": -1
-    },
-    "ExecutePipeline": {
-        "icon": "play_arrow",
-        "hotkey": "shift enter",
-        "color": "green",
-        "priority": 1
-    },
-    "ImportTorch": {
-        "icon": "import_export",
-        "priority": -1
-    },
-    "GenerateExecFile": {
-        "icon": "play_for_work",
-        "priority": -1
-    }
-}

src/visualizers/panels/ForwardViz/ForwardVizPanel.js

@@ -0,0 +1,67 @@
+/*globals define, _, WebGMEGlobal*/
+/*jshint browser: true*/
+
+define([
+    'js/Constants',
+    'deepforge/globals',
+    'js/PanelBase/PanelBaseWithHeader'
+], function (
+    CONSTANTS,
+    DeepForge,
+    PanelBaseWithHeader
+) {
+    'use strict';
+
+    var ForwardVizPanel;
+
+    ForwardVizPanel = function (layoutManager, params) {
+        var options = {};
+        //set properties from options
+        options[PanelBaseWithHeader.OPTIONS.LOGGER_INSTANCE_NAME] = 'ForwardViz';
+
+        //call parent's constructor
+        PanelBaseWithHeader.apply(this, [options, layoutManager]);
+
+        this._client = params.client;
+
+        //initialize UI
+        this._initialize();
+
+        this.logger.debug('ctor finished');
+    };
+
+    //inherit from PanelBaseWithHeader
+    _.extend(ForwardVizPanel.prototype, PanelBaseWithHeader.prototype);
+
+    ForwardVizPanel.prototype._initialize = function () {
+        this.control = this;
+        this.onActivate();
+    };
+
+    ForwardVizPanel.prototype.selectedObjectChanged = function(nodeId) {
+        if (nodeId === CONSTANTS.PROJECT_ROOT_ID) {
+            DeepForge.places.MyPipelines().then(id => WebGMEGlobal.State.registerActiveObject(id));
+        }
+    };
+
+    /* OVERRIDE FROM WIDGET-WITH-HEADER */
+    /* METHOD CALLED WHEN THE WIDGET'S READ-ONLY PROPERTY CHANGES */
+        //apply parent's onReadOnlyChanged
+    ForwardVizPanel.prototype.onReadOnlyChanged = function() {
+        PanelBaseWithHeader.prototype.onReadOnlyChanged.apply(this, arguments);
+
+    };
+
+    /* * * * * * * * Visualizer life cycle callbacks * * * * * * * */
+    ForwardVizPanel.prototype.destroy = function () {
+        PanelBaseWithHeader.prototype.destroy.call(this);
+    };
+
+    ForwardVizPanel.prototype.onReadOnlyChanged =
+    ForwardVizPanel.prototype.onResize =
+    ForwardVizPanel.prototype.onActivate =
+    ForwardVizPanel.prototype.onDeactivate = function () {
+    };
+
+    return ForwardVizPanel;
+});

src/visualizers/panels/JobEditor/JobEditorPanel.js

@@ -10,11 +10,13 @@ define([
     'panels/TilingViz/TilingVizPanel',
     'panels/OutputViewer/OutputViewerPanel',
     'panels/OperationCodeEditor/OperationCodeEditorPanel',
+    'deepforge/viz/Execute',
     'js/Constants'
 ], function (
     TilingViz,
     OutputViewer,
     OperationCodeEditor,
+    Execute,
     CONSTANTS
 ) {
     'use strict';
@@ -23,11 +25,16 @@ define([
 
     JobEditorPanel = function (layoutManager, params) {
         TilingViz.call(this, layoutManager, params);
+        Execute.call(this, this._client, this.logger);
         this.readOnly = false;
     };
 
     //inherit from PanelBaseWithHeader
-    _.extend(JobEditorPanel.prototype, TilingViz.prototype);
+    _.extend(
+        JobEditorPanel.prototype,
+        Execute.prototype,
+        TilingViz.prototype
+    );
 
     JobEditorPanel.prototype.getPanels = function () {
         if (this.readOnly) {
@@ -88,6 +95,10 @@ define([
             // update the OutputViewer controller
             var i = this._panels.length;
             this._panels[i-1].control.selectedObjectChanged(nodeId);
+            // Check if the job needs to be reconnected
+            if (!this.isReadOnly()) {
+                this.checkJobExecution(node);
+            }
         }
     };
 

src/visualizers/panels/LayerEditor/LayerEditorControl.js

@@ -32,6 +32,8 @@ define([
     LayerEditorControl.prototype._getObjectDescriptor = function (nodeId) {
         var desc = TextEditorControl.prototype._getObjectDescriptor.call(this, nodeId),
             node = this._client.getNode(nodeId),
+            baseId = node.getBaseId(),
+            base = this._client.getNode(baseId),
             hasCode = node.getValidAttributeNames().indexOf('code') > -1,
             template;
 
@@ -50,7 +52,16 @@ define([
         }
 
         if (template) {
-            desc.text = _.template(template)(desc);
+            var baseTorchType = 'nn.Module';
+            // If the base type is 'Criterion', set the base type to nn.Criterion
+            if (base.getAttribute('name') === 'Criterion') {
+                baseTorchType = 'nn.Criterion';
+            }
+
+            desc.text = _.template(template)({
+                name: desc.name,
+                baseTorchType: baseTorchType
+            });
         }
         return desc;
     };
@@ -82,11 +93,11 @@ define([
 
         this._client.startTransaction(msg);
 
-        TextEditorControl.prototype.saveTextFor.call(this, id, text);
-        this._client.setAttributes(id, 'name', layerSchema.name);
+        TextEditorControl.prototype.saveTextFor.call(this, id, text, true);
+        this._client.setAttribute(id, 'name', layerSchema.name);
 
         this._logger.debug(`Setting ctor args to ${ctorAttrs.join(',')}`);
-        this._client.setAttributes(id, Constants.CTOR_ARGS_ATTR, ctorAttrs.join(','));
+        this._client.setAttribute(id, Constants.CTOR_ARGS_ATTR, ctorAttrs.join(','));
 
         types = layerSchema.types || {};
         schema = this.getPointerMeta();
@@ -103,29 +114,29 @@ define([
 
         // Remove old pointers
         node.getPointerNames().filter(ptr => !currentPtrs[ptr])
-            .forEach(ptr => this._client.deleteMetaPointer(id, ptr));
+            .forEach(ptr => this._client.delPointerMeta(id, ptr));
 
         // Remove old attributes
         setterNames = Object.keys(layerSchema.setters);
         _.difference(currentAttrs, ctorAttrs, setterNames)
-            .forEach(attr => this._client.removeAttributeSchema(id, attr));
+            .forEach(attr => this._client.delAttributeMeta(id, attr));
 
         // Add setters
         for (i = setterNames.length; i--;) {
             schema = utils.getSetterSchema(setterNames[i], layerSchema.setters, layerSchema.defaults);
             // Get setter attr schema
             if (schema.hasOwnProperty('default')) {
-                this._client.setAttributes(id, setterNames[i], schema.default);
+                this._client.setAttribute(id, setterNames[i], schema.default);
                 delete schema.default;
             }
             if (types[setterNames[i]]) {
                 schema.type = types[setterNames[i]];
             }
-            this._client.setAttributeSchema(id, setterNames[i], schema);
+            this._client.setAttributeMeta(id, setterNames[i], schema);
         }
 
         ctorAttrs.forEach(attr =>
-            this._client.setAttributeSchema(id, attr, {
+            this._client.setAttributeMeta(id, attr, {
                 type: types[attr] || 'string'
             })
         );

src/visualizers/panels/LogViewer/LogViewerControl.js

@@ -2,9 +2,18 @@
 /*jshint browser: true*/
 
 // This is a read-only view of the 'stdout' attribute for a Job node
+// if the job is running, get the logs from the log-storage
 define([
+    'q',
+    'deepforge/api/JobLogsClient',
+    'js/Constants',
+    'deepforge/Constants',
     'panels/TextEditor/TextEditorControl'
 ], function (
+    Q,
+    JobLogsClient,
+    GME_CONSTANTS,
+    CONSTANTS,
     TextEditorControl
 ) {
 
@@ -19,11 +28,75 @@ define([
 
     _.extend(LogViewerControl.prototype, TextEditorControl.prototype);
 
+    LogViewerControl.prototype.getFullDescriptor = function (id) {
+        var desc = LogViewerControl.prototype._getObjectDescriptor.call(this, id);
+
+        return this._getRunningLogs(id).then(text => {
+            // Use attribute or running log if none
+            desc.text = desc.text || text;
+            return desc;
+        });
+    };
+
+    LogViewerControl.prototype.getUpdatedJobId = function (msg) {
+        // verify that it is the given notification type
+        if (msg.indexOf(CONSTANTS.STDOUT_UPDATE) !== -1) {
+            return msg.replace(/^[^\/]*\//, '');
+        }
+    };
+
+    LogViewerControl.prototype.selectedObjectChanged = function (id) {
+        TextEditorControl.prototype.selectedObjectChanged.call(this, id);
+        // Listen for notifications about updated logs
+        this.removeNotificationHandler();
+        this.notificationHandler = (sender, data) => {
+            var nodeId = this.getUpdatedJobId(data.message);
+            if (nodeId === id) {
+                this._onUpdate(id);
+            }
+        };
+        this._client.addEventListener(GME_CONSTANTS.CLIENT.NOTIFICATION, this.notificationHandler);
+    };
+
+    LogViewerControl.prototype.removeNotificationHandler = function () {
+        // Remove the notifications listener
+        if (this.notificationHandler) {
+            this._client.removeEventListener();
+            this.notificationHandler = null;
+        }
+    };
+
+    LogViewerControl.prototype.destroy = function () {
+        TextEditorControl.prototype.destroy.call(this);
+        this.removeNotificationHandler();
+    };
+
+    LogViewerControl.prototype._onLoad = function (id) {
+        this.getFullDescriptor(id).then(desc => this._widget.addNode(desc));
+    };
+
     LogViewerControl.prototype._onUpdate = function (id) {
         if (id === this._currentNodeId) {
-            TextEditorControl.prototype._onUpdate.call(this, id);
+            this.getFullDescriptor(id).then(desc => this._widget.updateNode(desc));
         }
     };
 
+    LogViewerControl.prototype._getRunningLogs = function (id) {
+        var logManager;
+
+        if (!this._client.getActiveBranchName() || !this._client.getActiveProjectId()) {
+            // Logs are only stored for a given branch
+            return Q().then(() => '');
+        }
+
+        logManager = new JobLogsClient({
+            logger: this._logger,
+            projectId: this._client.getActiveProjectId(),
+            branchName: this._client.getActiveBranchName()
+        });
+
+        return logManager.getLog(id);
+    };
+
     return LogViewerControl;
 });

src/visualizers/panels/MainView/MainViewControl.js

@@ -1,229 +0,0 @@
-/*globals define, WebGMEGlobal*/
-/*jshint browser: true*/
-
-define([
-    'blob/BlobClient',
-    'js/Constants',
-    'js/Utils/GMEConcepts',
-    'js/NodePropertyNames',
-    'deepforge/globals'
-], function (
-    BlobClient,
-    CONSTANTS,
-    GMEConcepts,
-    nodePropertyNames,
-    DeepForge
-) {
-
-    'use strict';
-
-    var MainViewControl;
-
-    MainViewControl = function (options) {
-
-        this._logger = options.logger.fork('Control');
-
-        this._client = options.client;
-
-        // Initialize core collections and variables
-        this._widget = options.widget;
-
-        this._currentNodeId = null;
-        this._embedded = options.embedded;
-
-        this.territory = {};
-        this.ui = {};
-        this._blobClient = new BlobClient({
-            logger: this._logger.fork('BlobClient')
-        });
-
-        this._initWidgetEventHandlers();
-        this._logger.debug('ctor finished');
-    };
-
-    MainViewControl.prototype._initWidgetEventHandlers = function () {
-        this._widget.deleteNode = id => {
-            var node = this._client.getNode(id),
-                baseId = node.getBaseId(),
-                base = this._client.getNode(baseId),
-                baseName = base.getAttribute('name'),
-                name = node.getAttribute('name'),
-                msg = `Deleting ${baseName} "${name}"`;
-
-            this._client.startTransaction(msg);
-            this._client.delMoreNodes([id]);
-            this._client.completeTransaction();
-        };
-
-        this._widget.dataUrlFor = id => {
-            var node = this._client.getNode(id),
-                hash = node.getAttribute('data');
-
-            if (hash) {
-                return this._blobClient.getDownloadURL(hash);
-            } else {
-                return null;
-            }
-        };
-
-        this._widget.toggleEmbeddedPanel = () => this.toggleEmbeddedPanel();
-    };
-
-    /* * * * * * * * Visualizer content update callbacks * * * * * * * */
-    // One major concept here is with managing the territory. The territory
-    // defines the parts of the project that the visualizer is interested in
-    // (this allows the browser to then only load those relevant parts).
-    MainViewControl.prototype.selectedObjectChanged = function (nodeId) {
-        this._logger.debug('activeObject nodeId \'' + nodeId + '\'');
-
-        // Remove current territory patterns
-        this.clearTerritoryRules();
-
-        this._currentNodeId = nodeId;
-
-        if (typeof this._currentNodeId === 'string') {
-            var terrTypes = [
-                /* [type, root dir] */
-                ['arch', 'MyArchitectures'],
-                ['artifact', 'MyArtifacts']
-            ];
-
-            terrTypes.forEach(pair => {
-                var type = pair[0],
-                    dirname = pair[1];
-
-                // Update the territory
-                this.territory[type] = {};
-                DeepForge.places[dirname]().then(id => {
-                    this.territory[type][id] = {children: 1};
-                    this.ui[type] = this._client.addUI(this, this.handleEvents.bind(this, type));
-                    this._client.updateTerritory(this.ui[type], this.territory[type]);
-                });
-            });
-        }
-    };
-
-    MainViewControl.prototype.handleEvents = function (type, events) {
-        var event;
-
-        // Remove the containing dir
-        events = events.filter(e => !this.territory[type][e.eid]);
-        this._logger.debug('_eventCallback \'' + i + '\' items');
-
-        for (var i = events.length; i--;) {
-            event = events[i];
-            switch (event.etype) {
-
-            case CONSTANTS.TERRITORY_EVENT_LOAD:
-                this.onLoad(type, event.eid);
-                break;
-            case CONSTANTS.TERRITORY_EVENT_UPDATE:
-                this._onUpdate(type, event.eid);
-                break;
-            case CONSTANTS.TERRITORY_EVENT_UNLOAD:
-                this._onUnload(event.eid);
-                break;
-            default:
-                break;
-            }
-        }
-
-        this._logger.debug('_eventCallback \'' + events.length + '\' items - DONE');
-    };
-
-    MainViewControl.prototype.onLoad = function(type, id) {
-        // Load a node of the given type
-        var desc = this._getObjectDescriptor(type, id);
-        if (type === 'arch') {
-            this._widget.addArch(desc);
-        } else {  // artifacts
-            this._widget.addArtifact(desc);
-        }
-    };
-
-    // This next function retrieves the relevant node information for the widget
-    MainViewControl.prototype._getArtifactDesc = function (id) {
-        var node = this._client.getNode(id),
-            data = node.getAttribute('data'),
-            desc = this._getBasicDesc(id);
-
-        desc.data = data;
-        return desc;
-    };
-
-    MainViewControl.prototype._getArchDesc =
-    MainViewControl.prototype._getBasicDesc = function (id) {
-        var node = this._client.getNode(id);
-
-        return {
-            id: id,
-            name: node.getAttribute('name')
-        };
-    };
-
-    MainViewControl.prototype._getObjectDescriptor = function (type, id) {
-        return type === 'arch' ?
-            this._getArchDesc(id) :
-            this._getArtifactDesc(id);
-    };
-
-    /* * * * * * * * Node Event Handling * * * * * * * */
-    MainViewControl.prototype._onUpdate = function (type, gmeId) {
-        var description = this._getObjectDescriptor(type, gmeId);
-        this._widget.updateNode(description);
-    };
-
-    MainViewControl.prototype._onUnload = function (gmeId) {
-        this._widget.removeNode(gmeId);
-    };
-
-    MainViewControl.prototype._stateActiveObjectChanged = function (model, activeObjectId) {
-        if (this._currentNodeId !== activeObjectId) {
-            this.selectedObjectChanged(activeObjectId);
-        }
-    };
-
-    /* * * * * * * * Visualizer life cycle callbacks * * * * * * * */
-    MainViewControl.prototype.destroy = function () {
-        this._detachClientEventListeners();
-        this.clearTerritoryRules();
-    };
-
-    MainViewControl.prototype._attachClientEventListeners = function () {
-        this._detachClientEventListeners();
-        if (!this._embedded) {
-            WebGMEGlobal.State.on('change:' + CONSTANTS.STATE_ACTIVE_OBJECT,
-                this._stateActiveObjectChanged, this);
-        }
-    };
-
-    MainViewControl.prototype._detachClientEventListeners = function () {
-        if (!this._embedded) {
-            WebGMEGlobal.State.off('change:' + CONSTANTS.STATE_ACTIVE_OBJECT,
-                this._stateActiveObjectChanged);
-        }
-    };
-
-    MainViewControl.prototype.onActivate = function () {
-        this._attachClientEventListeners();
-
-        if (typeof this._currentNodeId === 'string') {
-            WebGMEGlobal.State.registerSuppressVisualizerFromNode(true);
-            WebGMEGlobal.State.registerActiveObject(this._currentNodeId);
-            WebGMEGlobal.State.registerSuppressVisualizerFromNode(false);
-        }
-    };
-
-    MainViewControl.prototype.clearTerritoryRules = function () {
-        if (Object.keys(this.ui).length) {
-            Object.keys(this.ui).forEach(id =>
-                this._client.removeUI(this.ui[id]));
-        }
-    };
-
-    MainViewControl.prototype.onDeactivate = function () {
-        this._detachClientEventListeners();
-    };
-
-    return MainViewControl;
-});

src/visualizers/panels/MainView/MainViewPanel.js

@@ -1,167 +0,0 @@
-/*globals define, $, _, WebGMEGlobal*/
-/*jshint browser: true*/
-
-// The main panel shows the PipelineIndex w/ a bar on the left for viewing architectures
-// and pipelines
-define([
-    'js/PanelBase/PanelBaseWithHeader',
-    'js/PanelManager/IActivePanel',
-    'widgets/MainView/MainViewWidget',
-    './MainViewControl',
-    'panels/PipelineIndex/PipelineIndexPanel',
-    'panels/ExecutionIndex/ExecutionIndexPanel',
-    'deepforge/globals'
-], function (
-    PanelBaseWithHeader,
-    IActivePanel,
-    MainViewWidget,
-    MainViewControl,
-    PipelineIndexPanel,
-    ExecutionIndexPanel,
-    DeepForge
-) {
-    'use strict';
-
-    var MainViewPanel;
-
-    MainViewPanel = function (layoutManager, params) {
-        var options = {};
-        //set properties from options
-        options[PanelBaseWithHeader.OPTIONS.LOGGER_INSTANCE_NAME] = 'MainViewPanel';
-        options[PanelBaseWithHeader.OPTIONS.FLOATING_TITLE] = true;
-
-        //call parent's constructor
-        PanelBaseWithHeader.apply(this, [options, layoutManager]);
-
-        this._client = params.client;
-        this._embedded = params.embedded;
-
-        //initialize UI
-        this.$nav = $('<div>', {id: 'nav-container'});
-        this.$el.css({padding: 0});
-
-        this.embeddedPanels = [
-            PipelineIndexPanel,
-            ExecutionIndexPanel
-        ];
-        this.nextPanelIndex = 0;
-        this._lm = layoutManager;
-        this._params = params;
-        this.$el.append(this.$nav);
-        this._initialize();
-
-        this.logger.debug('ctor finished');
-    };
-
-    //inherit from PanelBaseWithHeader
-    _.extend(MainViewPanel.prototype, PanelBaseWithHeader.prototype);
-    _.extend(MainViewPanel.prototype, IActivePanel.prototype);
-
-    MainViewPanel.prototype._initialize = function () {
-        //set Widget title
-        this.setTitle('');
-
-        this.widget = new MainViewWidget(this.logger, this.$nav);
-
-        this.control = new MainViewControl({
-            logger: this.logger,
-            client: this._client,
-            embedded: this._embedded,
-            widget: this.widget
-        });
-
-        this.control.toggleEmbeddedPanel = this.toggleEmbeddedPanel.bind(this);
-        var selectedObjectChanged = this.control.selectedObjectChanged;
-        this.control.selectedObjectChanged = id => {
-            this.getEmbeddedNode().then(nodeId =>
-                this.embeddedPanel.control.selectedObjectChanged(nodeId));
-            selectedObjectChanged.call(this.control, id);
-        };
-
-        this.toggleEmbeddedPanel(true);
-        this.onActivate();
-    };
-
-    MainViewPanel.prototype.getEmbeddedNode = function() {
-        if (this.nextPanelIndex === 1) {
-            return DeepForge.places.MyPipelines();
-        } else {
-            return DeepForge.places.MyExecutions();
-        }
-    };
-
-    MainViewPanel.prototype.toggleEmbeddedPanel = function (silent) {
-        var Panel = this.embeddedPanels[this.nextPanelIndex];
-        this.nextPanelIndex = (this.nextPanelIndex + 1) % this.embeddedPanels.length;
-
-        if (this.embeddedPanel) {  // Remove current
-            this.embeddedPanel.destroy();
-            this.$embedded.remove();
-        }
-
-        this.embeddedPanel = new Panel(this._lm, this._params);
-        this.$embedded = this.embeddedPanel.$el;
-        this.$embedded.addClass('main-view-embedded');
-        this.$el.append(this.$embedded);
-
-        // Call on Resize and selectedObjectChanged
-        this.onResize(this.width, this.height);
-        if (!silent) {
-            this.getEmbeddedNode().then(nodeId =>
-                this.embeddedPanel.control.selectedObjectChanged(nodeId));
-        }
-    };
-
-    /* OVERRIDE FROM WIDGET-WITH-HEADER */
-    /* METHOD CALLED WHEN THE WIDGET'S READ-ONLY PROPERTY CHANGES */
-    MainViewPanel.prototype.onReadOnlyChanged = function (isReadOnly) {
-        //apply parent's onReadOnlyChanged
-        PanelBaseWithHeader.prototype.onReadOnlyChanged.call(this, isReadOnly);
-
-    };
-
-    MainViewPanel.prototype.onResize = function (width, height) {
-        var navWidth,
-            embeddedWidth;
-
-        this.logger.debug('onResize --> width: ' + width + ', height: ' + height);
-        this.widget.onWidgetContainerResize(width, height);
-        navWidth = this.widget.width();
-        embeddedWidth = width-navWidth;
-        this.$embedded.css({
-            width: embeddedWidth,
-            height: height,
-            left: navWidth,
-            margin: 'inherit'
-        });
-        this.embeddedPanel.onResize(embeddedWidth, height);
-        this.width = width;
-        this.height = height;
-    };
-
-    /* * * * * * * * Visualizer life cycle callbacks * * * * * * * */
-    MainViewPanel.prototype.destroy = function () {
-        this.control.destroy();
-        this.widget.destroy();
-
-        PanelBaseWithHeader.prototype.destroy.call(this);
-        WebGMEGlobal.KeyboardManager.setListener(undefined);
-        WebGMEGlobal.Toolbar.refresh();
-    };
-
-    MainViewPanel.prototype.onActivate = function () {
-        this.widget.onActivate();
-        this.control.onActivate();
-        WebGMEGlobal.KeyboardManager.setListener(this.widget);
-        WebGMEGlobal.Toolbar.refresh();
-    };
-
-    MainViewPanel.prototype.onDeactivate = function () {
-        this.widget.onDeactivate();
-        this.control.onDeactivate();
-        WebGMEGlobal.KeyboardManager.setListener(undefined);
-        WebGMEGlobal.Toolbar.refresh();
-    };
-
-    return MainViewPanel;
-});