v1.2.0

* Only retrieve member registry from nodes in set. Fixes #998

* WIP #998 Added test case

.codeclimate.yml

@@ -31,6 +31,8 @@ exclude_paths:
 - test/
 - src/common/lua.js
 - src/common/js-yaml.min.js
+- src/visualizers/widgets/Sidebar/lib/
 - src/visualizers/widgets/TextEditor/lib/
 - src/visualizers/widgets/PipelineIndex/styles/PipelineIndex.css
 - src/visualizers/widgets/LineGraph/lib/
+- src/visualizers/widgets/PipelineEditor/klay.js

.npmignore

@@ -0,0 +1,40 @@
+src/worker/tmp
+.env
+*.swp
+*.swo
+**.sass-cache
+# Logs
+logs
+*.log
+
+# Runtime data
+pids
+*.pid
+*.seed
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (http://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directory
+# https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git
+node_modules
+tmp/
+test-tmp/
+blob-local-storage/
+src/seeds/nn/hash.txt
+src/seeds/pipeline/hash.txt
+
+# docs
+images/

README.md

@@ -1,19 +1,22 @@
 [![Release State](https://img.shields.io/badge/state-beta-yellow.svg)](https://img.shields.io/badge/state-beta-yellow.svg)
 [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](./LICENSE)
-[![Build Status](https://travis-ci.org/dfst/deepforge.svg?branch=master)](https://travis-ci.org/dfst/deepforge)
-[![Join the chat at https://gitter.im/dfst/deepforge](https://badges.gitter.im/dfst/deepforge.svg)](https://gitter.im/dfst/deepforge?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
-[![Stories in Ready](https://badge.waffle.io/dfst/deepforge.png?label=ready&title=Ready)](https://waffle.io/dfst/deepforge)
-
-**Notice**: DeepForge is still a work in progress and is also lacking significant documentation! That being said, any contributions and/or feedback is greatly appreciated (and feel free to always ask any questions on the gitter)!
+[![Build Status](https://travis-ci.org/deepforge-dev/deepforge.svg?branch=master)](https://travis-ci.org/deepforge-dev/deepforge)
+[![Join the chat at https://gitter.im/deepforge-dev/deepforge](https://badges.gitter.im/deepforge-dev/deepforge.svg)](https://gitter.im/deepforge-dev/deepforge?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+[![Stories in Ready](https://badge.waffle.io/deepforge-dev/deepforge.png?label=ready&title=Ready)](https://waffle.io/deepforge-dev/deepforge)
 
 # DeepForge
-DeepForge is an open-source visual development environment for deep learning. Currently, it supports Convolutional Neural Networks, RNNs and LSTMs as well as the creation of custom layers. Additional features include:
+DeepForge is an open-source visual development environment for deep learning providing end-to-end support for creating deep learning models. This is achieved through providing the ability to design **architectures**, create training **pipelines**, and then execute these pipelines over a cluster. Using a notebook-esque api, users can get real-time feedback about the status of any of their **executions** including compare them side-by-side in real-time.
+
+![overview](images/overview.png "")
+
+Additional features include:
 - Graphical architecture editor
 - Training/testing pipeline creation
 - Distributed pipeline execution
 - Real-time pipeline feedback
 - Collaborative editing
 - Automatic version control.
+- Facilitates defining custom layers 
 
 ## Quick Start
 Simply run the following command to install deepforge with its dependencies:
@@ -28,13 +31,22 @@ Or, if you already have NodeJS (v6) installed, simply run
 npm install -g deepforge
 ```
 
-Next, start deepforge with `deepforge start`!
+Finally, start deepforge with `deepforge start`and navigate to [http://localhost:8888](http://localhost:8888) to start using DeepForge! For more, detailed instructions, check out the [wiki](https://github.com/dfst/deepforge/wiki/Installation-Guide).
 
-Finally, navigate to [http://localhost:8888](http://localhost:8888) to start using DeepForge! For more, detailed instructions, check out the [wiki](https://github.com/dfst/deepforge/wiki/Installation-Guide).
+**Note**: running deepforge w/ `deepforge start` will also require [MongoDB](https://www.mongodb.com/download-center?jmp=nav#community) to be installed locally.
 
 Also, be sure to check out the other available features of the `deepforge` cli; it can be used to update, manage your torch installation, uninstall deepforge and run individual components!
 
+## Additional Resources
+- [Intro to DeepForge Slides](https://docs.google.com/presentation/d/10_y5O3gHXSATfjHVLJg7dOdrz-tAXNWjlxhJ5SlA0ic/edit?usp=sharing)
+- [wiki](https://github.com/deepforge-dev/deepforge/wiki) containing overview, installation, configuration and developer information
+- [Starter Kit](https://github.com/deepforge-dev/examples/tree/master/starterkit) containing example pipelines demonstrating various deepforge features
+- [Examples](https://github.com/deepforge-dev/examples)
+
+- [Datamodel Developer Slides](https://docs.google.com/presentation/d/1hd3IyUlzW_TIPnzCnE-1pdz00Pw8WaIxYiOW_Hyog-M/edit#slide=id.p)
+
 ## Interested in contributing?
-Contributions are welcome! Either fork the project and submit some PR's or shoot me an email about getting more involved!
+Contributions are welcome! Either fork the project and submit some PR's or shoot me an email about getting more involved! If you have any questions, check out the [wiki](https://github.com/dfst/deepforge/wiki/) or drop me a line on the gitter!
+
 
 Sponsored by [Digital Reasoning](http://www.digitalreasoning.com/)

_config.yml

@@ -0,0 +1 @@
+theme: jekyll-theme-cayman

app.js

@@ -4,11 +4,17 @@
 var gmeConfig = require('./config'),
     webgme = require('webgme'),
     path = require('path'),
+    fs = require('fs'),
     rm_rf = require('rimraf'),
+    gracefulFs = require('graceful-fs'),
     myServer;
 
+process.chdir(__dirname);
 webgme.addToRequireJsPaths(gmeConfig);
 
+// Patch the 'fs' module to fix 'too many files open' error
+gracefulFs.gracefulify(fs);
+
 // Clear seed hash info
 ['nn', 'pipeline'].map(lib => path.join(__dirname, 'src', 'seeds', lib, 'hash.txt'))
     .forEach(file => rm_rf.sync(file));

bin/deepforge

@@ -1,6 +1,7 @@
 #!/usr/bin/env node
 
 var Command = require('commander').Command,
+    tcpPortUsed = require('tcp-port-used'),
     program = new Command(),
     childProcess = require('child_process'),
     rawSpawn = childProcess.spawn,
@@ -8,7 +9,8 @@ var Command = require('commander').Command,
     execSync = childProcess.execSync,
     path = require('path'),
     fs = require('fs'),
-    version = require('../package.json').version,
+    pkgJson = require('../package.json'),
+    version = pkgJson.version,
     exists = require('exists-file'),
     DEFAULT_CONFIG = require('./config.json'),
     merge = require('lodash.merge'),
@@ -117,32 +119,35 @@ var isLocalUri = function(protocol, uri) {
         uri.indexOf(protocol + '://127.0.0.1') === 0;
 };
 
-var checkMongo = function(args, notSilent) {
+var checkMongo = function(args, notSilent, mongoUri) {
     // check the webgme config
-    var gmeConfig = require('../config'),
-        mongoUri = gmeConfig.mongo.uri;
+    var gmeConfig = require('../config');
+
+    mongoUri = mongoUri || gmeConfig.mongo.uri;
 
     if (isLocalUri('mongodb', mongoUri)) {
+        var match = mongoUri.match(/:([0-9]+)/),
+            port = '80';
+
+        if (match) {
+            port = match[1];
+        }
+
         // Make sure mongo is running locally (using pgrep)
         try {
             execSync('pgrep mongod').toString();
             console.log('MongoDB is already running!');
         } catch (e) {  // no pIds
             console.log('Starting MongoDB...');
-            var match = mongoUri.match(/:([0-9]+)/),
-                port = '80';
-
-            if (match) {
-                port = match[1];
-            }
-
             startMongo(args, port, !notSilent);
         }
+        return tcpPortUsed.waitUntilUsed(+port, 100, 1000);
     } else if (notSilent) {
         console.log(`Cannot start remote mongo locally: ${mongoUri}`);
     } else {
         console.log(`Using remote mongo: ${mongoUri}`);
     }
+    return Q();
 };
 
 var startMongo = function(args, port, silent) {
@@ -208,28 +213,34 @@ var installTorch = function() {
         args = `clone https://github.com/torch/distro.git ${tgtDir} --recursive`.split(' ');
 
         return spawn('git', args)
-            .then(code => {
-                if (code !== 0) {
-                    if (code === 128) {
-                        console.error(`${tgtDir} is not empty. ` +
-                            'Please empty it or change the torch directory:\n' +
-                            '\n    deepforge config torch.dir NEW/TORCH/PATH\n');
+            .catch(result => {
+                var error = result.error || result.stderr ||
+                    `Torch install failed with exit code ${result.code}`;
 
-                    }
+                if (result.stderr.includes('unable to access')) {
+                    error = `Could not access the torch repository. Are you ` +
+                        `connected to the internet?\n`;
+                } else if (result.code === 128) {
+                    error = `${tgtDir} is not empty. ` +
+                        'Please empty it or change the torch directory:\n' +
+                        '\n    deepforge config torch.dir NEW/TORCH/PATH\n';
 
-                    throw `Torch install Failed with exit code ${code}`;
-                } else {  // continue installation
-                    process.chdir(tgtDir);
-                    return spawn('bash', ['install-deps'])
-                        .then(() => spawn('bash', ['install.sh'], true))
-                        .then(() => {
-                            storeConfig('torch.dir', tgtDir);
-                            console.log('Installed torch. Please close and ' +
-                                're-open your terminal to use DeepForge w/ ' +
-                                'torch support!');
-                            process.exit(0);
-                        });
                 }
+
+                console.error(error);
+                process.exit(result.code);
+            })
+            .then((code, stderr) => {
+                process.chdir(tgtDir);
+                return spawn('bash', ['install-deps'])
+                    .then(() => spawn('bash', ['install.sh'], true))
+                    .then(() => {
+                        storeConfig('torch.dir', tgtDir);
+                        console.log('Installed torch. Please close and ' +
+                            're-open your terminal to use DeepForge w/ ' +
+                            'torch support!');
+                        process.exit(0);
+                    });
             });
     } else {
         return Q();
@@ -242,21 +253,32 @@ var spawn = function(cmd, args, opts) {
         spawnOpts = typeof opts === 'object' ? opts : null,
         forwardStdin = opts === true,
         isOpen = true,
+        stderr = '',
         err;
 
     args = args || [];
     job = spawnOpts ? rawSpawn(cmd, args, spawnOpts) : rawSpawn(cmd, args);
     job.stdout.on('data', data => process.stdout.write(data));
-    job.stderr.on('data', data => process.stderr.write(data));
+    job.stderr.on('data', data => {
+        stderr += data;
+        process.stderr.write(data);
+    });
+
     job.on('close', code => {
         isOpen = false;
-        if (err) {
-            deferred.reject(err, code);
+        if (err || code !== 0) {
+            deferred.reject({
+                code: code,
+                stderr: stderr,
+                error: err
+            });
         } else {
             deferred.resolve(code);
         }
     });
-    job.on('error', e => err = e);
+    job.on('error', e => {
+        err = e;
+    });
 
     if (forwardStdin) {
         process.stdin.on('data', data => {
@@ -276,42 +298,49 @@ program.command('start')
     .option('-w, --worker [url]', 'start a worker and connect to given url. Defaults to local deepforge')
     .option('-m, --mongo', 'start MongoDB')
     .action(args => {
-        var main = path.join(__dirname, 'start-local.js');
+        var main = path.join(__dirname, 'start-local.js'),
+            current = Q();
 
         if (args.port) {
             process.env.PORT = args.port;
         }
 
+        if (args.mongo) {
+            current = current.then(() => checkMongo(args, true));
+        }
+
         if (args.server) {
-            checkMongo(args);
-            main = path.join(__dirname, '..', 'app.js');
-            spawn('node', [main]);
+            current = current
+                .then(() => checkMongo(args))
+                .then(() => {
+                    main = path.join(__dirname, '..', 'app.js');
+                    return spawn('node', [main]);
+                });
         }
 
         if (args.worker) {
             if (hasTorch()) {
-                installTorchExtras().then(() => {
-                    main = path.join(__dirname, 'start-worker.js');
-                    if (args.worker !== true) {
-                        spawn('node', [main, args.worker]);
-                    } else {
-                        spawn('node', [main]);
-                    }
-                });
+                current
+                    .then(() => installTorchExtras())
+                    .then(() => {
+                        main = path.join(__dirname, 'start-worker.js');
+                        if (args.worker !== true) {
+                            spawn('node', [main, args.worker]);
+                        } else {
+                            spawn('node', [main]);
+                        }
+                    });
             } else {
                 installTorch();
             }
         }
 
-        if (args.mongo) {
-            checkMongo(args, true);
-        }
-
         if (!args.server && !args.worker && !args.mongo) {
             // Starting everything
-            checkMongo(args);
+            current = current.then(() => checkMongo(args));
             if (hasTorch()) {
-                installTorchExtras().then(() => spawn('node', [main]));
+                current.then(() => installTorchExtras())
+                    .then(() => spawn('node', [main]));
             } else {
                 installTorch();
             }
@@ -334,7 +363,7 @@ program
         if (!args.torch || args.server) {
 
             if (args.git) {
-                pkg = 'dfst/deepforge';
+                pkg = pkgJson.repository.url;
             } else {
                 // Check the version
                 try {
@@ -447,12 +476,17 @@ program
         }
     });
 
+// extensions
+program
+    .command('extensions <command>', 'Manage deepforge extensions');
+
 module.exports = function(cmd) {
     var cmds = cmd.split(/\s+/).filter(w => !!w);
     cmds.unshift('./bin/deepforge');
     cmds.unshift('node');
     program.parse(cmds);
 };
+module.exports.checkMongo = checkMongo;
 
 if (require.main === module) {
     program.parse(process.argv);

bin/deepforge-extensions

@@ -0,0 +1,67 @@
+#!/usr/bin/env node
+
+var Command = require('commander').Command,
+    program = new Command(),
+    extender = require('../utils/extender');
+
+// Supported commands
+//  - add
+//  - remove
+//  - list
+//  - update
+program
+    .command('add <project>')
+    .description('Add an extension to deepforge')
+    .option('-n, --name <name>', 'Project name (if different from <project>)')
+    .action(project => {
+        console.log('loading extension from: ' + project);
+        extender.install(project)
+            .then(extConfig => 
+                console.log(`The ${extConfig.name} extension has been added to deepforge.`))
+            .fail(err => {
+                console.error('Could not install extension:\n');
+                console.error(err);
+                process.exit(1);
+            });
+    });
+
+program
+    .command('remove <name>').alias('rm')
+    .description('Remove an extension from deepforge')
+    .action(name => {
+        try {
+            extender.uninstall(name);
+            console.log(`${name} has been successfully removed!`);
+        } catch (e) {
+            console.error('Could not remove extension:');
+            console.error(e);
+            process.exit(1);
+        }
+    });
+
+program
+    .command('list').alias('ls')
+    .description('List installed deepforge extensions')
+    .action(() => {
+        var allExtConfigs = extender.getExtensionsConfig(),
+            types = Object.keys(allExtConfigs),
+            hasContents = false,
+            names;
+
+        for (var i = types.length; i--;) {
+            names = Object.keys(allExtConfigs[types[i]]);
+            if (names.length) {
+                hasContents = true;
+                console.log(types[i]);
+                for (var j = names.length; j--;) {
+                    console.log(`  ${names[j]}`);
+                }
+            }
+        }
+
+        if (!hasContents) {
+            console.log('No installed extensions');
+        }
+    });
+
+program.parse(process.argv);

bin/start-local.js

@@ -13,8 +13,8 @@ if (gmeConfig.blob.type === 'FS') {
 }
 
 process.env.NODE_ENV = 'local';
-execJob = spawn('npm', [
-    'start'
+execJob = spawn('node', [
+    path.join(__dirname, '..', 'app.js')
 ], env);
 execJob.stdout.pipe(process.stdout);
 execJob.stderr.pipe(process.stderr);

bin/start-worker.js

@@ -6,8 +6,7 @@ var path = require('path'),
     spawn = childProcess.spawn,
     rm_rf = require('rimraf'),
     projectConfig = require(__dirname + '/../config'),
-    executorSrc = path.join(__dirname, '..', 'node_modules', 'webgme', 'src',
-        'server', 'middleware', 'executor', 'worker'),
+    executorSrc = path.join(__dirname, '..', 'node_modules', '.bin', 'webgme-executor-worker'),
     id = Date.now(),
     workerRootPath = process.env.DEEPFORGE_WORKER_DIR || path.join(__dirname, '..', 'src', 'worker'),
     workerPath = path.join(workerRootPath, `worker_${id}`),
@@ -60,7 +59,7 @@ var startExecutor = function() {
 
     // Start the executor
     var execJob = spawn('node', [
-        'node_worker.js',
+        executorSrc,
         workerConfigPath,
         workerTmp
     ]);
@@ -74,20 +73,13 @@ var createConfigJson = function() {
 
     if (process.argv.length > 2) {
         address = process.argv[2];
+        if (!/^https?:\/\//.test(address)) {
+            address = 'http://' + address;
+        }
     }
 
     config[address] = {};
     fs.writeFile(workerConfigPath, JSON.stringify(config), startExecutor);
 };
 
-process.chdir(executorSrc);
-
-fs.mkdir(workerTmp, function() {
-    // npm install in this directory
-    var npmInstall = spawn('npm', ['install']);
-    npmInstall.stdout.pipe(process.stdout);
-    npmInstall.stderr.pipe(process.stderr);
-    npmInstall.on('close', function() {
-        createConfigJson();
-    });
-});
+fs.mkdir(workerTmp, createConfigJson);

config/components.json

@@ -2,13 +2,29 @@
     "AutoViz": {
         "preloadIds": [
             "ArchEditor",
+            "ArchIndex",
+            "PipelineIndex",
             "PipelineEditor",
             "OperationEditor",
             "ExecutionView"
-        ]
+        ],
+        "visualizerOverrides": {
+            "": "ForwardViz",
+            "MyArtifacts": "ArtifactIndex",
+            "MyArchitectures": "ArchIndex",
+            "MyExecutions": "ExecutionIndex",
+            "MyPipelines": "PipelineIndex"
+        }
+    },
+    "PipelineEditor": {
+        "itemName": "operation"
+    },
+    "ExecutionView": {
+        "itemName": "job"
     },
     "ArchEditor": {
         "hotkeys": "none",
+        "itemName": "layer",
         "LayerColors": {}
     },
     "BreadcrumbHeader": {
@@ -32,10 +48,6 @@
             "ImportTorch": {
                 "icon": "import_export",
                 "priority": -1
-            },
-            "GenerateExecFile": {
-                "icon": "play_for_work",
-                "priority": -1
             }
         }
     },
@@ -43,11 +55,11 @@
         "rootMenuClass": "deepforge-logo",
         "rootDisplayName": "DeepForge"
     },
-    "CHFLayout": {
+    "SidebarLayout": {
         "panels": [
             {
-                "id": "Header",
-                "panel": "BreadcrumbHeader/BreadcrumbHeaderPanel",
+                "id": "WorkerHeader",
+                "panel": "WorkerHeader/WorkerHeaderPanel",
                 "container": "header",
                 "DEBUG_ONLY": false
             },
@@ -63,6 +75,12 @@
                 "container": "center",
                 "DEBUG_ONLY": false
             },
+            {
+                "id": "Sidebar",
+                "panel": "Sidebar/SidebarPanel",
+                "container": "sidebar",
+                "DEBUG_ONLY": false
+            },
             {
                 "id": "ForgeActionButton",
                 "panel": "ForgeActionButton/ForgeActionButton",

config/config.webgme.js

@@ -9,6 +9,7 @@ var config = require('webgme/config/config.default'),
 // The paths can be loaded from the webgme-setup.json
 config.plugin.basePaths.push(__dirname + '/../src/plugins');
 config.plugin.basePaths.push(__dirname + '/../node_modules/webgme-simple-nodes/src/plugins');
+config.visualization.layout.basePaths.push(__dirname + '/../src/layouts');
 config.visualization.layout.basePaths.push(__dirname + '/../node_modules/webgme-chflayout/src/layouts');
 config.visualization.decoratorPaths.push(__dirname + '/../src/decorators');
 config.visualization.decoratorPaths.push(__dirname + '/../node_modules/webgme-easydag/src/decorators');
@@ -32,6 +33,8 @@ config.visualization.panelPaths.push(__dirname + '/../src/visualizers/panels');
 
 
 config.rest.components['execution/logs'] = __dirname + '/../src/routers/JobLogsAPI/JobLogsAPI.js';
+config.rest.components['job/origins'] = __dirname + '/../src/routers/JobOriginAPI/JobOriginAPI.js';
+config.rest.components['execution/pulse'] = __dirname + '/../src/routers/ExecPulse/ExecPulse.js';
 
 // Visualizer descriptors
 config.visualization.visualizerDescriptors.push(__dirname + '/../src/visualizers/Visualizers.json');
@@ -56,7 +59,7 @@ config.requirejsPaths = {
   'widgets/FloatingActionButton': './node_modules/webgme-fab/src/visualizers/widgets/FloatingActionButton'
 };
 
-config.visualization.layout.default = 'CHFLayout';
+config.visualization.layout.default = 'SidebarLayout';
 config.mongo.uri = 'mongodb://127.0.0.1:27017/deepforge';
 validateConfig(config);
 module.exports = config;

install.sh

@@ -42,10 +42,10 @@ detect_profile() {
 detect_profile
 
 set_node_version() {
-    # Install nodejs v6.2.0
-    echo "Installing NodeJS v6.2.0"
-    nvm install v6.2.0
-    nvm alias default v6.2.0
+    # Install nodejs v6.2.1
+    echo "Installing NodeJS v6.2.1"
+    nvm install v6.2.1
+    nvm alias default v6.2.1
 
     # Install npm@2
     npm install npm@2 -g

package.json

@@ -1,33 +1,44 @@
 {
   "name": "deepforge",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/deepforge-dev/deepforge.git"
+  },
   "bin": {
     "deepforge": "./bin/deepforge"
   },
   "scripts": {
-    "start": "node app.js",
+    "start": "./bin/deepforge start",
+    "postinstall": "node utils/reinstall-extensions.js",
     "start-dev": "NODE_ENV=dev node app.js",
     "local": "node ./bin/start-local.js",
     "worker": "node ./bin/start-worker.js",
-    "test": "mkdir ./test-tmp; node ./node_modules/mocha/bin/mocha --recursive test",
-    "watch-test": "./node_modules/nodemon/bin/nodemon.js --exec 'node ./node_modules/mocha/bin/mocha --recursive test'",
+    "test": "mkdir ./test-tmp; mocha --recursive test",
+    "watch-test": "nodemon --exec 'mocha --recursive test'",
     "build-nn": "node ./utils/nn-parser.js"
   },
-  "version": "0.16.0",
+  "version": "1.2.0",
   "dependencies": {
     "commander": "^2.9.0",
     "dotenv": "^2.0.0",
     "exists-file": "^2.1.0",
     "express": "^4.14.0",
+    "graceful-fs": "^4.1.10",
     "lodash.difference": "^4.1.2",
     "lodash.merge": "^4.5.1",
+    "lodash.template": "^4.4.0",
+    "mongodb": "^2.2.10",
     "nodemon": "^1.9.2",
+    "npm": "^4.0.5",
     "q": "1.4.1",
     "rimraf": "^2.4.0",
-    "webgme": "^2.0.0",
-    "webgme-autoviz": "dfst/webgme-autoviz",
+    "tcp-port-used": "^0.1.2",
+    "webgme": "^2.7.1",
+    "webgme-autoviz": "^2.2.0",
     "webgme-breadcrumbheader": "^2.1.1",
     "webgme-chflayout": "^2.0.0",
     "webgme-easydag": "dfst/webgme-easydag",
+    "webgme-executor-worker": "^1.0.1",
     "webgme-fab": "dfst/webgme-fab",
     "webgme-simple-nodes": "^2.1.0"
   },

src/common/Constants.js

@@ -1,31 +1,58 @@
 /* globals define */
-define({
-    LINE_OFFSET: 'lineOffset',
+(function(root, factory){
+    if(typeof define === 'function' && define.amd) {
+        define([], function(){
+            return factory();
+        });
+    } else if(typeof module === 'object' && module.exports) {
+        module.exports = factory();
+    } else {
+        root.CONSTANTS = factory();
+    }
+}(this, function() {
+    return {
+        CONTAINED_LAYER_SET: 'addLayers',
+        CONTAINED_LAYER_INDEX: 'index',
 
-    // DeepForge metadata creation in dist execution
-    START_CMD: 'deepforge-cmd',
+        LINE_OFFSET: 'lineOffset',
+        DISPLAY_COLOR: 'displayColor',
 
-    IMAGE: {  // all prefixed w/ 'IMG' for simple upload detection
-        PREFIX: 'IMG',
-        BASIC: 'IMG-B',
-        CREATE: 'IMG-C',
-        UPDATE: 'IMG-U',
-        NAME: 'IMAGE-N'  // No upload required
-    },
+        // DeepForge metadata creation in dist execution
+        START_CMD: 'deepforge-cmd',
 
-    GRAPH_CREATE: 'GRAPH',
-    GRAPH_PLOT: 'PLOT',
-    GRAPH_CREATE_LINE: 'LINE',
+        IMAGE: {  // all prefixed w/ 'IMG' for simple upload detection
+            PREFIX: 'IMG',
+            BASIC: 'IMG-B',
+            CREATE: 'IMG-C',
+            UPDATE: 'IMG-U',
+            NAME: 'IMAGE-N'  // No upload required
+        },
 
-    // Code Generation Constants
-    CTOR_ARGS_ATTR: 'ctor_arg_order',
+        GRAPH_CREATE: 'GRAPH',
+        GRAPH_PLOT: 'PLOT',
+        GRAPH_CREATE_LINE: 'LINE',
+        GRAPH_LABEL_AXIS: {
+            X: 'X',
+            Y: 'Y'
+        },
 
-    // Operation types
-    OP: {
-        INPUT: 'Input',
-        OUTPUT: 'Output'
-    },
+        // Code Generation Constants
+        CTOR_ARGS_ATTR: 'ctor_arg_order',
 
-    // Job stdout update
-    STDOUT_UPDATE: 'stdout_update'
-});
+        // Operation types
+        OP: {
+            INPUT: 'Input',
+            OUTPUT: 'Output'
+        },
+
+        // Heartbeat constants (ExecPulse router)
+        PULSE: {
+            DEAD: 0,
+            ALIVE: 1,
+            DOESNT_EXIST: 2
+        },
+
+        // Job stdout update
+        STDOUT_UPDATE: 'stdout_update'
+    };
+}));

src/common/LayerParser.js

@@ -265,6 +265,7 @@
     var findTorchClass = function(ast){ 
         var torchClassArgs,  // args for `torch.class(...)`
             name = '',
+            alias,
             baseType,
             params,
             setters = {},
@@ -283,6 +284,7 @@
                     name = torchClassArgs[0];
                     if(name !== ''){
                         name = name.replace('nn.', '');
+                        alias = func.names[0] || name;
                         if (torchClassArgs.length > 1) {
                             baseType = torchClassArgs[1].replace('nn.', '');
                         }
@@ -302,7 +304,7 @@
                 attrName;
 
             // Record the setter functions
-            if (isSetterMethod(curr, parent, name)) {
+            if (isSetterMethod(curr, parent, alias)) {
                 firstLine = curr.block.stats[0];
                 // just use the attribute attrName for now...
                 attrName = getSettingAttrName(firstLine);
@@ -316,7 +318,7 @@
                 } else {
                     setters[attrName] = schema;
                 }
-            } else if (isInitFn(curr, name)) {  // Record the defaults
+            } else if (isInitFn(curr, alias)) {  // Record the defaults
                 paramDefs = getAttrsAndVals(curr);
                 attrDefs = getClassAttrDefs(curr);
                 types = inferParamTypes(curr, paramDefs);

src/common/api/APIClient.js

@@ -0,0 +1,75 @@
+/*globals define*/
+define([
+    'q',
+    'superagent'
+], function(
+    Q,
+    superagent
+) {
+    'use strict';
+
+    // Wrap the ability to read, update, and delete logs using the JobLogsAPI
+    var APIClient = function(params) {
+        params = params || {};
+
+        this.logger = this.logger || params.logger.fork('APIClient');
+
+        // Get the server url
+        this.token = params.token;
+        this.origin = this._getServerUrl(params);
+        this.relativeUrl = this.relativeUrl || '';
+        this.url = this.origin + this.relativeUrl;
+
+        this.logger.debug(`Setting url to ${this.url}`);
+
+        this.branch = params.branchName;
+        this.project = params.projectId;
+        this._modifiedJobs = [];
+
+        this.logger.debug(`Using <project>:<branch>: "${this.project}"/"${this.branch}"`);
+        this.logger.info('ctor finished');
+    };
+
+    APIClient.prototype._getServerUrl = function(params) {
+        if (typeof window !== 'undefined') {
+            return window.location.origin;
+        }
+
+        // If not in browser, set using the params
+        var server = params.server || '127.0.0.1',
+            port = params.port || '80',
+            protocol = params.httpsecure ? 'https' : 'http';  // default is http
+
+        return params.origin || `${protocol}://${server}:${port}`;
+    };
+
+    APIClient.prototype.getUrl = function() {
+        return this.url;
+    };
+
+    APIClient.prototype._request = function(method, jobId, content) {
+        var deferred = Q.defer(),
+            req = superagent[method](this.getUrl(jobId));
+
+        this.logger.debug(`sending ${method} request to ${this.getUrl(jobId)}`);
+        if (this.token) {
+            req.set('Authorization', 'Bearer ' + this.token);
+        }
+
+        if (content) {
+            req = req.send(content);
+        }
+
+        req.end((err, res) => {
+            if (err || res.status > 399) {
+                return deferred.reject(res || err);
+            }
+
+            return deferred.resolve(res);
+        });
+
+        return deferred.promise;
+    };
+
+    return APIClient;
+});

src/common/api/ExecPulseClient.js

@@ -0,0 +1,44 @@
+/* globals define */
+define([
+    './APIClient'
+], function(
+    APIClient
+) {
+    'use strict';
+
+    var ExecPulseClient = function(params) {
+        this.relativeUrl = '/execution/pulse/';
+        this.logger = params.logger.fork('ExecPulseClient');
+        APIClient.call(this, params);
+    };
+
+    ExecPulseClient.prototype = Object.create(APIClient.prototype);
+
+    ExecPulseClient.prototype.getUrl = function(hash) {
+        return this.url + hash;
+    };
+
+    // - update the heartbeat
+    // - check the heartbeat
+    // - delete the heartbeat
+    ExecPulseClient.prototype.update = function(hash) {
+        return this._request('post', hash)
+            .catch(err => {
+                throw err.text || err;
+            });
+    };
+
+    ExecPulseClient.prototype.check = function(hash) {
+        return this._request('get', hash)
+            .then(res => JSON.parse(res.text))
+            .catch(err => {
+                throw err.text || err;
+            });
+    };
+
+    ExecPulseClient.prototype.clear = function(hash) {
+        return this._request('delete', hash);
+    };
+
+    return ExecPulseClient;
+});

src/common/api/JobLogsClient.js

@@ -1,51 +1,38 @@
 /* globals define */
 define([
+    './APIClient',
     'q',
     'superagent'
 ], function(
+    APIClient,
     Q,
     superagent
 ) {
     'use strict';
 
     // Wrap the ability to read, update, and delete logs using the JobLogsAPI
+    var METADATA_FIELDS = [
+        'lineCount'
+    ];
     var JobLogsClient = function(params) {
         params = params || {};
 
-        this.logger = params.logger.fork('JobLogsClient');
-
-        // Get the server url
-        this.token = params.token;
-        this.origin = this._getServerUrl(params);
         this.relativeUrl = '/execution/logs';
-        this.url = this.origin + this.relativeUrl;
-
-        this.logger.debug(`Setting url to ${this.url}`);
+        this.logger = params.logger.fork('JobLogsClient');
+        APIClient.call(this, params);
 
         // Get the project, branch name
         if (!(params.branchName && params.projectId)) {
             throw Error('"branchName" and "projectId" required');
         }
-        this.branch = params.branchName;
-        this.project = params.projectId;
+
         this._modifiedJobs = [];
 
         this.logger.debug(`Using <project>:<branch>: "${this.project}"/"${this.branch}"`);
         this.logger.info('ctor finished');
     };
 
-    JobLogsClient.prototype._getServerUrl = function(params) {
-        if (typeof window !== 'undefined') {
-            return window.location.origin;
-        }
-
-        // If not in browser, set using the params
-        var server = params.server || '127.0.0.1',
-            port = params.port || '80',
-            protocol = params.httpsecure ? 'https' : 'http';  // default is http
-
-        return params.origin || `${protocol}://${server}:${port}`;
-    };
+    JobLogsClient.prototype = Object.create(APIClient.prototype);
 
     // This method could be optimized - it could make a log of requests
     JobLogsClient.prototype.fork = function(forkName) {
@@ -79,53 +66,54 @@ define([
     };
 
     JobLogsClient.prototype.getUrl = function(jobId) {
+        var url = this.url;
+            
+        if (typeof jobId !== 'string') {
+            url = this.url + jobId.route;
+            jobId = jobId.jobId;
+        }
+
         return [
-            this.url,
+            url,
             encodeURIComponent(this.project),
             encodeURIComponent(this.branch),
             encodeURIComponent(jobId)
         ].join('/');
     };
 
-    JobLogsClient.prototype._logRequest = function(method, jobId, content) {
-        var deferred = Q.defer(),
-            req = superagent[method](this.getUrl(jobId));
-
-        this.logger.info(`sending ${method} request to ${this.getUrl(jobId)}`);
-        if (this.token) {
-            req.set('Authorization', 'Bearer ' + this.token);
-        }
-
-        if (content) {
-            req = req.send(content);
-        }
-
-        req.end((err, res) => {
-            if (err || res.status > 399) {
-                return deferred.reject(err || res.status);
-            }
-
-            return deferred.resolve(res);
-        });
-
-        return deferred.promise;
+    var hasRequiredFields = function(md) {
+        return METADATA_FIELDS.reduce((passing, nextField) => {
+            return passing && md.hasOwnProperty(nextField);
+        }, true);
     };
 
-    JobLogsClient.prototype.appendTo = function(jobId, text) {
+    JobLogsClient.prototype.appendTo = function(jobId, text, metadata) {
         this._modifiedJobs.push(jobId);
         this.logger.info(`Appending logs to ${jobId}`);
-        return this._logRequest('patch', jobId, {patch: text});
+
+        if (metadata && !hasRequiredFields(metadata)) {
+            throw Error(`Required metadata fields: ${METADATA_FIELDS.join(', ')}`);
+        }
+        metadata = metadata || {};
+        metadata.patch = text;
+        return this._request('patch', jobId, metadata);
     };
 
     JobLogsClient.prototype.getLog = function(jobId) {
         this.logger.info(`Getting logs for ${jobId}`);
-        return this._logRequest('get', jobId)
+        return this._request('get', jobId)
             .then(res => res.text);
     };
 
     JobLogsClient.prototype.deleteLog = function(jobId) {
         this.logger.info(`Deleting logs for ${jobId}`);
-        return this._logRequest('delete', jobId);
+        return this._request('delete', jobId);
+    };
+
+    JobLogsClient.prototype.getMetadata = function(jobId) {
+        this.logger.info(`Getting line count for ${jobId}`);
+        return this._request('get', {jobId: jobId, route: '/metadata'})
+            .then(res => JSON.parse(res.text));
     };
 
     return JobLogsClient;

src/common/api/JobOriginClient.js

@@ -0,0 +1,60 @@
+/* globals define */
+define([
+    './APIClient'
+], function(
+    APIClient
+) {
+    'use strict';
+
+    var JobOriginClient = function(params) {
+        this.relativeUrl = '/job/origins/';
+        this.logger = params.logger.fork('JobOriginClient');
+        APIClient.call(this, params);
+    };
+
+    JobOriginClient.prototype = Object.create(APIClient.prototype);
+
+    // - Record the origin
+    // - Look up the origin
+    // - Delete record
+    JobOriginClient.prototype.getUrl = function(hash) {
+        return this.url + hash;
+    };
+
+    JobOriginClient.prototype.record = function(hash, info) {
+        var jobInfo = {
+            hash: hash,
+            nodeId: info.nodeId,
+            job: info.job,
+            project: info.project || this.project,
+            branch: info.branch || this.branch,
+            execution: info.execution
+        };
+
+        return this._request('post', hash, jobInfo)
+            .catch(err => {
+                throw err.text || err;
+            });
+    };
+
+    JobOriginClient.prototype.getOrigin = function(hash) {
+        return this._request('get', hash)
+            .then(res => JSON.parse(res.text))
+            .catch(res => {
+                if (res.status && res.status === 404) {
+                    return null;
+                }
+                throw res;
+            });
+    };
+
+    JobOriginClient.prototype.fork = function(hash, forkName) {
+        return this._request('patch', hash, {branch: forkName});
+    };
+
+    JobOriginClient.prototype.deleteRecord = function(hash) {
+        return this._request('delete', hash);
+    };
+
+    return JobOriginClient;
+});

src/common/globals.js

@@ -57,7 +57,7 @@ define([
     };
 
     var createNamedNode = function(baseId, parentId, isMeta) {
-        var newId = client.createChild({parentId, baseId}),
+        var newId = client.createNode({parentId, baseId}),
             baseNode = client.getNode(baseId),
             basename = 'New' + baseNode.getAttribute('name'),
             newName = getUniqueName(parentId, basename);
@@ -72,7 +72,7 @@ define([
             client.setRegistry(newId, 'isAbstract', false);
         }
 
-        client.setAttributes(newId, 'name', newName);
+        client.setAttribute(newId, 'name', newName);
         return newId;
     };
 
@@ -83,7 +83,9 @@ define([
             exists = {},
             i = 2;
 
-        children.forEach(child => exists[child.getAttribute('name')] = true);
+        children
+            .filter(child => child !== null)
+            .forEach(child => exists[child.getAttribute('name')] = true);
 
         while (exists[name]) {
             name = basename + '_' + i;
@@ -191,7 +193,7 @@ define([
                 .getId();
 
         // Look up the parent container
-        DeepForge.places[placeName]().then(parentId => {
+        return DeepForge.places[placeName]().then(parentId => {
 
             client.startTransaction(msg);
             newId = createNamedNode(baseId, parentId, !!metasheetName);
@@ -265,14 +267,12 @@ define([
         }
 
         dataBaseId = dataBase.getId();
-        dataTypes = metanodes.filter(n => client.isTypeOf(n.getId(), dataBaseId))
+        dataTypes = metanodes.filter(n => n.isTypeOf(dataBaseId))
             .filter(n => !n.getRegistry('isAbstract'))
             .map(node => node.getAttribute('name'));
 
-        //this.logger.info(`Found ${dataTypes.length} data types`);
-
-        // Add the target type to the pluginMetadata... hacky :/
-        var metadata = WebGMEGlobal.allPluginsMetadata[UPLOAD_PLUGIN], 
+        // Add the target type to the pluginMetadata...
+        var metadata = WebGMEGlobal.allPluginsMetadata[UPLOAD_PLUGIN],
             config = metadata.configStructure
                 .find(opt => opt.name === DATA_TYPE_CONFIG.name);
 
@@ -297,7 +297,8 @@ define([
     };
 
     DeepForge.last = {};
-    DeepForge.create  = {};
+    DeepForge.create = {};
+    DeepForge.register = {};
     instances.forEach(type => {
         DeepForge.create[type] = function() {
             return createNew.call(null, type);
@@ -308,6 +309,10 @@ define([
         DeepForge.create[type] = function() {
             return createNew.call(null, type, type);
         };
+        DeepForge.register[type] = function(id) {
+            // Add the given element to the metasheet!
+            return addToMetaSheet(id, type);
+        };
     });
 
     DeepForge.create.Layer = createCustomLayer;

src/common/lua.js

@@ -2565,7 +2565,7 @@ function LuaContext(){
             case 'number':
                 return c;
             case 'string':
-                return parseInt(s) || dummy0;
+                return parseInt(c) || dummy0;
             default:
                 if (c == dummy0){
                     return c;

src/common/plugin/LocalExecutor.js

@@ -148,11 +148,14 @@ define([
                 // get the input node
                 if (dataNodes.length !== 0) {
                     var newNodes = this.core.copyNodes(dataNodes, parentNode),
-                        newName = this.core.getOwnAttribute(node, 'saveName');
+                        newName = this.core.getOwnAttribute(node, 'saveName'),
+                        createdAt = Date.now();
+
                     if (newName) {
-                        newNodes.forEach(node =>
-                            this.setAttribute(node, 'name', newName)
-                        );
+                        newNodes.forEach(node => {
+                            this.setAttribute(node, 'name', newName);
+                            this.setAttribute(node, 'createdAt', createdAt);
+                        });
                     }
                     var hashes = dataNodes.map(n => this.getAttribute(n, 'data'));
                     this.logger.info(`saving hashes: ${hashes.map(h => `"${h}"`)}`);

src/common/viz/Buttons.js

@@ -1,8 +1,10 @@
 /*globals define, WebGMEGlobal*/
 define([
+    'deepforge/globals',
     'widgets/EasyDAG/Buttons',
     'widgets/EasyDAG/Icons'
 ], function(
+    DeepForge,
     EasyDAGButtons,
     Icons
 ) {
@@ -57,9 +59,79 @@ define([
         return n && n.getBaseId();
     };
 
+    var CloneAndEdit = function(params) {
+        GoToBase.call(this, params);
+    };
+
+    CloneAndEdit.prototype = Object.create(GoToBase.prototype);
+    CloneAndEdit.prototype.BTN_CLASS = 'clone-and-edit';
+
+    CloneAndEdit.prototype._render = function() {
+        var lineRadius = GoToBase.SIZE - GoToBase.BORDER,
+            btnColor = '#a5d6a7';
+
+        if (this.disabled) {
+            btnColor = '#e0e0e0';
+        }
+
+        this.$el
+            .append('circle')
+            .attr('r', GoToBase.SIZE)
+            .attr('fill', btnColor);
+
+        // Show the 'code' icon
+        Icons.addIcon('code', this.$el, {
+            radius: lineRadius
+        });
+    };
+
+    CloneAndEdit.prototype._onClick = function(item) {
+        var node = client.getNode(item.id),
+            baseId = node && node.getBaseId(),
+            base = baseId && client.getNode(baseId),
+            typeId = base && base.getBaseId(),
+            type = typeId && client.getNode(typeId),
+            ctrName,
+            typeName,
+            name,
+            newId;
+
+        // Clone the given node's base and change to it
+        if (type) {
+            typeName = type.getAttribute('name');
+            ctrName = `My${typeName}s`;
+            if (DeepForge.places[ctrName]) {
+                DeepForge.places[ctrName]().then(ctrId => {
+                    type = base.getAttribute('name');
+                    client.startTransaction(`Creating new ${typeName} from ${item.name}`);
+                    newId = client.copyNode(baseId, ctrId);
+                    name = node.getAttribute('name');
+                    client.setAttribute(newId, 'name', 'Copy of ' + name);
+                    DeepForge.register[typeName](newId);
+
+                    client.completeTransaction();
+                    WebGMEGlobal.State.registerActiveObject(newId);
+                });
+            }
+        } else {
+            this._logger.warn('Could not find the base node!');
+        }
+    };
+
+    var Insert = function(params) {
+        EasyDAGButtons.ButtonBase.call(this, params);
+    };
+
+    Insert.prototype = Object.create(EasyDAGButtons.Add.prototype);
+    Insert.prototype._onClick = function(item) {
+        this.onInsertButtonClicked(item);
+    };
+
     return {
         DeleteOne: EasyDAGButtons.DeleteOne,
-        GoToBase: GoToBase
+        GoToBase: GoToBase,
+        CloneAndEdit: CloneAndEdit,
+        Insert: Insert
     };
 });
 

src/common/viz/Execute.js

@@ -3,21 +3,31 @@
 define([
     'q',
     'executor/ExecutorClient',
+    'deepforge/api/ExecPulseClient',
+    'deepforge/api/JobOriginClient',
+    'deepforge/Constants',
     'panel/FloatingActionButton/styles/Materialize'
 ], function(
     Q,
     ExecutorClient,
+    ExecPulseClient,
+    JobOriginClient,
+    CONSTANTS,
     Materialize
 ) {
 
     var Execute = function(client, logger) {
         this.client = this.client || client;
         this.logger = this.logger || logger;
+        this.pulseClient = new ExecPulseClient({
+            logger: this.logger
+        });
         this._executor = new ExecutorClient({
             logger: this.logger.fork('ExecutorClient'),
             serverPort: WebGMEGlobal.gmeConfig.server.port,
             httpsecure: window.location.protocol === 'https:'
         });
+        this.originManager = new JobOriginClient({logger: this.logger});
     };
 
     Execute.prototype.executeJob = function(node) {
@@ -102,6 +112,21 @@ define([
             .fail(err => this.logger.error(`Job cancel failed: ${err}`));
     };
 
+    Execute.prototype._setJobStopped = function(jobId, silent) {
+        if (!silent) {
+            var name = this.client.getNode(jobId).getAttribute('name');
+            this.client.startTransaction(`Stopping "${name}" job`);
+        }
+
+        this.client.delAttribute(jobId, 'jobId');
+        this.client.delAttribute(jobId, 'secret');
+        this.client.setAttribute(jobId, 'status', 'canceled');
+
+        if (!silent) {
+            this.client.completeTransaction();
+        }
+    };
+
     Execute.prototype.stopJob = function(job, silent) {
         var jobId;
 
@@ -109,18 +134,7 @@ define([
         jobId = job.getId();
 
         this.silentStopJob(job);
-
-        if (!silent) {
-            this.client.startTransaction(`Stopping "${name}" job`);
-        }
-
-        this.client.delAttributes(jobId, 'jobId');
-        this.client.delAttributes(jobId, 'secret');
-        this.client.setAttributes(jobId, 'status', 'canceled');
-
-        if (!silent) {
-            this.client.completeTransaction();
-        }
+        this._setJobStopped(jobId, silent);
     };
 
 
@@ -165,14 +179,17 @@ define([
     };
 
     Execute.prototype._stopExecution = function(execNode, inTransaction) {
-        var msg = `Canceling ${execNode.getAttribute('name')} execution`;
+        var msg = `Canceling ${execNode.getAttribute('name')} execution`,
+            jobIds;
 
         if (!inTransaction) {
             this.client.startTransaction(msg);
         }
 
-        this._silentStopExecution(execNode);
-        this.client.setAttributes(execNode.getId(), 'status', 'canceled');
+        jobIds = this._silentStopExecution(execNode);
+
+        this.client.setAttribute(execNode.getId(), 'status', 'canceled');
+        jobIds.forEach(jobId => this._setJobStopped(jobId, true));
 
         if (!inTransaction) {
             this.client.completeTransaction();
@@ -180,11 +197,88 @@ define([
     };
 
     Execute.prototype._silentStopExecution = function(execNode) {
-        var jobIds = execNode.getChildrenIds();
+        var runningJobIds = execNode.getChildrenIds()
+            .map(id => this.client.getNode(id))
+            .filter(job => this.isRunning(job));  // get running jobs
 
-        jobIds.map(id => this.client.getNode(id))
-            .filter(job => this.isRunning(job))  // get running jobs
-            .forEach(job => this.silentStopJob(job));  // stop them
+        runningJobIds.forEach(job => this.silentStopJob(job));  // stop them
+
+        return runningJobIds;
+    };
+
+    // Resuming Executions
+    Execute.prototype.checkJobExecution= function (job) {
+        var pipelineId = job.getParentId(),
+            pipeline = this.client.getNode(pipelineId);
+
+        // First check the parent execution. If it doesn't exist, then check the job
+        return this.checkPipelineExecution(pipeline)
+            .then(tryToStartJob => {
+                if (tryToStartJob) {
+                    return this._checkJobExecution(job);
+                }
+            });
+    };
+
+    Execute.prototype._checkJobExecution = function (job) {
+        var jobId = job.getAttribute('jobId'),
+            status = job.getAttribute('status');
+
+        if (status === 'running' && jobId) {
+            return this.pulseClient.check(jobId)
+                .then(status => {
+                    if (status !== CONSTANTS.PULSE.DOESNT_EXIST) {
+                        return this._onOriginBranch(jobId).then(onBranch => {
+                            if (onBranch) {
+                                this.runExecutionPlugin('ExecuteJob', {
+                                    node: job
+                                });
+                            }
+                        });
+                    } else {
+                        this.logger.warn(`Could not restart job: ${job.getId()}`);
+                    }
+                });
+        }
+        return Q();
+    };
+
+    Execute.prototype._onOriginBranch = function (hash) {
+        return this.originManager.getOrigin(hash)
+            .then(origin => {
+                var currentBranch = this.client.getActiveBranchName();
+                if (origin && origin.branch) {
+                    return origin.branch === currentBranch;
+                }
+                return false;
+            });
+    };
+
+    Execute.prototype.checkPipelineExecution = function (pipeline) {
+        var runId = pipeline.getAttribute('runId'),
+            status = pipeline.getAttribute('status'),
+            tryToStartJob = true;
+
+        if (status === 'running' && runId) {
+            return this.pulseClient.check(runId)
+                .then(status => {
+                    if (status === CONSTANTS.PULSE.DEAD) {
+                        // Check the origin branch
+                        return this._onOriginBranch(runId).then(onBranch => {
+                            if (onBranch) {
+                                this.runExecutionPlugin('ExecutePipeline', {
+                                    node: pipeline
+                                });
+                            }
+                        });
+                    }
+                    // only try to start if the pulse info doesn't exist
+                    tryToStartJob = status === CONSTANTS.PULSE.DOESNT_EXIST;
+                    return tryToStartJob;
+                });
+        } else {
+            return Q().then(() => tryToStartJob);
+        }
     };
 
     return Execute;

src/common/viz/NodePrompter.js

@@ -4,9 +4,11 @@
 // adding a "plus" button for creating new objects in line
 
 define([
+    'deepforge/Constants',
     'q',
     'css!./NodePrompter.css'
 ], function(
+    Constants,
     Q
 ) {
 
@@ -271,12 +273,14 @@ define([
     };
 
     var Container = function(svg, node) {  // used for positioning
+        var colorAttr = node.attributes[Constants.DISPLAY_COLOR];
         this.$el = svg.append('g');
         this.x = 0;
         this.y = 0;
         this.node = node;
         this.decorator = new node.Decorator({
             node: node,
+            color: colorAttr && colorAttr.value,
             parentEl: this.$el
         });
     };

src/common/viz/PipelineControl.js

@@ -65,7 +65,7 @@ define([
 
     PipelineControl.prototype.createNode = function(baseId) {
         var parentId = this._currentNodeId,
-            newNodeId = this._client.createChild({parentId, baseId});
+            newNodeId = this._client.createNode({parentId, baseId});
 
         return newNodeId;
     };
@@ -103,6 +103,10 @@ define([
                 delete desc.attributes.code;
             }
 
+            // Handle the display color
+            desc.displayColor = desc.attributes[CONSTANTS.DISPLAY_COLOR].value;
+            delete desc.attributes[CONSTANTS.DISPLAY_COLOR];
+
         } else if (desc.isConnection) {
             // Set src, dst to siblings and add srcPort, dstPort
             desc.srcPort = desc.src;

src/common/viz/RenameablePanel.js

@@ -49,7 +49,7 @@ define([
 
                 if (!/^\s*$/.test(newValue)) {
                     this._client.startTransaction(msg);
-                    this._client.setAttributes(nodeId, 'name', newValue);
+                    this._client.setAttribute(nodeId, 'name', newValue);
                     this._client.completeTransaction();
                 }
             }

src/common/viz/panels/ThumbnailControl.js

@@ -0,0 +1,41 @@
+/* globals define */
+define([
+    'panels/EasyDAG/EasyDAGControl'
+], function(
+    EasyDAGControl
+) {
+    var ThumbnailControl = function() {
+        EasyDAGControl.apply(this, arguments);
+    };
+
+    ThumbnailControl.prototype = Object.create(EasyDAGControl.prototype);
+
+    ThumbnailControl.prototype._initWidgetEventHandlers = function () {
+        EasyDAGControl.prototype._initWidgetEventHandlers.call(this);
+        this._widget.updateThumbnail = this.updateThumbnail.bind(this);
+    };
+
+    ThumbnailControl.prototype.updateThumbnail = function (svg) {
+        var node = this._client.getNode(this._currentNodeId),
+            name,
+            attrs,
+            currentThumbnail,
+            attrName = 'thumbnail',
+            msg;
+
+        if (node) {  // may have been deleted
+            name = node.getAttribute('name');
+            attrs = node.getValidAttributeNames();
+            currentThumbnail = node.getAttribute(attrName);
+            msg = `Updating pipeline thumbnail for "${name}"`;
+
+            if (attrs.indexOf(attrName) > -1 && currentThumbnail !== svg) {
+                this._client.startTransaction(msg);
+                this._client.setAttribute(this._currentNodeId, attrName, svg);
+                this._client.completeTransaction();
+            }
+        }
+    };
+
+    return ThumbnailControl;
+});

src/common/viz/widgets/Thumbnail.js

@@ -0,0 +1,84 @@
+/* globals define, $, _ */
+define([
+    'widgets/EasyDAG/EasyDAGWidget'
+], function(
+    EasyDAGWidget
+) {
+
+    var ThumbnailWidget = function() {
+        EasyDAGWidget.apply(this, arguments);
+    };
+
+    ThumbnailWidget.prototype = Object.create(EasyDAGWidget.prototype);
+
+    ThumbnailWidget.prototype.addNode = function() {
+        var result = EasyDAGWidget.prototype.addNode.apply(this, arguments);
+
+        this.refreshThumbnail();
+        return result;
+    };
+
+    ThumbnailWidget.prototype.removeNode = function() {
+        var result = EasyDAGWidget.prototype.removeNode.apply(this, arguments);
+
+        this.refreshThumbnail();
+        return result;
+    };
+
+    ThumbnailWidget.prototype._removeConnection = function() {
+        var result = EasyDAGWidget.prototype._removeConnection.apply(this, arguments);
+
+        this.refreshThumbnail();
+        return result;
+    };
+
+    ThumbnailWidget.prototype.addConnection = function() {
+        var result = EasyDAGWidget.prototype.addConnection.apply(this, arguments);
+
+        this.refreshThumbnail();
+        return result;
+    };
+
+    ////////////////////////// Thumbnail updates //////////////////////////
+    ThumbnailWidget.prototype.getSvgDistanceDim = function(dim) {
+        var maxValue = this._getMaxAlongAxis(dim),
+            nodes,
+            minValue;
+
+        nodes = this.graph.nodes().map(id => this.graph.node(id));
+        minValue = nodes.length ? Math.min.apply(null, nodes.map(node => node[dim] || 0)) : 0;
+        return maxValue-minValue;
+    };
+
+    ThumbnailWidget.prototype.getSvgWidth = function() {
+        return this.getSvgDistanceDim('x');
+    };
+
+    ThumbnailWidget.prototype.getSvgHeight = function() {
+        return this.height - 25;
+    };
+
+    ThumbnailWidget.prototype.getViewBox = function() {
+        var maxX = this.getSvgWidth('x'),
+            maxY = this.getSvgHeight('y');
+
+        return `0 0 ${maxX} ${maxY}`;
+    };
+
+    ThumbnailWidget.prototype.refreshThumbnail = _.debounce(function() {
+        // Get the svg...
+        var svg = document.createElement('svg'),
+            group = this.$svg.node(),
+            child;
+
+        svg.setAttribute('viewBox', this.getViewBox());
+        for (var i = 0; i < group.children.length; i++) {
+            child = $(group.children[i]);
+            svg.appendChild(child.clone()[0]);
+        }
+
+        this.updateThumbnail(svg.outerHTML);
+    }, 1000);
+
+    return ThumbnailWidget;
+});

src/decorators/ArtifactOpDecorator/EasyDAG/ArtifactOpDecorator.EasyDAGWidget.js

@@ -40,10 +40,40 @@ define([
     ArtifactOpDecorator.prototype.DECORATOR_ID = DECORATOR_ID;
 
     ArtifactOpDecorator.prototype.getTargetFilterFnFor = function() {
+        var currentNode,
+            pId,
+            peerIds,
+            currentDataId,
+            targetTypeIds = [];
+
+        // Get all connections from this node's data
+        if (this._node.outputs[0]) {
+            currentNode = this.client.getNode(this._node.id);
+            pId = currentNode.getParentId();
+            peerIds = this.client.getNode(pId).getChildrenIds();
+            currentDataId = this._node.outputs[0].id;
+
+            targetTypeIds = peerIds.map(id => this.client.getNode(id))
+                .filter(node => {
+                    var ptr = node.getPointer('src');
+                    return ptr.to === currentDataId;
+                })
+
+                // Get the target data types
+                .map(node => this.client.getNode(node.getPointer('dst').to).getMetaTypeId());
+        }
+
         return id => {
             var node = this.client.getNode(id),
-                isMetaTgt = node.getId() === node.getMetaTypeId();
-            return isMetaTgt === this.castOpts.metaTgt;
+                isMetaTgt = node.getId() === node.getMetaTypeId(),
+                isValidType = true;
+
+            // make sure it is a type of the target types
+            isValidType = targetTypeIds.reduce((passing, typeId) => {
+                return passing && node.isTypeOf(typeId);
+            }, true);
+
+            return isValidType && isMetaTgt === this.castOpts.metaTgt;
         };
     };
 
@@ -57,13 +87,13 @@ define([
             this.client.startTransaction(`Removing output of ${this.name}`);
             this.client.delPointer(this._node.id, name);
             if (outputId) {
-                this.client.delAttributes(outputId, 'data');
+                this.client.delAttribute(outputId, 'data');
             }
             this.client.completeTransaction();
         } else if (name === this.castOpts.ptr) {  // set the casted value
             this.client.startTransaction(`Setting output of ${this.name} to ${to}`);
             this.castOutputType(to);
-            this.client.makePointer(this._node.id, name, to);
+            this.client.setPointer(this._node.id, name, to);
             this.client.completeTransaction();
         } else {
             DecoratorBase.prototype.savePointer.call(this, name, to);

src/decorators/ContainerLayerDecorator/ContainerLayerDecorator.js

@@ -0,0 +1,39 @@
+/*globals define, _*/
+/*jshint browser: true, camelcase: false*/
+
+define([
+    'js/Decorators/DecoratorBase',
+    './EasyDAG/ContainerLayerDecorator.EasyDAGWidget'
+], function (
+    DecoratorBase,
+    ContainerLayerDecoratorEasyDAGWidget
+) {
+
+    'use strict';
+
+    var ContainerLayerDecorator,
+        __parent__ = DecoratorBase,
+        __parent_proto__ = DecoratorBase.prototype,
+        DECORATOR_ID = 'ContainerLayerDecorator';
+
+    ContainerLayerDecorator = function (params) {
+        var opts = _.extend({loggerName: this.DECORATORID}, params);
+
+        __parent__.apply(this, [opts]);
+
+        this.logger.debug('ContainerLayerDecorator ctor');
+    };
+
+    _.extend(ContainerLayerDecorator.prototype, __parent_proto__);
+    ContainerLayerDecorator.prototype.DECORATORID = DECORATOR_ID;
+
+    /*********************** OVERRIDE DecoratorBase MEMBERS **************************/
+
+    ContainerLayerDecorator.prototype.initializeSupportedWidgetMap = function () {
+        this.supportedWidgetMap = {
+            EasyDAG: ContainerLayerDecoratorEasyDAGWidget
+        };
+    };
+
+    return ContainerLayerDecorator;
+});

src/decorators/ContainerLayerDecorator/EasyDAG/ContainerLayerDecorator.EasyDAGWidget.css

@@ -0,0 +1,25 @@
+.condense .nested-layers {
+  display: none;
+}
+
+.nested-layers .hover-box {
+  stroke: black;
+  stroke-dasharray: 4, 4;
+  stroke-opacity: 0;
+}
+
+.nested-layers .unhovered .button {
+  opacity: 0;
+}
+
+.nested-layers .hovered .button {
+  opacity: 1;
+}
+
+.nested-layers .unhovered .hover-box {
+  stroke-opacity: 0;
+}
+
+.nested-layers .hovered .hover-box {
+  stroke-opacity: 0;
+}

src/decorators/ContainerLayerDecorator/EasyDAG/ContainerLayerDecorator.EasyDAGWidget.js

@@ -0,0 +1,428 @@
+/*globals define, _, */
+/*jshint browser: true, camelcase: false*/
+
+define([
+    'decorators/LayerDecorator/EasyDAG/LayerDecorator.EasyDAGWidget',
+    'js/Constants',
+    'deepforge/Constants',
+    './NestedLayer',
+    'widgets/EasyDAG/Buttons',
+    'css!./ContainerLayerDecorator.EasyDAGWidget.css'
+], function (
+    LayerDecorator,
+    GME_CONSTANTS,
+    CONSTANTS,
+    NestedLayer,
+    Buttons
+) {
+
+    'use strict';
+
+    var ContainerLayerDecorator,
+        ZOOM = 0.8,
+        DECORATOR_ID = 'ContainerLayerDecorator';
+
+    // Container layer nodes need to be able to nest the containedLayers
+    // in order inside of themselves when expanded
+    ContainerLayerDecorator = function (options) {
+        this.nestedLayers = {};
+        LayerDecorator.call(this, options);
+        this.$nested = this.$el.append('g')
+            .attr('class', 'nested-layers');
+
+        // If clicked, deselect the given nested layer
+        this.$el.on('click', () => {
+            if (this.expanded) {
+                Object.keys(this.nestedLayers).forEach(id => {
+                    this.nestedLayers[id].widget.onBackgroundClick();
+                });
+            }
+        });
+        this.onNestedRefresh = _.debounce(this.updateExpand.bind(this), 50);
+
+        // Add event handlers
+        NestedLayer.prototype.addLayerBefore = function(layerId) {
+            var decorator = this._parent,
+                index = decorator._node.containedLayers.indexOf(this.id);
+            return decorator.addLayerAt(layerId, index - 1);
+        };
+
+        NestedLayer.prototype.addLayerAfter = function(layerId) {
+            var decorator = this._parent,
+                index = decorator._node.containedLayers.indexOf(this.id);
+            return decorator.addLayerAt(layerId, index + 1);
+        };
+
+        NestedLayer.prototype.isLast = function() {
+            var index = this._parent._node.containedLayers.length - 1;
+            return this._parent._node.containedLayers[index] === this.id;
+        };
+
+        NestedLayer.prototype.isFirst = function() {
+            return this._parent._node.containedLayers[0] === this.id;
+        };
+
+        NestedLayer.prototype.moveLayerForward = function() {
+            return this.moveLayer(true);
+        };
+
+        NestedLayer.prototype.moveLayerBackward = function() {
+            return this.moveLayer();
+        };
+
+        NestedLayer.prototype.moveLayer = function(forward) {
+            var decorator = this._parent,
+                index = decorator._node.containedLayers.indexOf(this.id),
+                client = decorator.client,
+                msg;
+
+            decorator._node.containedLayers.splice(index, 1);
+            if (forward) {
+                index = Math.max(0, index - 1);
+            } else {
+                index++;
+            }
+
+            decorator._node.containedLayers.splice(index, 0, this.id);
+
+            msg = `Swapping nested layers at ${index} and ${forward ? index-1 : index+1}`;
+            client.startTransaction(msg);
+            decorator._updateNestedIndices();
+            client.completeTransaction();
+        };
+
+        NestedLayer.prototype.onLastNodeRemoved = function() {
+            var decorator = this._parent,
+                index = decorator._node.containedLayers.indexOf(this.id),
+                msg = `Removing nested layer of ${decorator._node.name} at position ${index}`;
+
+            decorator.client.startTransaction(msg);
+            decorator.client.deleteNode(this.id);
+            decorator.client.completeTransaction();
+        };
+        this.updateNestedTerritory();
+    };
+
+    _.extend(ContainerLayerDecorator.prototype, LayerDecorator.prototype);
+
+    ContainerLayerDecorator.prototype.DECORATOR_ID = DECORATOR_ID;
+
+    ContainerLayerDecorator.prototype._updateNestedIndices = function() {
+        this._node.containedLayers.forEach((layerId, index) => {
+            // Set the layer's member registry to it's index
+            this.client.setMemberRegistry(
+                this._node.id,
+                layerId,
+                CONSTANTS.CONTAINED_LAYER_SET,
+                CONSTANTS.CONTAINED_LAYER_INDEX,
+                index
+            );
+        });
+    };
+
+    ContainerLayerDecorator.prototype.addLayerAt = function(baseId, index) {
+        var client = this.client,
+            parentId = this._node.id,
+            archNode,
+            newId,
+            msg;
+
+        // Get the index of the given layer
+        index = Math.max(index, 0);
+
+        archNode = client.getAllMetaNodes()
+            .find(node => node.getAttribute('name') === 'Architecture');
+
+        // Create a new Architecture node in the given node
+        msg = `Adding layer to ${this._node.name} at position ${index}`;
+        client.startTransaction(msg);
+
+        newId = client.createNode({
+            parentId: parentId,
+            baseId: archNode.getId()
+        });
+        // Create the selected layer
+        client.createNode({
+            parentId: newId,
+            baseId: baseId
+        });
+        client.addMember(parentId, newId, CONSTANTS.CONTAINED_LAYER_SET);
+        this._node.containedLayers.splice(index, 0, newId);
+        this._updateNestedIndices();
+
+        client.completeTransaction();
+    };
+
+    ContainerLayerDecorator.prototype.condense = function() {
+        // hide the nested layers
+        this.$el.attr('class', 'centering-offset condense');
+        this.removeCreateNestedBtn();
+        return LayerDecorator.prototype.condense.apply(this, arguments);
+    };
+
+    ContainerLayerDecorator.prototype.updateNestedTerritory = function() {
+        // Add the nested layers and update
+        if (!this._nestedTerritoryUI) {
+            this._nestedTerritoryUI = this.client.addUI(this, this._containedEvents.bind(this));
+        }
+        this._territory = {};
+        this._node.containedLayers.forEach(id => this._territory[id] = {children: 0});
+        this.client.updateTerritory(this._nestedTerritoryUI, this._territory);
+    };
+
+    ContainerLayerDecorator.prototype._containedEvents = function(events) {
+        for (var i = events.length; i--;) {
+            switch (events[i].etype) {
+            case GME_CONSTANTS.TERRITORY_EVENT_LOAD:
+                if (!this.nestedLayers[events[i].eid]) {
+                    this.createNestedWidget(events[i].eid);
+                }
+                break;
+
+            case GME_CONSTANTS.TERRITORY_EVENT_UNLOAD:
+                this.removeNestedWidget(events[i].eid);
+                break;
+            }
+        }
+        if (events.length > 1) {  // if more than just 'complete' event
+            this.updateExpand();
+        }
+    };
+
+    ContainerLayerDecorator.prototype.update = function(node) {
+        var attrsUpdated = false,
+            attrs = this._attributes;
+
+        this._node = node;
+        // Update the attributes
+        this.setAttributes();
+        attrsUpdated = !_.isEqual(attrs, this._attributes);
+
+        // Check for a new nested layer
+        var hasNewLayers = this._node.containedLayers
+            .filter(id => !this.nestedLayers[id])
+            .length > 0;
+
+        if (hasNewLayers) {
+            this.updateNestedTerritory();
+        } else {
+            // Update the order of the nested layers
+            if (this._selected) {
+                this.expand();
+            } else {
+                this.condense();
+            }
+        }
+        // Only reset fieldsWidth if the attribute has gotten larger
+        if (attrsUpdated) {
+            this.fieldsWidth = null;
+        }
+    };
+
+    ContainerLayerDecorator.prototype.updateExpand = function() {
+        if (this.expanded) {
+            this.expand();
+        }
+    };
+
+    ContainerLayerDecorator.prototype.createNestedWidget = function(id) {
+        if (!this.$nested) {
+            this.$nested = this.$el.append('g')
+                .attr('class', 'nested-layers');
+        }
+
+        this.nestedLayers[id] = new NestedLayer({
+            $container: this.$nested,
+            parent: this,
+            client: this.client,
+            logger: this.logger,
+            onRefresh: this.onNestedRefresh,
+            id: id
+        });
+        return this.nestedLayers[id];
+    };
+
+    ContainerLayerDecorator.prototype.removeNestedWidget = function(id) {
+        this.nestedLayers[id].destroy();
+        delete this.nestedLayers[id];
+        this.updateExpand();
+    };
+
+    ContainerLayerDecorator.prototype._renderInfo = function(top, width) {
+        var isAnUpdate = this.expanded,
+            y = top;
+
+        // Add the attribute fields
+        this.clearFields();
+        this.$attributes = this.$el.append('g')
+            .attr('fill', '#222222');
+
+        if (!isAnUpdate) {
+            this.$attributes.attr('opacity', 0);
+        }
+
+        y = this.createAttributeFields(y, width);
+        y = this.createPointerFields(y, width);
+
+        if (y !== top) {
+            y += this.ROW_HEIGHT/2;
+        }
+        return y;
+    };
+
+    ContainerLayerDecorator.prototype.expand = function() {
+        // This should be rendered with the attributes
+        var height,
+            width,
+
+            // Attributes
+            initialY = 25,
+            isAnUpdate = this.expanded,
+            NAME_MARGIN = 15,
+            nestedMargin = 15,  // minimum
+            margin = 5,
+            y = margin + initialY,
+            x = margin,
+            i;
+
+        // Shift name down
+        this.$name.attr('y', 20);
+
+        // Add the nested children
+        var ids = this._node.containedLayers.filter(id => this.nestedLayers[id]),
+            totalNestedWidth = 0,
+            maxNestedHeight = 0,
+            fieldWidth,
+            widget;
+
+        if (ids.length === 0) {
+            maxNestedHeight = CreateNestedBtn.SIZE * 2;
+        } else {
+            for (i = 0; i < ids.length; i++) {
+                widget = this.nestedLayers[ids[i]].widget;
+                totalNestedWidth += widget.getSvgWidth() * ZOOM;
+                maxNestedHeight = Math.max(widget.getSvgHeight() * ZOOM, maxNestedHeight);
+
+                // Update the buttons (in case of reorder)
+                this.nestedLayers[ids[i]].refreshButtons();
+            }
+        }
+
+        fieldWidth = this.fieldsWidth + 3 * NAME_MARGIN;
+        width = Math.max(
+            this.nameWidth + 2 * NAME_MARGIN,
+            this.size.width,
+            fieldWidth,
+            totalNestedWidth + (ids.length + 1) * nestedMargin
+        );
+
+        // Render attributes
+        y = this._renderInfo(y, fieldWidth);
+        y += nestedMargin;
+
+        // Update width, height
+        height = y + maxNestedHeight + nestedMargin;
+
+        // Equally space the nested widgets
+        nestedMargin = (width - totalNestedWidth)/(ids.length + 1);
+        x = nestedMargin - width/2;
+        for (i = 0; i < ids.length; i++) {
+            this.nestedLayers[ids[i]].$el
+                .attr('transform', `translate(${x}, ${y}) scale(${ZOOM})`);
+            x += this.nestedLayers[ids[i]].widget.getSvgWidth() * ZOOM + nestedMargin;
+        }
+
+        this.removeCreateNestedBtn();
+
+        if (ids.length === 0) {
+            // Add the 'create nested layer' button if no nested layers
+            this.$createNestedBtn = new CreateNestedBtn({
+                context: this,
+                $pEl: this.$el,
+                y: y + CreateNestedBtn.SIZE
+            });
+        }
+
+        this.$body
+            .transition()
+            .attr('x', -width/2)
+            .attr('y', 0)
+            .attr('rx', 0)
+            .attr('ry', 0)
+            .attr('width', width)
+            .attr('height', height)
+            .each('end', () => {
+                if (!isAnUpdate) {
+                    this.$attributes.attr('opacity', 1);
+                    this.$el.attr('class', 'centering-offset expand');
+                }
+            });
+
+        if (this.height !== height || this.width !== width) {
+            this.height = height;
+            this.width = width;
+            this.expanded = true;
+            this.$el
+                .attr('transform', `translate(${this.width/2}, 0)`);
+
+            this.onResize();
+        }
+    };
+
+    ContainerLayerDecorator.prototype.removeCreateNestedBtn = function() {
+        if (this.$createNestedBtn) {
+            this.$createNestedBtn.remove();
+            this.$createNestedBtn = null;
+        }
+    };
+
+    ContainerLayerDecorator.prototype.destroyNested = function() {
+        Object.keys(this.nestedLayers).forEach(id => this.nestedLayers[id].destroy());
+        this.nestedLayers = {};
+
+        if (this.$nested) {
+            this.$nested.remove();
+            this.$nested = this.$el.append('g')
+                .attr('class', 'nested-layers');
+        }
+    };
+
+    ContainerLayerDecorator.prototype.destroy = function() {
+        LayerDecorator.prototype.destroy.call(this);
+        if (this._nestedTerritoryUI) {
+            this.client.removeUI(this._nestedTerritoryUI);
+            this._nestedTerritoryUI = null;
+        }
+        this.destroyNested();
+    };
+
+    var CreateNestedBtn = function(params) {
+        params.title = 'Add nested layer';
+        Buttons.Add.call(this, params);
+    };
+
+    CreateNestedBtn.SIZE = Buttons.Add.SIZE;
+    CreateNestedBtn.prototype = Object.create(Buttons.Add.prototype);
+
+    CreateNestedBtn.prototype._onClick = function() {
+        // Call addLayerAfter and prompt for a layer
+        this.promptLayer()
+            .then(layerId => this.addLayerAt(layerId, 0));
+    };
+
+    ContainerLayerDecorator.prototype.expandAll = function() {
+        this.expand();
+        // For each of the nested layers, expand all their nodes
+        Object.keys(this.nestedLayers)
+            .forEach(id => this.nestedLayers[id].widget.expandAllNodes());
+    };
+
+    ContainerLayerDecorator.prototype.condenseAll = function() {
+        this.condense();
+        // For each of the nested layers, expand all their nodes
+        Object.keys(this.nestedLayers)
+            .forEach(id => this.nestedLayers[id].widget.expandAllNodes(true));
+    };
+
+    return ContainerLayerDecorator;
+});

src/decorators/ContainerLayerDecorator/EasyDAG/NestedLayer.js

@@ -0,0 +1,175 @@
+/*globals define, _ */
+define([
+    'panels/ArchEditor/ArchEditorControl',
+    'widgets/ArchEditor/ArchEditorWidget',
+    'widgets/EasyDAG/Buttons'
+], function(
+    ArchEditor,
+    ArchEditorWidget,
+    Buttons
+) {
+    var nop = () => {};
+    var NestedLayer = function(opts) {
+        this.$el = opts.$container.append('g')
+            .attr('class', 'nested-layer');
+
+        this.id = opts.id;
+        this._parent = opts.parent;
+        this.logger = opts.logger;
+
+        this.refreshButtons = _.debounce(this.updateButtons.bind(this), 100);
+        this.$outline = this.$el.append('rect')  // for hover detection
+            .attr('fill-opacity', 0)
+            .attr('x', 0)
+            .attr('y', 0);
+
+        this.$content = this.$el.append('g');
+        this.initHover();
+
+        this.widget = new ArchEditorWidget({
+            logger: this.logger.fork('ArchWidget'),
+            autoCenter: false,
+            svg: this.$content
+        });
+        this.widget.setTitle =
+        this.widget.updateEmptyMsg = nop;
+        this.onRefresh = opts.onRefresh;
+        this.widget.refreshExtras = this.onWidgetRefresh.bind(this);
+
+        this.control = new ArchEditor({
+            logger: this.logger.fork('ArchControl'),
+            client: opts.client,
+            embedded: true,
+            widget: this.widget
+        });
+        this.control._onUnload = () => {
+            ArchEditor.prototype._onUnload.apply(this.control, arguments);
+            // If it was the last node, remove it
+            var node = this.control._client.getNode(this.id);
+            if (node.getChildrenIds().length === 0) {
+                this.onLastNodeRemoved();
+            }
+        };
+
+        // hack :(
+        this.control.$btnModelHierarchyUp = {
+            show: nop,
+            hide: nop
+        };
+        this.widget.active = true;
+        this.control.selectedObjectChanged(this.id);
+    };
+
+    NestedLayer.prototype.initHover = function() {
+        this.$hover = this.$el.append('g')
+            .attr('class', 'hover-items');
+
+
+
+        this.$el.on('mouseenter', this.onHover.bind(this));
+        this.$el.on('mouseleave', this.onUnhover.bind(this));
+
+        // Buttons
+        this.$leftBtn = new Buttons.Add({
+            hide: true,
+            icon: this.isFirst() ? 'plus' : 'chevron-left',
+            $pEl: this.$hover
+        });
+
+        this.$rightBtn = new Buttons.Add({
+            hide: true,
+            icon: this.isLast() ? 'plus' : 'chevron-right',
+            $pEl: this.$hover
+        });
+
+        this.$deleteBtn = new Buttons.DeleteOne({
+            hide: true,
+            title: 'Delete',
+            $pEl: this.$hover
+        });
+
+        this.$leftBtn._onClick = this.clickLeft.bind(this);
+        this.$rightBtn._onClick = this.clickRight.bind(this);
+        this.$deleteBtn._onClick = () => this.onLastNodeRemoved();
+
+        this.$leftHint = this.$leftBtn.$el.append('title');
+        this.$rightHint = this.$rightBtn.$el.append('title');
+        this.refreshButtons();
+    };
+
+    NestedLayer.prototype.updateButtons = function() {
+        this.$leftBtn.icon = this.isFirst() ? 'plus' : 'chevron-left';
+        this.$rightBtn.icon = this.isLast() ? 'plus' : 'chevron-right';
+
+        this.$leftHint.text(this.isFirst() ?
+            'Add nested layer' :
+            'Move nested layer left'
+        );
+        this.$rightHint.text(this.isLast() ?
+            'Add nested layer' :
+            'Move nested layer right'
+        );
+
+        this.$leftBtn.render();
+        this.$rightBtn.render();
+    };
+
+    NestedLayer.prototype.clickLeft = function() {
+        if (this.isFirst()) {
+            this.promptLayer()
+                .then(layerId => this.addLayerBefore(layerId));
+        } else {
+            this.moveLayerForward();
+        }
+        this.onUnhover();
+    };
+
+    NestedLayer.prototype.promptLayer = function() {
+        var nodes = this.widget.getValidInitialNodes();
+
+        return this.widget.promptLayer(nodes)
+            .then(selected => selected.node.id);
+    };
+
+    NestedLayer.prototype.clickRight = function() {
+        if (this.isLast()) {
+            this.promptLayer()
+                .then(layerId => this.addLayerAfter(layerId));
+        } else {
+            this.moveLayerBackward();
+        }
+        this.onUnhover();
+    };
+
+    NestedLayer.prototype.onHover = function() {
+        this.refreshButtons();
+        this.$hover.attr('class', 'hover-items hovered');
+    };
+
+    NestedLayer.prototype.onUnhover = function() {
+        this.$hover.attr('class', 'hover-items unhovered');
+    };
+
+    NestedLayer.prototype.onWidgetRefresh = function() {
+        var width = this.widget.getSvgWidth(),
+            height = this.widget.getSvgHeight();
+
+        this.$outline
+            .attr('width', width)
+            .attr('height', height);
+
+        this.$leftBtn.$el.attr('transform', `translate(0, ${height/2})`);
+        this.$rightBtn.$el
+            .attr('transform', `translate(${width}, ${height/2})`);
+
+        this.onRefresh();
+    };
+
+    NestedLayer.prototype.destroy = function() {
+        this.control.destroy();
+        this.widget.destroy();
+        this.$el.remove();
+    };
+
+    return NestedLayer;
+});

src/decorators/DcOpDecorator/EasyDAG/DcOpDecorator.EasyDAGWidget.js

@@ -47,11 +47,11 @@ define([
             // create the outputId node
             outputId = this._createOutputNode(baseId);
         } else {
-            this.client.makePointer(outputId, CONSTANTS.POINTER_BASE, baseId);
+            this.client.setPointer(outputId, CONSTANTS.POINTER_BASE, baseId);
         }
         // Copy the data content to the output node
         hash = target.getAttribute('data');
-        this.client.setAttributes(outputId, 'data', hash);
+        this.client.setAttribute(outputId, 'data', hash);
     };
 
     DcOpDecorator.prototype._createOutputNode = function(baseId) {
@@ -69,7 +69,7 @@ define([
             return metaType.getAttribute('name') === 'Outputs';
         });
 
-        return this.client.createChild({
+        return this.client.createNode({
             baseId: baseId,
             parentId: outputCntrId
         });

src/decorators/LayerDecorator/EasyDAG/LayerDecorator.EasyDAGWidget.js

@@ -77,11 +77,11 @@ define([
         // Create a nested "architecture" node and set the ptr target to it
         baseId = base.getId();
         this.client.startTransaction(msg);
-        tgtId = this.client.createChild({
+        tgtId = this.client.createNode({
             parentId: this._node.id,
             baseId: baseId
         });
-        this.client.setAttributes(tgtId, 'name', `${ptr} (${this._node.name})`);
+        this.client.setAttribute(tgtId, 'name', `${ptr} (${this._node.name})`);
         this.savePointer(ptr, tgtId);
         this.client.completeTransaction();
         WebGMEGlobal.State.registerActiveObject(tgtId);
@@ -95,7 +95,7 @@ define([
             // If the target is contained in the current node, delete it!
             if (currentId.indexOf(this._node.id) === 0) {
                 this.client.startTransaction(`Removing layer for ${ptr} of ${name}`);
-                this.client.delMoreNodes([currentId]);
+                this.client.deleteNode(currentId);
                 this.client.completeTransaction();
                 this.logger.info(`Removed ${ptr} and deleted target (${currentId})`);
             } else {

src/decorators/OpIntDecorator/EasyDAG/OpIntDecorator.EasyDAGWidget.js

@@ -102,8 +102,8 @@ define([
             msg = `Deleting "${name}" attribute from "${opName}" operation`;
 
         this.client.startTransaction(msg);
-        this.client.removeAttributeSchema(this._node.id, name);
-        this.client.delAttributes(this._node.id, name);
+        this.client.delAttributeMeta(this._node.id, name);
+        this.client.delAttribute(this._node.id, name);
         this.client.completeTransaction();
     };
 
@@ -129,14 +129,14 @@ define([
 
         if (name !== desc.name) {  // Renaming attribute
             if (name) {
-                this.client.removeAttributeSchema(this._node.id, name);
-                this.client.delAttributes(this._node.id, name);
+                this.client.delAttributeMeta(this._node.id, name);
+                this.client.delAttribute(this._node.id, name);
             }
             name = desc.name;
         }
 
-        this.client.setAttributeSchema(this._node.id, name, schema);
-        this.client.setAttributes(this._node.id, name, desc.defaultValue);
+        this.client.setAttributeMeta(this._node.id, name, schema);
+        this.client.setAttribute(this._node.id, name, desc.defaultValue);
         this.client.completeTransaction();
     };
 

src/decorators/OperationDecorator/EasyDAG/OperationDecorator.EasyDAGWidget.js

@@ -2,9 +2,11 @@
 /*jshint browser: true, camelcase: false*/
 
 define([
+    'deepforge/Constants',
     'decorators/EllipseDecorator/EasyDAG/EllipseDecorator.EasyDAGWidget',
     'css!./OperationDecorator.EasyDAGWidget.css'
 ], function (
+    CONSTANTS,
     DecoratorBase
 ) {
 
@@ -13,6 +15,7 @@ define([
     var OperationDecorator,
         NAME_MARGIN = 25,
         DECORATOR_ID = 'OperationDecorator',
+        OPERATION_COLORS = {},
         PORT_TOOLTIP_OPTS = {
             tipJoint: 'left',
             removeElementsOnHide: true,
@@ -24,8 +27,10 @@ define([
     //     - highlight ports
     //     - unhighlight ports
     //     - report the location of specific ports
+    OPERATION_COLORS[CONSTANTS.OP.OUTPUT] = '#b0bec5';
+    OPERATION_COLORS[CONSTANTS.OP.INPUT] = '#b0bec5';
     OperationDecorator = function (options) {
-        options.color = options.color || '#78909c';
+        options.color = OPERATION_COLORS[options.node.name] || options.color || '#78909c';
         DecoratorBase.call(this, options);
 
         this.id = this._node.id;
@@ -73,14 +78,15 @@ define([
 
     OperationDecorator.prototype.showPorts = function(ids, areInputs) {
         var allPorts = areInputs ? this._node.inputs : this._node.outputs,
-            ports = ids ? allPorts.filter(port => ids.indexOf(port.id) > -1) : allPorts,
             x = -this.width/2,
-            dx = this.width/(ports.length+1),
+            dx = this.width/(allPorts.length+1),
             y = areInputs ? 0 : this.height;  // (this.height/2);
 
-        ports.forEach(port => {
+        allPorts.forEach(port => {
             x += dx;
-            this.renderPort(port, x, y, areInputs);
+            if (!ids || ids.indexOf(port.id) > -1) {
+                this.renderPort(port, x, y, areInputs);
+            }
         });
     };
 
@@ -117,7 +123,7 @@ define([
         tooltip = new Opentip(portIcon[0][0], PORT_TOOLTIP_OPTS);
         tooltip.setContent(port.name);
         portIcon.on('mouseenter', () => tooltip.show());
-        portIcon.on('mouseout', () => tooltip.hide());
+        portIcon.on('mouseleave', () => tooltip.hide());
         this.$portTooltips[port.id] = tooltip;
     };
 

src/layouts/SidebarLayout/SidebarLayout.css

@@ -0,0 +1,12 @@
+.ui-layout-center .layout-center {
+    left: 40px;
+    height: 100%;
+    width: 100%;
+    position: absolute;
+}
+
+.ui-layout-sidebar {
+    top: 64px;
+    width: 40px;
+    bottom: 27px;
+}

src/layouts/SidebarLayout/SidebarLayout.js

@@ -0,0 +1,57 @@
+/*globals define, */
+define([
+    'layout/CHFLayout/CHFLayout/CHFLayout',
+    'text!./templates/SidebarLayout.html',
+    'css!./SidebarLayout.css'
+], function(
+    CHFLayout,
+    SidebarTemplate
+) {
+    'use strict';
+    
+    var SidebarLayout = function(params) {
+        params = params || {};
+        params.template = SidebarTemplate;
+        CHFLayout.call(this, params);
+    };
+
+    SidebarLayout.prototype = Object.create(CHFLayout.prototype);
+
+    SidebarLayout.prototype.getComponentId = function () {
+        return 'SidebarLayout';
+    };
+
+    /**
+     * Initialize the html page. This example is using the jQuery Layout plugin.
+     *
+     * @return {undefined}
+     */
+    SidebarLayout.prototype.init = function() {
+        CHFLayout.prototype.init.apply(this, arguments);
+        this._sidebarPanel = this._body.find('div.ui-layout-sidebar');
+        this._centerPanel = this._body.find('div.layout-center');
+    };
+
+    /**
+     * Add a panel to a given container. This is defined in the corresponding
+     * layout config JSON file.
+     *
+     * @param {Panel} panel
+     * @param {String} container
+     * @return {undefined}
+     */
+    SidebarLayout.prototype.addToContainer = function(panel, container) {
+        if (container === 'sidebar') {
+            this._sidebarPanel.append(panel.$pEl);
+        } else {
+            CHFLayout.prototype.addToContainer.apply(this, arguments);
+        }
+    };
+
+    SidebarLayout.prototype._onCenterResize = function() {
+        var width = this._centerPanel.width() - this._sidebarPanel.width();
+        this._canvas.setSize(width, this._centerPanel.height());
+    };
+
+    return SidebarLayout;
+});

src/layouts/SidebarLayout/templates/SidebarLayout.html

@@ -0,0 +1,7 @@
+<div class="ui-layout-center" style="position: relative;">
+    <div class="layout-center"></div>
+    <div class="float"></div>
+</div>
+<div class="ui-layout-sidebar"></div>
+<div class="ui-layout-north"></div>
+<div class="ui-layout-south"></div>

src/plugins/CheckLibraries/CheckLibraries.js

@@ -92,7 +92,7 @@ define([
 
                     this.logger.info(`${tuple[0]} version info:\n${projVersion} ` +
                         `(project)\n${latest} (latest)`);
-                    return latest !== projVersion;
+                    return projVersion < latest;
                 });
 
                 return Q.all(tuples.map(tuple => this.uploadSeed.apply(this, tuple)));

src/plugins/CreateTorchMeta/schemas/nn.json

@@ -65,7 +65,14 @@
       "affine"
     ],
     "setters": {},
-    "defaults": {},
+    "types": {
+      "eps": "number",
+      "momentum": "number"
+    },
+    "defaults": {
+      "momentum": 0.1,
+      "eps": 0.00001
+    },
     "type": "Misc"
   },
   {
@@ -82,6 +89,35 @@
     "defaults": {},
     "type": "Misc"
   },
+  {
+    "name": "Bottle",
+    "baseType": "Container",
+    "params": [
+      "module",
+      "nInputDim",
+      "nOutputDim"
+    ],
+    "setters": {},
+    "types": {
+      "nInputDim": "number",
+      "module": "nn.Module"
+    },
+    "defaults": {
+      "nInputDim": 2
+    },
+    "type": "Container"
+  },
+  {
+    "name": "CAdd",
+    "baseType": "Module",
+    "params": [
+      "params"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Misc"
+  },
   {
     "name": "CAddTable",
     "baseType": "Module",
@@ -102,6 +138,24 @@
     "defaults": {},
     "type": "Misc"
   },
+  {
+    "name": "CMaxTable",
+    "baseType": "Module",
+    "params": [],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Misc"
+  },
+  {
+    "name": "CMinTable",
+    "baseType": "Module",
+    "params": [],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Misc"
+  },
   {
     "name": "CMul",
     "baseType": "Module",
@@ -175,7 +229,7 @@
     "setters": {},
     "types": {},
     "defaults": {},
-    "type": "Containers"
+    "type": "Simple"
   },
   {
     "name": "ConcatTable",
@@ -184,7 +238,7 @@
     "setters": {},
     "types": {},
     "defaults": {},
-    "type": "Misc"
+    "type": "Container"
   },
   {
     "name": "Contiguous",
@@ -296,13 +350,16 @@
     "params": [
       "p",
       "v1",
-      "inplace"
+      "inplace",
+      "stochasticInference"
     ],
     "setters": {},
     "types": {
-      "p": "number"
+      "p": "number",
+      "stochasticInference": "boolean"
     },
     "defaults": {
+      "stochasticInference": false,
       "p": 0.5
     },
     "type": "Simple"
@@ -357,8 +414,11 @@
   {
     "name": "GradientReversal",
     "baseType": "Module",
-    "params": [],
+    "params": [
+      "lambda"
+    ],
     "setters": {},
+    "types": {},
     "defaults": {},
     "type": "Misc"
   },
@@ -382,7 +442,8 @@
     "baseType": "Module",
     "params": [
       "min_value",
-      "max_value"
+      "max_value",
+      "inplace"
     ],
     "setters": {},
     "types": {
@@ -508,9 +569,7 @@
   {
     "name": "Log",
     "baseType": "Module",
-    "params": [
-      "inputSize"
-    ],
+    "params": [],
     "setters": {},
     "types": {},
     "defaults": {},
@@ -865,7 +924,7 @@
     "setters": {},
     "types": {},
     "defaults": {},
-    "type": "Misc"
+    "type": "Container"
   },
   {
     "name": "ParallelCriterion",
@@ -885,7 +944,7 @@
     "setters": {},
     "types": {},
     "defaults": {},
-    "type": "Misc"
+    "type": "Container"
   },
   {
     "name": "PartialLinear",
@@ -900,6 +959,17 @@
     "defaults": {},
     "type": "Misc"
   },
+  {
+    "name": "PixelShuffle",
+    "baseType": "Module",
+    "params": [
+      "upscaleFactor"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Misc"
+  },
   {
     "name": "Power",
     "baseType": "Module",
@@ -939,6 +1009,17 @@
     "defaults": {},
     "type": "Transfer"
   },
+  {
+    "name": "ReLU6",
+    "baseType": "Module",
+    "params": [
+      "inplace"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Transfer"
+  },
   {
     "name": "Replicate",
     "baseType": "Module",
@@ -1156,6 +1237,18 @@
     "defaults": {},
     "type": "Convolution"
   },
+  {
+    "name": "SpatialClassNLLCriterion",
+    "baseType": "Criterion",
+    "params": [
+      "weights",
+      "sizeAverage"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Criterion"
+  },
   {
     "name": "SpatialContrastiveNormalization",
     "baseType": "Module",
@@ -1279,6 +1372,56 @@
     },
     "type": "Convolution"
   },
+  {
+    "name": "SpatialDilatedConvolution",
+    "baseType": "SpatialConvolution",
+    "params": [
+      "nInputPlane",
+      "nOutputPlane",
+      "kW",
+      "kH",
+      "dW",
+      "dH",
+      "padW",
+      "padH",
+      "dilationW",
+      "dilationH"
+    ],
+    "setters": {},
+    "types": {
+      "dilationW": "number",
+      "dilationH": "number"
+    },
+    "defaults": {
+      "dilationH": 1,
+      "dilationW": 1
+    },
+    "type": "Convolution"
+  },
+  {
+    "name": "SpatialDilatedMaxPooling",
+    "baseType": "SpatialMaxPooling",
+    "params": [
+      "kW",
+      "kH",
+      "dW",
+      "dH",
+      "padW",
+      "padH",
+      "dilationW",
+      "dilationH"
+    ],
+    "setters": {},
+    "types": {
+      "dilationW": "number",
+      "dilationH": "number"
+    },
+    "defaults": {
+      "dilationH": 1,
+      "dilationW": 1
+    },
+    "type": "Misc"
+  },
   {
     "name": "SpatialDivisiveNormalization",
     "baseType": "Module",
@@ -1301,13 +1444,16 @@
     "name": "SpatialDropout",
     "baseType": "Module",
     "params": [
-      "p"
+      "p",
+      "stochasticInference"
     ],
     "setters": {},
     "types": {
-      "p": "number"
+      "p": "number",
+      "stochasticInference": "boolean"
     },
     "defaults": {
+      "stochasticInference": false,
       "p": 0.5
     },
     "type": "Convolution"
@@ -1387,6 +1533,14 @@
     "defaults": {},
     "type": "Convolution"
   },
+  {
+    "name": "SpatialLogSoftMax",
+    "baseType": "Module",
+    "params": [],
+    "setters": {},
+    "defaults": {},
+    "type": "Misc"
+  },
   {
     "name": "SpatialMaxPooling",
     "baseType": "Module",
@@ -1497,6 +1651,17 @@
     },
     "type": "Convolution"
   },
+  {
+    "name": "SpatialUpSamplingBilinear",
+    "baseType": "Module",
+    "params": [
+      "params"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Misc"
+  },
   {
     "name": "SpatialUpSamplingNearest",
     "baseType": "Module",
@@ -1627,6 +1792,22 @@
     "defaults": {},
     "type": "Convolution"
   },
+  {
+    "name": "TemporalDynamicKMaxPooling",
+    "baseType": "Module",
+    "params": [
+      "minK",
+      "factor"
+    ],
+    "setters": {},
+    "types": {
+      "factor": "number"
+    },
+    "defaults": {
+      "factor": 0
+    },
+    "type": "Misc"
+  },
   {
     "name": "TemporalMaxPooling",
     "baseType": "Module",
@@ -1764,16 +1945,81 @@
     "type": "Convolution"
   },
   {
-    "name": "VolumetricDropout",
-    "baseType": "Module",
+    "name": "VolumetricDilatedConvolution",
+    "baseType": "VolumetricConvolution",
     "params": [
-      "p"
+      "nInputPlane",
+      "nOutputPlane",
+      "kT",
+      "kW",
+      "kH",
+      "dT",
+      "dW",
+      "dH",
+      "padT",
+      "padW",
+      "padH",
+      "dilationT",
+      "dilationW",
+      "dilationH"
     ],
     "setters": {},
     "types": {
-      "p": "number"
+      "dilationT": "number",
+      "dilationW": "number",
+      "dilationH": "number"
     },
     "defaults": {
+      "dilationH": 1,
+      "dilationW": 1,
+      "dilationT": 1
+    },
+    "type": "Misc"
+  },
+  {
+    "name": "VolumetricDilatedMaxPooling",
+    "baseType": "VolumetricMaxPooling",
+    "params": [
+      "kT",
+      "kW",
+      "kH",
+      "dT",
+      "dW",
+      "dH",
+      "padT",
+      "padW",
+      "padH",
+      "dilationT",
+      "dilationW",
+      "dilationH"
+    ],
+    "setters": {},
+    "types": {
+      "dilationT": "number",
+      "dilationW": "number",
+      "dilationH": "number"
+    },
+    "defaults": {
+      "dilationH": 1,
+      "dilationW": 1,
+      "dilationT": 1
+    },
+    "type": "Misc"
+  },
+  {
+    "name": "VolumetricDropout",
+    "baseType": "Module",
+    "params": [
+      "p",
+      "stochasticInference"
+    ],
+    "setters": {},
+    "types": {
+      "p": "number",
+      "stochasticInference": "boolean"
+    },
+    "defaults": {
+      "stochasticInference": false,
       "p": 0.5
     },
     "type": "Convolution"
@@ -1864,6 +2110,22 @@
     "defaults": {},
     "type": "Convolution"
   },
+  {
+    "name": "VolumetricReplicationPadding",
+    "baseType": "Module",
+    "params": [
+      "pleft",
+      "pright",
+      "ptop",
+      "pbottom",
+      "pfront",
+      "pback"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Misc"
+  },
   {
     "name": "WeightedEuclidean",
     "baseType": "Module",
@@ -1887,4 +2149,4 @@
     "defaults": {},
     "type": "Criterion"
   }
-]
+]

src/plugins/ExecuteJob/ExecuteJob.Files.js

@@ -0,0 +1,346 @@
+/*globals define*/
+define([
+    './templates/index',
+    'q',
+    'underscore',
+    'deepforge/Constants'
+], function(
+    Templates,
+    Q,
+    _,
+    CONSTANTS
+) {
+
+    var SKIP_ATTRIBUTES = [
+        'code',
+        'stdout',
+        'execFiles',
+        'jobId',
+        'secret',
+        CONSTANTS.LINE_OFFSET,
+        CONSTANTS.DISPLAY_COLOR
+    ];
+    var ExecuteJob = function() {
+    };
+
+    ExecuteJob.prototype.createOperationFiles = function (node) {
+        var files = {};
+        // For each operation, generate the output files:
+        //   inputs/<arg-name>/init.lua  (respective data deserializer)
+        //   pointers/<name>/init.lua  (result of running the main plugin on pointer target - may need a rename)
+        //   outputs/<name>/  (make dirs for each of the outputs)
+        //   outputs/init.lua  (serializers for data outputs)
+        //
+        //   attributes.lua (returns lua table of operation attributes)
+        //   init.lua (main file -> calls main and serializes outputs)
+        //   <name>.lua (entry point -> calls main operation code)
+
+        // add the given files
+        this.logger.info('About to create dist execution files');
+        files['start.js'] = _.template(Templates.START)(CONSTANTS);
+        return this.createEntryFile(node, files)
+            .then(() => this.createClasses(node, files))
+            .then(() => this.createCustomLayers(node, files))
+            .then(() => this.createInputs(node, files))
+            .then(() => this.createOutputs(node, files))
+            .then(() => this.createMainFile(node, files))
+            .then(() => {
+                this.createAttributeFile(node, files);
+                return Q.ninvoke(this, 'createPointers', node, files);
+            });
+    };
+
+    ExecuteJob.prototype.createEntryFile = function (node, files) {
+        this.logger.info('Creating entry files...');
+        return this.getOutputs(node)
+            .then(outputs => {
+                var name = this.getAttribute(node, 'name'),
+                    content = {};
+
+                // inputs and outputs
+                content.name = name;
+                content.outputs = outputs;
+
+                files['init.lua'] = _.template(Templates.ENTRY)(content);
+
+                // Create the deepforge file
+                files['deepforge.lua'] = _.template(Templates.DEEPFORGE)(CONSTANTS);
+            });
+    };
+
+    ExecuteJob.prototype.createClasses = function (node, files) {
+        var metaDict = this.core.getAllMetaNodes(this.rootNode),
+            isClass,
+            metanodes,
+            classNodes,
+            inheritanceLvl = {},
+            code;
+
+        this.logger.info('Creating custom layer file...');
+        metanodes = Object.keys(metaDict).map(id => metaDict[id]);
+        isClass = this.getTypeDictFor('Complex', metanodes);
+
+        classNodes = metanodes.filter(node => {
+            var base = this.core.getBase(node),
+                baseId,
+                count = 1;
+
+            // Count the sets back to a class node
+            while (base) {
+                baseId = this.core.getPath(base);
+                if (isClass[baseId]) {
+                    inheritanceLvl[this.core.getPath(node)] = count;
+                    return true;
+                }
+                base = this.core.getBase(base);
+                count++;
+            }
+
+            return false;
+        });
+
+        // Get the code definitions for each
+        // Sort by levels of inheritance...
+        code = classNodes.sort((a, b) => {
+            var aId = this.core.getPath(a),
+                bId = this.core.getPath(b);
+
+            return inheritanceLvl[aId] > inheritanceLvl[bId];
+        }).map(node =>
+            `require './${this.getAttribute(node, 'name')}.lua'`
+        ).join('\n');
+
+        // Create the class files
+        classNodes.forEach(node => {
+            var name = this.getAttribute(node, 'name');
+            files[`classes/${name}.lua`] = this.getAttribute(node, 'code');
+        });
+
+        // Create the custom layers file
+        files['classes/init.lua'] = code;
+    };
+
+    ExecuteJob.prototype.getTypeDictFor = function (name, metanodes) {
+        var isType = {};
+        // Get all the custom layers
+        for (var i = metanodes.length; i--;) {
+            if (this.getAttribute(metanodes[i], 'name') === name) {
+                isType[this.core.getPath(metanodes[i])] = true;
+            }
+        }
+        return isType;
+    };
+
+    ExecuteJob.prototype.createCustomLayers = function (node, files) {
+        var metaDict = this.core.getAllMetaNodes(this.rootNode),
+            isCustomLayer,
+            metanodes,
+            customLayers,
+            code;
+
+        this.logger.info('Creating custom layer file...');
+        metanodes = Object.keys(metaDict).map(id => metaDict[id]);
+        isCustomLayer = this.getTypeDictFor('CustomLayer', metanodes);
+
+        customLayers = metanodes.filter(node =>
+            this.core.getMixinPaths(node).some(id => isCustomLayer[id]));
+
+        // Get the code definitions for each
+        code = 'require \'nn\'\n\n' + customLayers
+            .map(node => this.getAttribute(node, 'code')).join('\n');
+
+        // Create the custom layers file
+        files['custom-layers.lua'] = code;
+    };
+
+    ExecuteJob.prototype.createInputs = function (node, files) {
+        var tplContents,
+            inputs;
+
+        this.logger.info('Retrieving inputs and deserialize fns...');
+        return this.getInputs(node)
+            .then(allInputs => {
+                // For each input, match the connection with the input name
+                //   [ name, type ] => [ name, type, node ]
+                //
+                // For each input,
+                //  - create the deserializer
+                //  - put it in inputs/<name>/init.lua
+                //  - copy the data asset to /inputs/<name>/init.lua
+                inputs = allInputs
+                    .filter(pair => !!this.getAttribute(pair[2], 'data'));  // remove empty inputs
+
+                files.inputAssets = {};  // data assets
+                return Q.all(inputs.map(pair => {
+                    var name = pair[0],
+                        node = pair[2],
+                        nodeId = this.core.getPath(node),
+                        fromNodeId;
+
+                    // Get the deserialize function. First, try to get it from
+                    // the source method (this guarantees that the correct
+                    // deserialize method is used despite any auto-upcasting
+                    fromNodeId = this.inputPortsFor[nodeId][0] || nodeId;
+
+                    return this.core.loadByPath(this.rootNode, fromNodeId)
+                        .then(fromNode => {
+                            var deserFn,
+                                base,
+                                className;
+
+                            deserFn = this.getAttribute(fromNode, 'deserialize');
+
+                            if (this.isMetaTypeOf(node, this.META.Complex)) {
+                                // Complex objects are expected to define their own
+                                // (static) deserialize factory method
+                                base = this.core.getMetaType(node);
+                                className = this.getAttribute(base, 'name');
+                                deserFn = `return ${className}.deserialize(path)`;
+                            }
+
+                            return {
+                                name: name,
+                                code: deserFn
+                            };
+                        });
+                }));
+            })
+            .then(_tplContents => {
+                tplContents = _tplContents;
+                var hashes = inputs.map(pair => {
+                    var hash = this.getAttribute(pair[2], 'data');
+                    files.inputAssets[pair[0]] = hash;
+                    return {
+                        hash: hash,
+                        name: pair[0]
+                    };
+                });
+
+                return Q.all(hashes.map(pair => 
+                    this.blobClient.getMetadata(pair.hash)
+                        .fail(err => this.onBlobRetrievalFail(node, pair.name, err))));
+            })
+            .then(metadatas => {
+                // Create the deserializer
+                tplContents.forEach((ctnt, i) => {
+                    // Get the name of the given asset
+                    ctnt.filename = metadatas[i].name;
+                    files['inputs/' + ctnt.name + '/init.lua'] = _.template(Templates.DESERIALIZE)(ctnt);
+                });
+                return files;
+            });
+    };
+
+    ExecuteJob.prototype.createOutputs = function (node, files) {
+        // For each of the output types, grab their serialization functions and
+        // create the `outputs/init.lua` file
+        this.logger.info('Creating outputs/init.lua...');
+        return this.getOutputs(node)
+            .then(outputs => {
+                var outputTypes = outputs
+                // Get the serialize functions for each
+                    .map(tuple => {
+                        var node = tuple[2],
+                            serFn = this.getAttribute(node, 'serialize');
+
+                        if (this.isMetaTypeOf(node, this.META.Complex)) {
+                            // Complex objects are expected to define their own
+                            // serialize methods
+                            serFn = 'if data ~= nil then data:serialize(path) end';
+                        }
+
+                        return [tuple[1], serFn];
+                    });
+
+                files['outputs/init.lua'] = _.template(Templates.SERIALIZE)({types: outputTypes});
+            });
+    };
+
+    ExecuteJob.prototype.createMainFile = function (node, files) {
+        this.logger.info('Creating main file...');
+        return this.getInputs(node)
+            .then(inputs => {
+                var name = this.getAttribute(node, 'name'),
+                    code = this.getAttribute(node, 'code'),
+                    pointers = this.core.getPointerNames(node).filter(ptr => ptr !== 'base'),
+                    content = {
+                        name: name
+                    };
+
+                // Get input data arguments
+                content.inputs = inputs
+                    .map(pair => [pair[0], !this.getAttribute(pair[2], 'data')]);  // remove empty inputs
+
+                // Defined variables for each pointers
+                content.pointers = pointers
+                    .map(id => [id, this.core.getPointerPath(node, id) === null]);
+
+                // Add remaining code
+                content.code = code;
+
+                files['main.lua'] = _.template(Templates.MAIN)(content);
+
+                // Set the line offset
+                var lineOffset = this.getLineOffset(files['main.lua'], code);
+                this.setAttribute(node, CONSTANTS.LINE_OFFSET, lineOffset);
+            });
+    };
+
+    ExecuteJob.prototype.getLineOffset = function (main, snippet) {
+        var i = main.indexOf(snippet),
+            lines = main.substring(0, i).match(/\n/g);
+
+        return lines ? lines.length : 0;
+    };
+
+    ExecuteJob.prototype.createAttributeFile = function (node, files) {
+        var numOrBool = /^(-?\d+\.?\d*((e|e-)\d+)?|(true|false))$/,
+            table;
+
+        this.logger.info('Creating attributes file...');
+        table = '{\n\t' + this.core.getAttributeNames(node)
+            .filter(attr => SKIP_ATTRIBUTES.indexOf(attr) === -1)
+            .map(name => {
+                var value = this.getAttribute(node, name);
+                if (!numOrBool.test(value)) {
+                    value = `"${value}"`;
+                }
+                return [name, value];
+            })
+            .map(pair => pair.join(' = '))
+            .join(',\n\t') + '\n}';
+
+        files['attributes.lua'] = `-- attributes of ${this.getAttribute(node, 'name')}\nreturn ${table}`;
+    };
+
+    ExecuteJob.prototype.createPointers = function (node, files, cb) {
+        var pointers,
+            nIds;
+
+        this.logger.info('Creating pointers file...');
+        pointers = this.core.getPointerNames(node)
+            .filter(name => name !== 'base')
+            .filter(id => this.core.getPointerPath(node, id) !== null);
+
+        nIds = pointers.map(p => this.core.getPointerPath(node, p));
+        files.ptrAssets = {};
+        Q.all(
+            nIds.map(nId => this.getPtrCodeHash(nId))
+        )
+        .then(resultHashes => {
+            var name = this.getAttribute(node, 'name');
+            this.logger.info(`Pointer generation for ${name} FINISHED!`);
+            resultHashes.forEach((hash, index) => {
+                files.ptrAssets[`pointers/${pointers[index]}/init.lua`] = hash;
+            });
+            return cb(null, files);
+        })
+        .fail(e => {
+            this.logger.error(`Could not generate pointer files for ${this.getAttribute(node, 'name')}: ${e.toString()}`);
+            return cb(e);
+        });
+    };
+
+    return ExecuteJob;
+});
+

src/plugins/ExecuteJob/ExecuteJob.Metadata.js

@@ -0,0 +1,146 @@
+/*globals define*/
+define([
+    'deepforge/Constants'
+], function(
+    CONSTANTS
+) {
+
+    var ExecuteJob = function() {
+    };
+
+    ExecuteJob.prototype[CONSTANTS.GRAPH_CREATE] = function (job, id) {
+        var graph,
+            name = Array.prototype.slice.call(arguments, 2).join(' '),
+            jobId = this.core.getPath(job);
+
+        id = jobId + '/' + id;
+        this.logger.info(`Creating graph ${id} named ${name}`);
+
+        // Check if the graph already exists
+        graph = this._getExistingMetadata(jobId, 'Graph', name);
+        if (!graph) {
+            graph = this.createNode('Graph', job);
+
+            if (name) {
+                this.setAttribute(graph, 'name', name);
+            }
+            this.createIdToMetadataId[graph] = id;
+        }
+
+        this._metadata[id] = graph;
+    };
+
+    ExecuteJob.prototype[CONSTANTS.GRAPH_LABEL_AXIS.X] = function (job, id) {
+        var name = Array.prototype.slice.call(arguments, 2).join(' '),
+            jobId = this.core.getPath(job),
+            graph;
+
+        id = jobId + '/' + id;
+        this.logger.info(`Labeling the x-axis of ${id}: ${name}`);
+
+        graph = this._metadata[id];
+        this.setAttribute(graph, 'xlabel', name);
+    };
+
+    ExecuteJob.prototype[CONSTANTS.GRAPH_LABEL_AXIS.Y] = function (job, id) {
+        var name = Array.prototype.slice.call(arguments, 2).join(' '),
+            jobId = this.core.getPath(job),
+            graph;
+
+        id = jobId + '/' + id;
+        this.logger.info(`Labeling the y-axis of ${id}: ${name}`);
+
+        graph = this._metadata[id];
+        this.setAttribute(graph, 'ylabel', name);
+    };
+
+    ExecuteJob.prototype[CONSTANTS.GRAPH_PLOT] = function (job, id, x, y) {
+        var jobId = this.core.getPath(job),
+            nonNum = /[^\d-\.]*/g,
+            line,
+            points;
+            
+
+        id = jobId + '/' + id;
+        this.logger.info(`Adding point ${x}, ${y} to ${id}`);
+        line = this._metadata[id];
+        if (!line) {
+            this.logger.warn(`Can't add point to non-existent line: ${id}`);
+            return;
+        }
+
+        // Clean the points by removing and special characters
+        x = x.replace(nonNum, '');
+        y = y.replace(nonNum, '');
+        points = this.getAttribute(line, 'points');
+        points += `${x},${y};`;
+        this.setAttribute(line, 'points', points);
+    };
+
+    ExecuteJob.prototype[CONSTANTS.GRAPH_CREATE_LINE] = function (job, graphId, id) {
+        var jobId = this.core.getPath(job),
+            graph = this._metadata[jobId + '/' + graphId],
+            name = Array.prototype.slice.call(arguments, 3).join(' '),
+            line;
+
+        // Create a 'line' node in the given Graph metadata node
+        name = name.replace(/\s+$/, '');
+        line = this.createNode('Line', graph);
+        this.setAttribute(line, 'name', name);
+        this._metadata[jobId + '/' + id] = line;
+        this.createIdToMetadataId[line] = jobId + '/' + id;
+    };
+
+    ExecuteJob.prototype[CONSTANTS.IMAGE.BASIC] =
+    ExecuteJob.prototype[CONSTANTS.IMAGE.UPDATE] =
+    ExecuteJob.prototype[CONSTANTS.IMAGE.CREATE] = function (job, hash, imgId) {
+        var name = Array.prototype.slice.call(arguments, 3).join(' '),
+            imageNode = this._getImageNode(job, imgId, name);
+
+        this.setAttribute(imageNode, 'data', hash);
+    };
+
+    ExecuteJob.prototype[CONSTANTS.IMAGE.NAME] = function (job, imgId) {
+        var name = Array.prototype.slice.call(arguments, 2).join(' '),
+            imageNode = this._getImageNode(job, imgId, name);
+
+        this.setAttribute(imageNode, 'name', name);
+    };
+
+    ExecuteJob.prototype._getImageNode = function (job, imgId, name) {
+        var jobId = this.core.getPath(job),
+            id = jobId + '/IMAGE/' + imgId,
+            imageNode = this._metadata[id];  // Look for the metadata imageNode
+
+        if (!imageNode) {
+
+            // Check if the imageNode already exists
+            imageNode = this._getExistingMetadata(jobId, 'Image', name);
+            if (!imageNode) {
+                this.logger.info(`Creating image ${id} named ${name}`);
+                imageNode = this.createNode('Image', job);
+                this.setAttribute(imageNode, 'name', name);
+                this.createIdToMetadataId[imageNode] = id;
+            }
+            this._metadata[id] = imageNode;
+        }
+        return imageNode;
+    };
+
+    ExecuteJob.prototype._getExistingMetadata = function (jobId, type, name) {
+        var oldMetadata = this._oldMetadataByName[jobId] && this._oldMetadataByName[jobId][type],
+            node,
+            id;
+
+        if (oldMetadata && oldMetadata[name]) {
+            id = oldMetadata[name];
+            node = this._markForDeletion[jobId][id];
+            delete this._markForDeletion[jobId][id];
+            this.createdMetadataIds[jobId].push(id);  // used for resuming jobs
+        }
+
+        return node || null;
+    };
+
+    return ExecuteJob;
+});

src/plugins/ExecuteJob/ExecuteJob.SafeSave.js

@@ -0,0 +1,377 @@
+/*globals define*/
+define([
+    'plugin/PluginBase',
+    'common/storage/constants',
+    'q',
+    'common/util/assert'
+], function(
+    PluginBase,
+    STORAGE_CONSTANTS,
+    Q,
+    assert
+) {
+
+    var CREATE_PREFIX = 'created_node_',
+        INDEX = 1;
+
+    var ExecuteJob = function() {
+        this.forkNameBase = null;
+        this.runningJobHashes = [];
+        this._currentSave = Q();
+    };
+
+    ExecuteJob.prototype.getCreateId = function () {
+        return CREATE_PREFIX + (++INDEX);
+    };
+
+    ExecuteJob.prototype.isCreateId = function (id) {
+        return (typeof id === 'string') && (id.indexOf(CREATE_PREFIX) === 0);
+    };
+
+    ExecuteJob.prototype.createNode = function (baseType, parent) {
+        var id = this.getCreateId(),
+            parentId = this.isCreateId(parent) ? parent : this.core.getPath(parent);
+
+        this.logger.info(`Creating ${id} of type ${baseType} in ${parentId}`);
+        assert(this.META[baseType], `Cannot create node w/ unrecognized type: ${baseType}`);
+        this.creations[id] = {
+            base: baseType,
+            parent: parentId
+        };
+        return id;
+    };
+
+    ExecuteJob.prototype.deleteNode = function (nodeId) {
+        this.deletions.push(nodeId);
+    };
+
+    ExecuteJob.prototype.delAttribute = function (node, attr) {
+        return this.setAttribute(node, attr, null);
+    };
+
+    ExecuteJob.prototype.setAttribute = function (node, attr, value) {
+        var nodeId;
+
+        if (this.isCreateId(node)) {
+            nodeId = node;
+        } else {
+            nodeId = this.core.getPath(node);
+            assert(typeof nodeId === 'string', `Cannot set attribute of ${nodeId}`);
+        }
+
+        if (value !== null) {
+            this.logger.info(`Setting ${attr} of ${nodeId} to ${value}`);
+        } else {
+            this.logger.info(`Deleting ${attr} of ${nodeId}`);
+        }
+
+        if (!this.changes[nodeId]) {
+            this.changes[nodeId] = {};
+        }
+        this.changes[nodeId][attr] = value;
+    };
+
+    ExecuteJob.prototype.getAttribute = function (node, attr) {
+        var nodeId;
+
+        assert(this.deletions.indexOf(nodeId) === -1,
+            `Cannot get ${attr} from deleted node ${nodeId}`);
+
+        // Check if it was newly created
+        if (this.isCreateId(node)) {
+            nodeId = node;
+            assert(this.creations[nodeId], `Creation node not updated: ${nodeId}`);
+            node = this.META[this.creations[nodeId].base];
+        } else {
+            nodeId = this.core.getPath(node);
+        }
+
+        // Check the most recent changes, then the currentChanges, then the model
+        var value = this._getValueFrom(nodeId, attr, node, this.changes) ||
+            this._getValueFrom(nodeId, attr, node, this.currentChanges);
+
+        if (value) {
+            return value;
+        }
+
+        return this.core.getAttribute(node, attr);
+    };
+
+    ExecuteJob.prototype._getValueFrom = function (nodeId, attr, node, changes) {
+        var base;
+        if (changes[nodeId] && changes[nodeId][attr] !== undefined) {
+            // If deleted the attribute, get the default (inherited) value
+            if (changes[nodeId][attr] === null) {
+                base = this.isCreateId(nodeId) ? node : this.core.getBase(node);
+                return this.getAttribute(base, attr);
+            }
+            return changes[nodeId][attr];
+        }
+    };
+
+    ExecuteJob.prototype._applyNodeChanges = function (node, changes) {
+        var attr,
+            value;
+
+        this.logger.info(`About to apply changes for ${this.core.getPath(node)}`);
+        for (var i = changes.length; i--;) {
+            attr = changes[i][0];
+            value = changes[i][1];
+            if (value !== null) {
+                this.logger.info(`Setting ${attr} to ${value} (${this.core.getPath(node)})`);
+                this.core.setAttribute(node, attr, value);
+            } else {
+                this.core.delAttribute(node, attr);
+            }
+        }
+        return node;
+    };
+
+    ExecuteJob.prototype.applyModelChanges = function () {
+        return this.applyCreations()
+            .then(() => this.applyChanges())
+            .then(() => this.applyDeletions());
+    };
+
+    ExecuteJob.prototype.applyChanges = function () {
+        var nodeIds = Object.keys(this.changes),
+            attrs,
+            value,
+            changes,
+            promises = [],
+            changesFor = {},
+            id,
+            promise;
+
+        this.logger.info('Collecting changes to apply in commit');
+
+        for (var i = nodeIds.length; i--;) {
+            changes = [];
+            attrs = Object.keys(this.changes[nodeIds[i]]);
+            for (var a = attrs.length; a--;) {
+                value = this.changes[nodeIds[i]][attrs[a]];
+                changes.push([attrs[a], value]);
+            }
+            changesFor[nodeIds[i]] = changes;
+
+            assert(changes, `changes are invalid for ${nodeIds[i]}: ${changes}`);
+            assert(!this.isCreateId(nodeIds[i]),
+                `Creation id not resolved to actual id: ${nodeIds[i]}`);
+            promise = this.core.loadByPath(this.rootNode, nodeIds[i]);
+            promises.push(promise);
+        }
+
+        this.currentChanges = this.changes;
+        this.changes = {};
+        // Need to differentiate between read/write changes.
+        this.logger.info(`About to apply changes for ${promises.length} nodes`);
+        return Q.all(promises)
+            .then(nodes => {
+                for (var i = nodes.length; i--;) {
+                    id = this.core.getPath(nodes[i]);
+                    assert(nodes[i], `node is ${nodes[i]} (${nodeIds[i]})`);
+                    this._applyNodeChanges(nodes[i], changesFor[id]);
+                }
+
+                // Local model is now up-to-date. No longer need currentChanges
+                this.currentChanges = {};
+            });
+    };
+
+    ExecuteJob.prototype.applyCreations = function () {
+        var nodeIds = Object.keys(this.creations),
+            tiers = this.createCreationTiers(nodeIds),
+            creations = this.creations,
+            newIds = {},
+            promise = Q(),
+            tier;
+
+        this.logger.info('Applying node creations');
+        for (var i = 0; i < tiers.length; i++) {
+            tier = tiers[i];
+            // Chain the promises, loading each tier sequentially
+            promise = promise.then(this.applyCreationTier.bind(this, creations, newIds, tier));
+        }
+
+        this.creations = {};
+        return promise;
+    };
+
+    ExecuteJob.prototype.applyCreationTier = function (creations, newIds, tier) {
+        var promises = [],
+            parentId,
+            node;
+
+        for (var j = tier.length; j--;) {
+            node = creations[tier[j]];
+            assert(node, `Could not find create info for ${tier[j]}`);
+            parentId = newIds[node.parent] || node.parent;
+            promises.push(this.applyCreation(tier[j], node.base, parentId));
+        }
+        return Q.all(promises).then(nodes => {
+            // Record the newIds so they can be used to resolve creation ids
+            // in subsequent tiers
+            for (var i = tier.length; i--;) {
+                newIds[tier[i]] = this.core.getPath(nodes[i]);
+            }
+        });
+    };
+
+    // Figure out the dependencies between nodes to create.
+    // eg, if newId1 is to be created in newId2, then newId2 will
+    // be in an earlier tier than newId1. Essentially a topo-sort
+    // on a tree structure
+    ExecuteJob.prototype.createCreationTiers = function (nodeIds) {
+        var tiers = [],
+            prevTier = {},
+            tier = {},
+            id,
+            prevLen,
+            i;
+
+        // Create first tier (created inside existing nodes)
+        for (i = nodeIds.length; i--;) {
+            id = nodeIds[i];
+            if (!this.isCreateId(this.creations[id].parent)) {
+                tier[id] = true;
+                nodeIds.splice(i, 1);
+            }
+        }
+        prevTier = tier;
+        tiers.push(Object.keys(tier));
+
+        // Now, each tier consists of the nodes to be created inside a
+        // node from the previous tier
+        while (nodeIds.length) {
+            prevLen = nodeIds.length;
+            tier = {};
+            for (i = nodeIds.length; i--;) {
+                id = nodeIds[i];
+                if (prevTier[this.creations[id].parent]) {
+                    tier[id] = true;
+                    nodeIds.splice(i, 1);
+                }
+            }
+            prevTier = tier;
+            tiers.push(Object.keys(tier));
+            // Every iteration should find at least one node
+            assert(prevLen > nodeIds.length,
+                `Created empty create tier! Remaining: ${nodeIds.join(', ')}`);
+        }
+
+        return tiers;
+    };
+
+    ExecuteJob.prototype.applyCreation = function (tmpId, baseType, parentId) {
+        var base = this.META[baseType],
+            nodeId,
+            id;
+
+        this.logger.info(`Applying creation of ${tmpId} (${baseType}) in ${parentId}`);
+
+        assert(!this.isCreateId(parentId),
+            `Did not resolve parent id: ${parentId} for ${tmpId}`);
+        assert(base, `Invalid base type: ${baseType}`);
+        return this.core.loadByPath(this.rootNode, parentId)
+            .then(parent => this.core.createNode({base, parent}))
+            .then(node => {  // Update the _metadata records
+                id = this.createIdToMetadataId[tmpId];
+                delete this.createIdToMetadataId[tmpId];
+                this._metadata[id] = node;
+
+                // Update creations
+                nodeId = this.core.getPath(node);
+                if (this.changes[tmpId]) {
+                    assert(!this.changes[nodeId],
+                        `Newly created node cannot already have changes! (${nodeId})`);
+                    this.changes[nodeId] = this.changes[tmpId];
+                    delete this.changes[tmpId];
+                }
+                return node;
+            });
+    };
+
+    ExecuteJob.prototype.applyDeletions = function () {
+        var deletions = this.deletions;
+
+        this.deletions = [];
+        return Q.all(deletions.map(id => this.core.loadByPath(this.rootNode, id)))
+            .then(nodes => {
+                for (var i = nodes.length; i--;) {
+                    this.core.deleteNode(nodes[i]);
+                }
+            });
+    };
+
+    // Override 'save' to notify the user on fork
+    ExecuteJob.prototype.save = function (msg) {
+        this._currentSave = this._currentSave
+            .then(() => this.updateForkName(this.forkNameBase))
+            .then(() => this.applyModelChanges())
+            .then(() => PluginBase.prototype.save.call(this, msg))
+            .then(result => {
+                this.logger.info(`Save finished w/ status: ${result.status}`);
+                if (result.status === STORAGE_CONSTANTS.FORKED) {
+                    return this.onSaveForked(result.forkName);
+                } else if (result.status === STORAGE_CONSTANTS.MERGED) {
+                    this.logger.debug('Merged changes. About to update plugin nodes');
+                    return this.updateNodes();
+                }
+            });
+
+        return this._currentSave;
+    };
+
+    ExecuteJob.prototype.onSaveForked = function (forkName) {
+        var name = this.getAttribute(this.activeNode, 'name'),
+            msg = `"${name}" execution has forked to "${forkName}"`;
+        this.currentForkName = forkName;
+
+        this.logManager.fork(forkName);
+        this.runningJobHashes.forEach(jobId => this.originManager.fork(jobId, forkName));
+        this.sendNotification(msg);
+    };
+
+    ExecuteJob.prototype.updateNodes = function (hash) {
+        var activeId = this.core.getPath(this.activeNode);
+
+        hash = hash || this.currentHash;
+        return Q.ninvoke(this.project, 'loadObject', hash)
+            .then(commitObject => {
+                return this.core.loadRoot(commitObject.root);
+            })
+            .then(rootObject => {
+                this.rootNode = rootObject;
+                return this.core.loadByPath(rootObject,activeId);
+            })
+            .then(activeObject => this.activeNode = activeObject)
+            .then(() => {
+                var metaNames = Object.keys(this.META);
+
+                return Q.all(metaNames.map(name => this.updateMetaNode(name)));
+            })
+            .then(() => {
+                var mdNodes,
+                    mdIds;
+
+                mdIds = Object.keys(this._metadata)
+                    .filter(id => !this.isCreateId(this._metadata[id]));
+
+                mdNodes = mdIds.map(id => this.core.getPath(this._metadata[id]))
+                    .map(nodeId => this.core.loadByPath(this.rootNode, nodeId));
+
+                return Q.all(mdNodes).then(nodes => {
+                    for (var i = nodes.length; i--;) {
+                        this._metadata[mdIds[i]] = nodes[i];
+                    }
+                });
+            });
+    };
+
+    ExecuteJob.prototype.updateMetaNode = function (name) {
+        var id = this.core.getPath(this.META[name]);
+        return this.core.loadByPath(this.rootNode, id).then(node => this.META[name] = node);
+    };
+
+    return ExecuteJob;
+});

src/plugins/ExecuteJob/templates/deepforge.ejs

@@ -52,6 +52,14 @@ function Graph:line(name, opts)
     return deepforge._Line(self.id, name, opts)
 end
 
+function Graph:xlabel(name)
+    deepforge._cmd('<%= GRAPH_LABEL_AXIS.X %>', self.id, name)
+end
+
+function Graph:ylabel(name)
+    deepforge._cmd('<%= GRAPH_LABEL_AXIS.Y %>', self.id, name)
+end
+
 -- Image support
 local function saveImage(name, tensor)
     require 'image'

src/plugins/ExecuteJob/templates/start.ejs

@@ -74,7 +74,7 @@ requirejs([
     var checkFinished = () => {
         if (exitCode !== null && remainingImageCount === 0) {
             log('finished!');
-            process.exit(exitCode);
+            cleanup();
         }
     };
 
@@ -201,14 +201,45 @@ requirejs([
 
     // Download the large files
     var inputData = JSON.parse(fs.readFileSync('./input-data.json')),
-        inputPaths = Object.keys(inputData);
+        inputPaths = Object.keys(inputData),
+        pid,
+        job,
+        cleanup;
+
+    // Make sure to kill the spawned process group on exit
+    cleanup = function() {
+        if (job) {
+            pid = job.pid;
+            job = null;
+            log(`killing process group: ${pid}`);
+            process.kill(-pid, 'SIGTERM');
+            if (exitCode !== null) {
+                log(`exiting w/ code ${exitCode}`);
+                process.exit(exitCode);
+            }
+        }
+    };
+
+    process.on('exit', () => {
+        log('received "exit" event')
+        cleanup();
+    });
+    process.on('SIGINT', function() {
+        log('received "SIGINT" event')
+        cleanup();
+        process.exit(130);
+    });
+    process.on('uncaughtException', () => {
+        log('received "uncaughtException" event')
+        cleanup();
+    });
 
     // Request the data from the blob
     prepareCache()
         .then(() => Q.all(inputPaths.map(ipath => getData(ipath, inputData[ipath]))))
         .then(() => {
             // Run 'th init.lua' and merge the stdout, stderr
-            var job = spawn('th', ['init.lua']);
+            job = spawn('th', ['init.lua'], {detached: true});
             job.stdout.on('data', onStdout);
             job.stderr.on('data', onStderr);
             job.on('close', code => {

src/plugins/ExecutePipeline/ExecutePipeline.js

@@ -4,17 +4,17 @@
 define([
     'plugin/CreateExecution/CreateExecution/CreateExecution',
     'plugin/ExecuteJob/ExecuteJob/ExecuteJob',
-    'deepforge/JobLogsClient',
     'common/storage/constants',
     'common/core/constants',
+    'deepforge/Constants',
     'q',
     'text!./metadata.json',
     'underscore'
 ], function (
     CreateExecution,
     ExecuteJob,
-    JobLogsClient,
     STORAGE_CONSTANTS,
+    GME_CONSTANTS,
     CONSTANTS,
     Q,
     pluginMetadata,
@@ -33,9 +33,9 @@ define([
     var ExecutePipeline = function () {
         // Call base class' constructor.
         CreateExecution.call(this);
+        ExecuteJob.call(this);
         this.pluginMetadata = pluginMetadata;
 
-        this._currentSave = Q();
         this.changes = {};
         this.currentChanges = {};  // read-only changes being applied
         this.creations = {};
@@ -101,7 +101,8 @@ define([
      * @param {function(string, plugin.PluginResult)} callback - the result callback
      */
     ExecutePipeline.prototype.main = function (callback) {
-        var startPromise;
+        var startPromise,
+            runId;
 
         this.initRun();
         if (this.core.isTypeOf(this.activeNode, this.META.Pipeline)) {
@@ -118,58 +119,133 @@ define([
             return callback('Current node is not a Pipeline or Execution!', this.result);
         }
 
-        // Get the gmeConfig...
-        this.logManager = new JobLogsClient({
-            logger: this.logger,
-            port: this.gmeConfig.server.port,
-            branchName: this.branchName,
-            projectId: this.projectId
-        });
         this._callback = callback;
         this.currentForkName = null;
 
         startPromise
-        .then(() => this.core.loadSubTree(this.activeNode))
-        .then(subtree => {
-            var children = subtree
-                .filter(n => this.core.getParent(n) === this.activeNode);
+            .then(() => this.core.loadSubTree(this.activeNode))
+            .then(subtree => {
+                var children;
 
-            this.pipelineName = this.getAttribute(this.activeNode, 'name');
-            this.logger.debug(`Loaded subtree of ${this.pipelineName}. About to build cache`);
-            this.buildCache(subtree);
-            this.logger.debug('Parsing execution for job inter-dependencies');
-            this.parsePipeline(children);  // record deps, etc
+                children = subtree
+                    .filter(n => this.core.getParent(n) === this.activeNode);
 
-            this.logger.debug('Clearing old results');
-            return this.clearResults();
-        })
-        .then(() => this.executePipeline())
-        .fail(e => this.logger.error(e));
+                this.pipelineName = this.getAttribute(this.activeNode, 'name');
+                this.forkNameBase = this.pipelineName;
+                this.logger.debug(`Loaded subtree of ${this.pipelineName}. About to build cache`);
+                this.buildCache(subtree);
+                this.logger.debug('Parsing execution for job inter-dependencies');
+                this.parsePipeline(children);  // record deps, etc
+
+                // Detect if resuming execution
+                runId = this.getAttribute(this.activeNode, 'runId');
+                return this.isResuming().then(resuming => {
+                    if (resuming) {
+                        this.currentRunId = runId;
+                        this.startExecHeartBeat();
+                        return this.resumePipeline();
+                    }
+                    return this.startPipeline();
+                });
+
+            })
+            .fail(e => this.logger.error(e));
     };
 
-    // Override 'save' to prevent race conditions while saving
-    ExecutePipeline.prototype.save = function (msg) {
-        // When 'save'  is called, it should still finish any current save op
-        // before continuing
-        this._currentSave = this._currentSave
-            .then(() => this.updateForkName(this.pipelineName))
-            .then(() => this.applyModelChanges())
-            .then(() => CreateExecution.prototype.save.call(this, msg))
-            .then(result => {
-                var msg;
-                if (result.status === STORAGE_CONSTANTS.FORKED) {
-                    this.currentForkName = result.forkName;
-                    this.logManager.fork(result.forkName);
-                    msg = `"${this.pipelineName}" execution has forked to "${result.forkName}"`;
-                    this.sendNotification(msg);
-                } else if (result.status === STORAGE_CONSTANTS.MERGED) {
-                    this.logger.debug('Merged changes. About to update plugin nodes');
-                    return this.updateNodes();
+    ExecutePipeline.prototype.isResuming = function () {
+        var currentlyRunning = this.getAttribute(this.activeNode, 'status') === 'running',
+            runId = this.getAttribute(this.activeNode, 'runId');
+
+        if (runId && currentlyRunning) {
+            // Verify that it is on the correct branch
+            return this.originManager.getOrigin(runId)
+                .then(origin => {
+                    if (origin && origin.branch === this.branchName) {
+                        return this.pulseClient.check(runId)
+                            // If it is dead (not unknown!), then resume
+                            .then(status => status === CONSTANTS.PULSE.DEAD);
+                    } else {
+                        return false;
+                    }
+                });
+        }
+        return Q().then(() => false);
+    };
+
+    ExecutePipeline.prototype.resumePipeline = function () {
+        var nodes = Object.keys(this.nodes).map(id => this.nodes[id]),
+            allJobs = nodes.filter(node => this.core.isTypeOf(node, this.META.Job)),
+            status,
+            jobs = {
+                success: [],
+                failed: [],
+                running: [],
+                pending: []
+            };
+
+        this.logger.info(`Resuming pipeline execution: ${this.currentRunId}`);
+
+        // Get all completed jobs' operations and update records for these
+        for (var i = allJobs.length; i--;) {
+            status = this.getAttribute(allJobs[i], 'status');
+            if (!jobs[status]) {
+                jobs[status] = [];
+            }
+
+            // If any running jobs are missing jobIds, set them to pending
+            if (status === 'running' && !this.canResumeJob(allJobs[i])) {
+                jobs.pending.push(allJobs[i]);
+            } else {
+                jobs[status].push(allJobs[i]);
+            }
+        }
+
+        // Remove finished jobs from incomingCounts
+        jobs.success.concat(jobs.failed, jobs.running)
+            .map(job => this.core.getPath(job))
+            .forEach(id => delete this.incomingCounts[id]);
+
+        return Q.all(allJobs.map(job => this.recordOldMetadata(job, true)))
+            .then(() => Q.all(jobs.success.map(job => this.getOperation(job))))
+            .then(ops => ops.forEach(op => this.updateJobCompletionRecords(op)))
+            .then(() => {
+                
+                if (jobs.running.length) {  // Resume all running jobs
+                    return Q.all(jobs.running.map(job => this.resumeJob(job)));
+                } else if (this.completedCount === this.totalCount) {
+                    return this.onPipelineComplete();
+                } else {
+                    // If none are running, try to start the next ones
+                    return this.executeReadyOperations();
                 }
+            })
+            .fail(err => this._callback(err));
+    };
 
-            });
+    ExecutePipeline.prototype.startPipeline = function () {
+        var rand = Math.floor(Math.random()*10000),
+            commit = this.commitHash.replace('#', '');
 
-        return this._currentSave;
+        this.logger.debug('Clearing old results');
+        this.currentRunId = `Pipeline_${commit}_${Date.now()}_${rand}`;
+
+        // Record the execution origin
+        this.originManager.record(this.currentRunId, {
+            nodeId: this.core.getPath(this.activeNode),
+            job: 'N/A',
+            execution: this.getAttribute(this.activeNode, 'name')
+        });
+
+        this.startExecHeartBeat();
+        return this.clearResults()
+            .then(() => this.executePipeline())
+            .fail(e => this.logger.error(e));
+    };
+
+    ExecutePipeline.prototype.onSaveForked = function (forkName) {
+        // Update the origin on fork
+        this.originManager.fork(this.currentRunId, forkName);
+        return ExecuteJob.prototype.onSaveForked.call(this, forkName);
     };
 
     ExecutePipeline.prototype.updateNodes = function (hash) {
@@ -220,6 +296,7 @@ define([
         this.logger.info('Setting all jobs status to "pending"');
         this.logger.debug(`Making a commit from ${this.currentHash}`);
         this.setAttribute(this.activeNode, 'startTime', Date.now());
+        this.setAttribute(this.activeNode, 'runId', this.currentRunId);
         this.core.delAttribute(this.activeNode, 'endTime');
         return this.save(`Initializing ${this.pipelineName} for execution`);
     };
@@ -297,10 +374,10 @@ define([
     };
 
     ExecutePipeline.prototype.getSiblingIdContaining = function (nodeId) {
-        var parentId = this.core.getPath(this.activeNode) + CONSTANTS.PATH_SEP,
+        var parentId = this.core.getPath(this.activeNode) + GME_CONSTANTS.PATH_SEP,
             relid = nodeId.replace(parentId, '');
 
-        return parentId + relid.split(CONSTANTS.PATH_SEP).shift();
+        return parentId + relid.split(GME_CONSTANTS.PATH_SEP).shift();
     };
 
     ExecutePipeline.prototype.executePipeline = function() {
@@ -358,7 +435,8 @@ define([
             msg += 'finished!';
         }
 
-        this.isDeleted().then(isDeleted => {
+        return this.isDeleted().then(isDeleted => {
+            this.stopExecHeartBeat();
             if (!isDeleted) {
 
                 this.logger.debug(`Pipeline "${name}" complete!`);
@@ -441,10 +519,9 @@ define([
 
     ExecutePipeline.prototype.onOperationComplete = function (opNode) {
         var name = this.getAttribute(opNode, 'name'),
-            nextPortIds = this.getOperationOutputIds(opNode),
             jNode = this.core.getParent(opNode),
-            resultPorts,
             jobId = this.core.getPath(jNode),
+            counts,
             hasReadyOps;
 
         // Set the operation to 'success'!
@@ -457,35 +534,9 @@ define([
         this.save(`Operation "${name}" in ${this.pipelineName} completed successfully`)
             .then(() => {
 
-                // Transport the data from the outputs to any connected inputs
-                //   - Get all the connections from each outputId
-                //   - Get the corresponding dst outputs
-                //   - Use these new ids for checking 'hasReadyOps'
-                resultPorts = nextPortIds.map(id => this.inputPortsFor[id])
-                    .reduce((l1, l2) => l1.concat(l2), []);
+                counts = this.updateJobCompletionRecords(opNode);
+                hasReadyOps = counts.indexOf(0) > -1;
 
-                resultPorts
-                    .map((id, i) => [this.nodes[id], this.nodes[nextPortIds[i]]])
-                    .forEach(pair => {  // [ resultPort, nextPort ]
-                        var result = pair[0],
-                            next = pair[1],
-                            hash = this.getAttribute(result, 'data');
-                        
-                        this.logger.info(`forwarding data (${hash}) from ${this.core.getPath(result)} ` +
-                            `to ${this.core.getPath(next)}`);
-                        this.setAttribute(next, 'data', hash);
-                        this.logger.info(`Setting ${jobId} data to ${hash}`);
-                    });
-
-                // For all the nextPortIds, decrement the corresponding operation's incoming counts
-                hasReadyOps = nextPortIds.map(id => this.getSiblingIdContaining(id))
-                    .reduce((l1, l2) => l1.concat(l2), [])
-
-                    // decrement the incoming counts for each operation id
-                    .map(opId => --this.incomingCounts[opId])
-                    .indexOf(0) > -1;
-
-                this.completedCount++;
                 this.logger.debug(`Operation "${name}" completed. ` + 
                     `${this.totalCount - this.completedCount} remaining.`);
                 if (hasReadyOps) {
@@ -496,9 +547,47 @@ define([
             });
     };
 
+    ExecutePipeline.prototype.updateJobCompletionRecords = function (opNode) {
+        var nextPortIds = this.getOperationOutputIds(opNode),
+            resultPorts,
+            counts;
+
+        // Transport the data from the outputs to any connected inputs
+        //   - Get all the connections from each outputId
+        //   - Get the corresponding dst outputs
+        //   - Use these new ids for checking 'hasReadyOps'
+
+        resultPorts = nextPortIds.map(id => this.inputPortsFor[id])  // dst -> src port
+            .reduce((l1, l2) => l1.concat(l2), []);
+
+        resultPorts
+            .map((id, i) => [this.nodes[id], this.nodes[nextPortIds[i]]])
+            .forEach(pair => {  // [ resultPort, nextPort ]
+                var result = pair[0],
+                    next = pair[1],
+                    hash = this.getAttribute(result, 'data');
+                
+                this.logger.info(`forwarding data (${hash}) from ${this.core.getPath(result)} ` +
+                    `to ${this.core.getPath(next)}`);
+                this.setAttribute(next, 'data', hash);
+                //this.logger.info(`Setting ${jobId} data to ${hash}`);
+            });
+
+        // For all the nextPortIds, decrement the corresponding operation's incoming counts
+        counts = nextPortIds.map(id => this.getSiblingIdContaining(id))
+            .reduce((l1, l2) => l1.concat(l2), [])
+
+            // decrement the incoming counts for each operation id
+            .map(opId => --this.incomingCounts[opId]);
+
+        this.completedCount++;
+        return counts;
+    };
+
     ExecutePipeline.prototype.getOperationOutputIds = function(node) {
         var jobId = this.getSiblingIdContaining(this.core.getPath(node));
 
+        // Map the job to it's output ports
         return this.outputsOf[jobId] || [];
     };
 

src/plugins/Export/Export.js

@@ -0,0 +1,925 @@
+/*globals define */
+/*jshint node:true, browser:true*/
+
+define([
+    'text!./metadata.json',
+    'text!./deepforge.ejs',
+    './format',
+    'plugin/PluginBase',
+    'deepforge/plugin/PtrCodeGen',
+    'deepforge/Constants',
+    'blob/BlobConfig',
+    'underscore',
+    'q'
+], function (
+    pluginMetadata,
+    DeepForgeBaseCode,
+    FORMATS,
+    PluginBase,
+    PtrCodeGen,
+    CONSTANTS,
+    BlobConfig,
+    _,
+    Q
+) {
+    'use strict';
+
+    pluginMetadata = JSON.parse(pluginMetadata);
+    var HEADER_LENGTH = 60,
+        SKIP_ATTRS = {
+            lineOffset: true,
+            code: true
+        },
+        RESERVED = /^(and|break|do|else|elseifend|false|for|function|if|in|local|nil|not|orrepeat|return|then|true|until|while|print)$/,
+        DeepForgeTpl = _.template(DeepForgeBaseCode);
+
+    /**
+     * Initializes a new instance of Export.
+     * @class
+     * @augments {PluginBase}
+     * @classdesc This class represents the plugin Export.
+     * @constructor
+     */
+    var Export = function () {
+        // Call base class' constructor.
+        PluginBase.call(this);
+        this.initRecords();
+    };
+
+    /**
+     * Metadata associated with the plugin. Contains id, name, version, description, icon, configStructue etc.
+     * This is also available at the instance at this.pluginMetadata.
+     * @type {object}
+     */
+    Export.metadata = pluginMetadata;
+
+    // Prototypical inheritance from PluginBase.
+    Export.prototype = Object.create(PluginBase.prototype);
+    Export.prototype.constructor = Export;
+
+    Export.prototype.initRecords = function() {
+        this.pluginMetadata = pluginMetadata;
+
+        this._srcIdFor = {};  // input path -> output data node path
+
+        this._nameFor = {};  // input path -> opname
+        this._outputNames = {};
+        this._baseNameFor = {};
+        this._dataNameFor = {};  
+        this._instanceNames = {};
+        this._opBaseNames = {};
+        this._fnNameFor = {};
+        this._functions = {};  // function definitions for the operations
+
+        // topo sort stuff
+        this._nextOps = {};
+        this._incomingCnts = {};
+
+        this._operations = {};
+        this.activeNodeId = null;
+        this.activeNodeDepth = null;
+
+        this.isInputOp = {};
+        this._portCache = {};
+        this.inputNode = {};
+        this.outputDataToOpId = {};
+        this.isOutputOp = {};
+    };
+
+    /**
+     * Main function for the plugin to execute. This will perform the execution.
+     * Notes:
+     * - Always log with the provided logger.[error,warning,info,debug].
+     * - Do NOT put any user interaction logic UI, etc. inside this method.
+     * - callback always has to be called even if error happened.
+     *
+     * @param {function(string, plugin.PluginResult)} callback - the result callback
+     */
+    Export.prototype.main = function (callback) {
+        this.initRecords();
+
+        // Get all the children and call generate exec file
+        this.activeNodeId = this.core.getPath(this.activeNode);
+        this.activeNodeDepth = this.activeNodeId.split('/').length + 1;
+
+        if (this.isMetaTypeOf(this.activeNode, this.META.Execution)) {
+            this.activeNodeDepth++;
+        }
+
+        return this.core.loadChildren(this.activeNode)
+            .then(nodes => this.generateOutputFiles(nodes))
+            .catch(err => callback(err))
+            .then(hash => {
+                this.result.addArtifact(hash);
+                this.result.setSuccess(true);
+                callback(null, this.result);
+            })
+            .fail(err => callback(err));
+    };
+
+    Export.prototype.getCurrentConfig = function () {
+        var config = PluginBase.prototype.getCurrentConfig.call(this);
+        config.staticInputs = config.staticInputs || [];
+        return config;
+    };
+
+    Export.prototype.getExporterFor = function (name) {
+        var Exporter = function() {},
+            format = FORMATS[name],
+            exporter;
+
+        Exporter.prototype = this;
+        exporter = new Exporter();
+
+        if (typeof format === 'function') {
+            exporter.main = format;
+        } else {
+            _.extend(exporter, format);
+        }
+        return exporter;
+    };
+
+    Export.prototype.generateOutputFiles = function (children) {
+        var name = this.core.getAttribute(this.activeNode, 'name');
+
+        return this.createCodeSections(children)
+            .then(sections => {
+                // Get the selected format
+                var config = this.getCurrentConfig(),
+                    format = config.format || 'Basic CLI',
+                    exporter,
+                    staticInputs,
+                    files;
+
+                this.logger.info(`About to retrieve ${config.format} exporter`);
+                exporter = this.getExporterFor(format);
+
+                staticInputs = config.staticInputs.map(id => {
+                    var opId = id.split('/').splice(0, this.activeNodeDepth).join('/'),
+                        port = this._portCache[id];
+
+                    return {
+                        portId: id,
+                        id: opId,
+                        hash: this.core.getAttribute(port, 'data'),
+                        name: this._nameFor[opId]
+                    };
+                });
+
+                this.logger.info('Invoking exporter "main" function...');
+                try {
+                    files = exporter.main(sections, staticInputs, config.extensionConfig);
+                } catch (e) {
+                    this.logger.error(`Exporter failed: ${e.toString()}`);
+                    throw e;
+                }
+                // If it returns a string, just put a single file
+                if (typeof files === 'string') {
+                    return this.blobClient.putFile(`${name}.lua`, files);
+                } else {  // filename -> content
+                    var artifact = this.blobClient.createArtifact(name),
+                        objects = {};
+
+                    Object.keys(files).forEach(key => {
+                        if (BlobConfig.hashRegex.test(files[key])) {
+                            objects[key] = files[key];
+                            delete files[key];
+                        }
+                    });
+
+                    return artifact.addFiles(files)
+                        .then(() => artifact.addObjectHashes(objects))
+                        .then(() => artifact.save());
+                }
+            });
+    };
+
+    Export.prototype.createCodeSections = function (children) {
+        // Convert opNodes' jobs to the nested operations
+        var opNodes,
+            nodes;
+
+        return this.unpackJobs(children)
+            .then(_nodes => {
+                nodes = _nodes;
+                opNodes = nodes
+                    .filter(node => this.isMetaTypeOf(node, this.META.Operation));
+
+                // Sort the connections to come first
+                nodes
+                    .map(node => [
+                        node,
+                        this.isMetaTypeOf(node, this.META.Transporter) ? -1 : 1
+                    ])
+                    .sort((a, b) => a[1] < b[1] ? -1 : 1);
+
+                return Q.all(nodes.map(node => this.registerNode(node)));
+            })
+            .then(() => Q.all(opNodes
+                .filter(n => {
+                    var id = this.core.getPath(n);
+                    return !this.isInputOp[id];
+                })
+                .map(node => this.createOperation(node)))
+            )
+            .then(operations => {
+                var opDict = {},
+                    firstOpIds;
+
+                firstOpIds = opNodes.map(n => this.core.getPath(n))
+                    .filter(id => !this._incomingCnts[id]);
+
+                operations.forEach(op => opDict[op.id] = op);
+
+                // Toposort!
+                return this.sortOperations(opDict, firstOpIds);
+            })
+            .then(operations => this.generateCodeSections(operations))
+            .fail(err => this.logger.error(err));
+    };
+
+    Export.prototype.unpackJobs = function (nodes) {
+        return Q.all(
+            nodes.map(node => {
+                if (!this.isMetaTypeOf(node, this.META.Job)) {
+                    return node;
+                }
+                return this.core.loadChildren(node)
+                    .then(children =>
+                        children.find(c => this.isMetaTypeOf(c, this.META.Operation))
+                    );
+            })
+        );
+    };
+
+    Export.prototype.sortOperations = function (operationDict, opIds) {
+        var nextIds = [],
+            sorted = opIds,
+            dstIds,
+            id;
+
+        if (!opIds.length) {
+            return [];
+        }
+
+        // Decrement all next ops
+        dstIds = opIds.map(id => this._nextOps[id])
+            .reduce((l1, l2) => l1.concat(l2), []);
+
+        for (var i = dstIds.length; i--;) {
+            id = dstIds[i];
+            if (--this._incomingCnts[id] === 0) {
+                nextIds.push(id);
+            }
+        }
+
+        // append
+        return sorted
+            .map(id => operationDict[id])
+            .filter(op => !!op)
+            .concat(this.sortOperations(operationDict, nextIds));
+    };
+
+    Export.prototype.generateCodeSections = function(sortedOps) {
+        // Create the code sections:
+        //  - operation definitions
+        //  - pipeline definition
+        //  - main
+        var code = {},
+            baseIds = [],
+            outputOps = [],
+            mainOps = [];
+
+        // Define the operation functions...
+        code.operations = {};
+        for (var i = 0; i < sortedOps.length; i++) {
+            if (this.isInputOp[sortedOps[i].id]) {
+                continue;
+            }
+            if (!this.isOutputOp[sortedOps[i].id]) {
+                if (!baseIds.includes(sortedOps[i].baseId)) {  // new definition
+                    code.operations[sortedOps[i].basename] = this.defineOperationFn(sortedOps[i]);
+                    baseIds.push(sortedOps[i].baseId);
+                }
+                mainOps.push(sortedOps[i]);
+            } else {
+                outputOps.push(sortedOps[i]);
+            }
+        }
+
+        // Define the pipeline function
+        code.pipelines = this.definePipelineFn(mainOps, outputOps);
+
+        // Define the serializers/deserializers
+        this.addCodeSerializers(code);
+
+        // Define the main input names
+        code.pipelineName = Object.keys(code.pipelines)[0];
+        code.pipelineInputNames = Object.keys(this.isInputOp).map(id => this._nameFor[id]);
+
+        // Add custom class definitions
+        this.addCustomClasses(code);
+
+        // Add custom layer definitions
+        this.addCustomLayers(code);
+
+        return code;
+    };
+
+    // expose this utility function to format extensions
+    var indent = Export.prototype.indent = function(text, spaces) {
+        spaces = spaces || 3;
+        return text.replace(/^/mg, new Array(spaces+1).join(' '));
+    };
+
+    Export.prototype.defineOperationFn = function(operation) {
+        var lines = [],
+            args = operation.inputNames || [];
+
+        // Create the function definition
+        args.unshift('attributes');
+        // Add the refs to the end
+        args = args.concat(operation.refNames);
+
+        args = args.join(', ');
+
+        lines.push(`local function ${operation.basename}(${args})`);
+        lines.push(indent(operation.code));
+        lines.push('end');
+
+        return lines.join('\n');
+    };
+
+    Export.prototype.definePipelineFn = function(sortedOps, outputOps) {
+        var inputArgs = Object.keys(this.isInputOp).map(id => this._nameFor[id]),
+            name = this.core.getAttribute(this.activeNode, 'name'),
+            safename = getUniqueName(name, this._opBaseNames),
+            results = [],
+            result = {},
+            returnStat,
+            fnbody;
+
+        // Call each function in order, with the respective attributes, etc
+        fnbody = sortedOps.map(op => this.getOpInvocation(op)).join('\n');
+
+        // Create the return statement
+        results.push('\n\nresults = {}');
+        outputOps.map(op => this.getOutputPair(op))
+            .forEach(pair => results.push(`results['${pair[0]}'] = ${pair[1]}`));
+        results.push('return results');
+        returnStat = results.join('\n');
+
+        // Merge the fnbody, return statement and the function def
+        result[safename] = `local function ${safename} (${inputArgs.join(', ')})\n` +
+            `${indent(fnbody + returnStat)}\nend`;
+
+        return result;
+    };
+
+    Export.prototype.getOutputPair = function(operation) {
+        var input = operation.inputValues[0].slice(),
+            value;
+
+        // Get the src operation name and data value name
+        input[0] += '_results';
+        value = input.join('.');
+        return [this._nameFor[operation.id], value];
+    };
+
+    Export.prototype.addCodeSerializers = function(sections) {
+        var loadNodes = {},
+            saveNodes = {};
+
+        // Add the serializer fn names for each input
+        sections.serializerFor = {};
+        sections.deserializerFor = {};
+
+        Object.keys(this.isOutputOp).map(id => {
+            var name = this._nameFor[id];
+            sections.serializerFor[name] = `__save['${name}']`;
+        });
+
+        // Add the serializer definitions
+        Object.keys(this.isInputOp).forEach(id => {
+            var node = this.inputNode[id],
+                name = this._nameFor[id];
+
+            loadNodes[id] = node;
+            sections.deserializerFor[name] = `__load['${this._nameFor[id]}']`;
+        });
+
+        sections.deserializers = this.createTorchFnDict(
+            '__load',
+            loadNodes,
+            'deserialize',
+            'path'
+        );
+
+        // Add the deserializer definitions
+        Object.keys(this.outputDataToOpId).forEach(dataId => {
+            var opId = this.outputDataToOpId[dataId];
+            // The key is used for the output name resolution. The
+            // value is used for the serialization fn look-up. So,
+            // the key is the output operation id and the value is
+            // the data port connected to the output operation
+            saveNodes[opId] = this._portCache[this._srcIdFor[dataId]];
+        });
+
+        sections.serializers = this.createTorchFnDict(
+            '__save',
+            saveNodes,
+            'serialize',
+            'path, data'
+        );
+
+        // Add a saveOutputs method for convenience
+        sections.serializeOutputsDef = [
+            'local function __saveOutputs(data)',
+            indent(Object.keys(this.isOutputOp).map(id => {
+                var name = this._nameFor[id];
+                return [
+                    `print('saving ${name}...')`,
+                    `${sections.serializerFor[name]}('${name}', data['${name}'])`
+                ].join('\n');
+
+            }).join('\n')),
+            'end'
+        ].join('\n');
+
+        sections.serializeOutputs = '__saveOutputs(outputs)';
+    };
+
+    Export.prototype.createTorchFnDict = function(name, nodeDict, attr, args) {
+        return [
+            `local ${name} = {}`,
+            Object.keys(nodeDict).map(id => {
+                var node = nodeDict[id];
+                return [
+                    `${name}['${this._nameFor[id]}'] = function(${args})`,
+                    indent(this.core.getAttribute(node, attr)),
+                    'end'
+                ].join('\n');
+            }).join('\n')
+        ].join('\n');
+    };
+
+    Export.prototype.addCustomClasses = function(sections) {
+        var metaDict = this.core.getAllMetaNodes(this.rootNode),
+            isClass,
+            metanodes,
+            classNodes,
+            inheritanceLvl = {};
+
+        this.logger.info('Creating custom layer file...');
+        metanodes = Object.keys(metaDict).map(id => metaDict[id]);
+        isClass = this.getTypeDictFor('Complex', metanodes);
+
+        // Store the dependencies for each class
+        sections.classDependencies = {};
+
+        classNodes = metanodes.filter(node => {
+            var base = this.core.getBase(node),
+                baseId,
+                deps = [],
+                name,
+                count = 1;
+
+            // Count the sets back to a class node
+            while (base) {
+                deps.push(this.core.getAttribute(base, 'name'));
+                baseId = this.core.getPath(base);
+                if (isClass[baseId]) {
+                    inheritanceLvl[this.core.getPath(node)] = count;
+                    name = this.core.getAttribute(node, 'name');
+                    sections.classDependencies[name] = deps;
+                    return true;
+                }
+                base = this.core.getBase(base);
+                count++;
+            }
+
+            return false;
+        });
+
+        // Get the code definitions for each
+        sections.classes = {};
+        classNodes
+            .sort((a, b) => {
+                var aId = this.core.getPath(a),
+                    bId = this.core.getPath(b);
+
+                return inheritanceLvl[aId] > inheritanceLvl[bId];
+            })
+            .forEach(node => {
+                var name = this.core.getAttribute(node, 'name'),
+                    code = this.core.getAttribute(node, 'code');
+
+                sections.classes[name] = code;
+            });
+
+        // order classes by dependency
+        sections.orderedClasses = Object.keys(sections.classes)
+            .sort((a, b) => {
+                // if a depends on b, switch them (return 1)
+                if (sections.classDependencies[a].includes(b)) {
+                    return 1;
+                }
+                return -1;
+            });
+    };
+
+    Export.prototype.addCustomLayers = function(sections) {
+        var metaDict = this.core.getAllMetaNodes(this.rootNode),
+            isCustomLayer,
+            metanodes,
+            customLayers;
+
+        this.logger.info('Creating custom layer file...');
+        metanodes = Object.keys(metaDict).map(id => metaDict[id]);
+        isCustomLayer = this.getTypeDictFor('CustomLayer', metanodes);
+
+        customLayers = metanodes.filter(node =>
+            this.core.getMixinPaths(node).some(id => isCustomLayer[id]));
+
+        // Get the code definitions for each
+        sections.layers = {};
+        customLayers
+            .map(layer => [
+                this.core.getAttribute(layer, 'name'),
+                this.core.getAttribute(layer, 'code')
+            ])
+            .forEach(pair => sections.layers[pair[0]] = pair[1]);
+    };
+
+
+    Export.prototype.getTypeDictFor = function (name, metanodes) {
+        var isType = {};
+        // Get all the custom layers
+        for (var i = metanodes.length; i--;) {
+            if (this.core.getAttribute(metanodes[i], 'name') === name) {
+                isType[this.core.getPath(metanodes[i])] = true;
+            }
+        }
+        return isType;
+    };
+
+    var toAttrString = function(attr) {
+        if (/^\d+\.?\d*$/.test(attr) || /^(true|false|nil)$/.test(attr)) {
+            return attr;
+        }
+        return `"${attr}"`;
+    };
+
+    Export.prototype.getOpInvocation = function(op) {
+        var lines = [],
+            attrs,
+            refInits = [],
+            args;
+
+        attrs = '{' +
+            Object.keys(op.attributes).map(key => `${key}=${toAttrString(op.attributes[key])}`)
+            .join(',') +
+        '}';
+
+        lines.push(`local ${op.name}_attrs = ${attrs}`);
+        args = (op.inputValues || [])
+            .map(val => val instanceof Array ? `${val[0]}_results.${val[1]}` : val);
+
+        args.unshift(op.name + '_attrs');
+            
+        // Create the ref init functions
+        refInits = op.refs.map((code, index) => {
+            return [
+                `local function create_${op.refNames[index]}()`,
+                indent(code),
+                'end'
+            ].join('\n');
+        });
+        lines = lines.concat(refInits);
+        args = args.concat(op.refNames.map(name => `create_${name}()`));
+        args = args.join(', ');
+        lines.push(`local ${op.name}_results = ${op.basename}(${args})`);
+
+        return lines.join('\n');
+    };
+
+    Export.prototype.getOutputName = function(node) {
+        var basename = this.core.getAttribute(node, 'saveName');
+
+        return getUniqueName(basename, this._outputNames, true);
+    };
+
+    Export.prototype.getVariableName = function (/*node*/) {
+        var c = Object.keys(this.isInputOp).length;
+
+        if (c !== 1) {
+            return `input${c}`;
+        }
+
+        return 'input';
+    };
+
+    Export.prototype.registerNode = function (node) {
+        if (this.isMetaTypeOf(node, this.META.Operation)) {
+            return this.registerOperation(node);
+        } else if (this.isMetaTypeOf(node, this.META.Transporter)) {
+            return this.registerTransporter(node);
+        }
+    };
+
+    var getUniqueName = function(namebase, takenDict, unsafeAllowed) {
+        var name,
+            i = 2,
+            isUnsafe = function(name) {
+                return !unsafeAllowed && RESERVED.test(name);
+            };
+
+        if (!unsafeAllowed) {
+            namebase = namebase.replace(/[^A-Za-z\d]/g, '_');
+        }
+        name = namebase;
+        // Get a unique operation name
+        while (takenDict[name] || isUnsafe(name)) {
+            name = namebase + '_' + i;
+            i++;
+        }
+        takenDict[name] = true;
+
+        return name;
+    };
+
+    Export.prototype.registerOperation = function (node) {
+        var name = this.core.getAttribute(node, 'name'),
+            id = this.core.getPath(node),
+            base = this.core.getBase(node),
+            baseId = this.core.getPath(base),
+            baseName = this.core.getAttribute(base, 'name');
+
+        // If it is an Input/Output operation, assign it a variable name
+        if (baseName === CONSTANTS.OP.INPUT) {
+            this.isInputOp[id] = node;
+            name = this.getVariableName(node);
+        } else if (baseName === CONSTANTS.OP.OUTPUT) {
+            this.isOutputOp[id] = node;
+            name = this.getOutputName(node);
+        } else {
+            // get a unique operation instance name
+            name = getUniqueName(name, this._instanceNames);
+        }
+
+        this._nameFor[id] = name;
+
+        // get a unique operation base name
+        if (!this._fnNameFor[baseId]) {
+            name = this.core.getAttribute(base, 'name');
+            name = getUniqueName(name, this._opBaseNames);
+            this._fnNameFor[baseId] = name;
+        }
+
+        // For operations, register all output data node names by path
+        return this.core.loadChildren(node)
+            .then(cntrs => {
+                var outputs = cntrs.find(n => this.isMetaTypeOf(n, this.META.Outputs)),
+                    inputs = cntrs.find(n => this.isMetaTypeOf(n, this.META.Inputs));
+
+                return Q.all([inputs, outputs].map(cntr => this.core.loadChildren(cntr)));
+            })
+            .then(data => {
+                var inputs = data[0],
+                    outputs = data[1];
+
+                // Get the input type
+                outputs.forEach(output => {
+                    var dataId = this.core.getPath(output);
+
+                    name = this.core.getAttribute(output, 'name');
+                    this._dataNameFor[dataId] = name;
+
+                    this._portCache[dataId] = output;
+                });
+                inputs.forEach(input => 
+                    this._portCache[this.core.getPath(input)] = input
+                );
+
+                // Extra recording for input/output nodes in the pipeline
+                if (this.isInputOp[id]) {
+                    this.inputNode[id] = outputs[0];
+                } else if (this.isOutputOp[id]) {
+                    this.outputDataToOpId[this.core.getPath(inputs[0])] = id;
+                }
+            });
+    };
+
+    Export.prototype.registerTransporter = function (node) {
+        var outputData = this.core.getPointerPath(node, 'src'),
+            inputData = this.core.getPointerPath(node, 'dst'),
+            srcOpId = this.getOpIdFor(outputData),
+            dstOpId = this.getOpIdFor(inputData);
+
+        this._srcIdFor[inputData] = outputData;
+
+        // Store the next operation ids for the op id
+        if (!this._nextOps[srcOpId]) {
+            this._nextOps[srcOpId] = [];
+        }
+        this._nextOps[srcOpId].push(dstOpId);
+
+        // Increment the incoming counts for each dst op
+        this._incomingCnts[dstOpId] = this._incomingCnts[dstOpId] || 0;
+        this._incomingCnts[dstOpId]++;
+    };
+
+    Export.prototype.getOpIdFor = function (dataId) {
+        var ids = dataId.split('/'),
+            depth = ids.length;
+
+        ids.splice(this.activeNodeDepth - depth);
+        return ids.join('/');
+    };
+
+    // For each operation...
+    //   - unpack the inputs from prev ops
+    //   - add the attributes table (if used)
+    //     - check for '\<attributes\>' in code
+    //   - add the references
+    //     - generate the code
+    //     - replace the `return <thing>` w/ `<ref-name> = <thing>`
+    Export.prototype.createOperation = function (node) {
+        var id = this.core.getPath(node),
+            baseId = this.core.getPath(this.core.getBase(node)),
+            attrNames = this.core.getValidAttributeNames(node),
+            operation = {};
+
+        operation.name = this._nameFor[id];
+        operation.basename = this._fnNameFor[baseId];
+        operation.baseId = baseId;
+        operation.id = id;
+        operation.code = this.core.getAttribute(node, 'code');
+        operation.attributes = {};
+        for (var i = attrNames.length; i--;) {
+            if (!SKIP_ATTRS[attrNames[i]]) {
+                operation.attributes[attrNames[i]] = this.core.getAttribute(node, attrNames[i]);
+            }
+        }
+
+        // Get all the input names (and sources)
+        return this.core.loadChildren(node)
+            .then(containers => {
+                var inputs;
+
+                inputs = containers
+                    .find(cntr => this.isMetaTypeOf(cntr, this.META.Inputs));
+
+                this.logger.info(`${operation.name} has ${containers.length} cntrs`);
+                return this.core.loadChildren(inputs);
+            })
+            .then(data => {
+                // Get the input names and sources
+                var inputNames = data.map(d => this.core.getAttribute(d, 'name')),
+                    ids = data.map(d => this.core.getPath(d)),
+                    srcIds = ids.map(id => this._srcIdFor[id]);
+
+                operation.inputNames = inputNames || [];
+                operation.inputValues = inputNames.map((name, i) => {
+                    var id = srcIds[i],
+                        srcDataName = this._dataNameFor[id],
+                        srcOpId = this.getOpIdFor(id),
+                        srcOpName = this._nameFor[srcOpId];
+
+                    if (this.isInputOp[srcOpId]) {
+                        return this._nameFor[srcOpId];
+                    } else {
+                        return [srcOpName, srcDataName];
+                    }
+                });
+
+                return operation;
+
+            })
+            .then(operation => {
+
+                // For each reference, run the plugin and retrieve the generated code
+                operation.refNames = [];
+
+                if (!this.isInputOp[operation.id]) {
+                    operation.refNames = this.core.getPointerNames(node)
+                        .filter(name => name !== 'base');
+                }
+
+                var refs = operation.refNames
+                    .map(ref => [ref, this.core.getPointerPath(node, ref)]);
+
+                return Q.all(
+                    refs.map(pair => this.genPtrSnippet.apply(this, pair))
+                );
+            })
+            .then(codeFiles => {
+                operation.refs = codeFiles;
+                return operation;
+            });
+    };
+
+    Export.prototype.genPtrSnippet = function (ptrName, pId) {
+        return this.getPtrCodeHash(pId)
+            .then(hash => this.blobClient.getObjectAsString(hash));
+    };
+
+    Export.prototype.createHeader = function (title, length) {
+        var len;
+        title = ` ${title} `;
+        length = length || HEADER_LENGTH;
+
+        len = Math.max(
+            Math.floor((length - title.length)/2),
+            2
+        );
+
+        return [
+            '',
+            title,
+            ''
+        ].join(new Array(len+1).join('-')) + '\n';
+
+    };
+
+    Export.prototype.genOperationCode = function (operation) {
+        var header = this.createHeader(`"${operation.name}" Operation`),
+            codeParts = [],
+            body = [];
+
+        codeParts.push(header);
+        codeParts.push(`local ${operation.name}_results`);
+        codeParts.push('do');
+
+        if (operation.inputs.length) {
+            body.push(operation.inputs.join('\n'));
+        }
+
+        if (operation.refs.length) {
+            body.push(operation.refs.join('\n'));
+        }
+
+        body.push(operation.code);
+
+        codeParts.push(indent(body.join('\n')));
+        codeParts.push('end');
+        codeParts.push('');
+
+        operation.code = codeParts.join('\n');
+        return operation;
+    };
+
+    _.extend(Export.prototype, PtrCodeGen.prototype);
+
+    // Extra utilities for export types
+    Export.prototype.INIT_CLASSES_FN = '__init_classes';
+    Export.prototype.INIT_LAYERS_FN = '__init_layers';
+    Export.prototype.getAllDefinitions = function (sections) {
+        var code = [],
+            classes,
+            initClassFn,
+            initLayerFn;
+
+        classes = sections.orderedClasses
+            // Create fns from the classes
+            .map(name => this.indent(sections.classes[name])).join('\n');
+
+        initClassFn = [
+            `local function ${this.INIT_CLASSES_FN}()`,
+            this.indent(classes),
+            'end'
+        ].join('\n');
+
+        code = code.concat(initClassFn);
+
+        // wrap the layers in a function
+        initLayerFn = [
+            `local function ${this.INIT_LAYERS_FN}()`,
+            this.indent(_.values(sections.layers).join('\n\n')),
+            'end'
+        ].join('\n');
+        code = code.concat(initLayerFn);
+
+        // Add operation fn definitions
+        code = code.concat(_.values(sections.operations));
+        code = code.concat(_.values(sections.pipelines));
+
+        // define deserializers, serializers
+        code.push(sections.deserializers);
+        code.push(sections.serializers);
+
+        code.push(this.getDeepforgeObject());
+        code.push('deepforge.initialize()');
+
+        code.push(sections.serializeOutputsDef);
+        return code.join('\n\n');
+    };
+
+    Export.prototype.getDeepforgeObject = function (content) {
+        content = content || {};
+        content.initCode = content.initCode || `${this.INIT_CLASSES_FN}()\n${'   '}${this.INIT_LAYERS_FN}()`;
+        return DeepForgeTpl(content);
+    };
+
+    return Export;
+});

src/plugins/Export/deepforge.ejs

@@ -0,0 +1,47 @@
+-- Instantiate the deepforge object
+deepforge = {}
+
+function deepforge.initialize()
+    require 'nn'
+    require 'rnn'
+    <%= initCode %>
+end
+
+-- Graph support
+torch.class('deepforge.Graph')
+
+function deepforge.Graph:__init(name)
+    -- nop
+end
+
+torch.class('deepforge._Line')
+
+function deepforge._Line:__init(graphId, name, opts)
+   -- nop
+end
+
+function deepforge._Line:add(x, y)
+   -- nop
+end
+
+function deepforge.Graph:line(name, opts)
+    return deepforge._Line(self.id, name, opts)
+end
+
+-- Image support
+function deepforge.image(name, tensor)
+   -- nop
+end
+
+torch.class('deepforge.Image')
+function deepforge.Image:__init(name, tensor)
+   -- nop
+end
+
+function deepforge.Image:update(tensor)
+   -- nop
+end
+
+function deepforge.Image:title(name)
+   -- nop
+end

src/plugins/Export/format.js

@@ -0,0 +1,13 @@
+
+/* globals define*/
+// The supported export formats and metadata
+define([
+    './formats/cli/cli'
+], function(
+    Format0
+) {
+
+    return {
+        'Basic CLI': Format0
+    };
+});

src/plugins/Export/format.js.ejs

@@ -0,0 +1,21 @@
+<% // Add default format
+formats.unshift({ name: 'cli', main: 'cli.js', displayName: 'Basic CLI' })
+%>
+/* globals define*/
+// The supported export formats and metadata
+define([
+<%= formats.map(function(format) {
+    return '    \'./formats/' + format.name + '/' +
+        path.basename(format.main.replace(/\.js$/, '')) + '\''
+    })
+    .join(',\n') %>
+], function(
+<%= formats.map(function(f, index) { return '    Format' + index; }).join(',\n') %>
+) {
+
+    return {
+<%= formats.map(function(f, index) {
+        return '        \'' + f.displayName + '\': Format' + index;
+    }).join(',\n') %>
+    };
+});

src/plugins/Export/formats/cli/cli.js

@@ -0,0 +1,103 @@
+/*globals define*/
+// Simple torch cli for the given pipeline
+define([
+], function(
+) {
+
+    var TOBOOLEAN = 
+`local function toboolean(str)
+    if str == 'true' then
+        return true
+    elseif str == 'false' then
+        return false
+    end
+end`;
+
+    var CliExporter = {};
+
+    CliExporter.deserializersFromString = function(sections) {
+        var hasBool = false;
+
+        // Add serializers given cli string input
+        Object.keys(this.isInputOp).forEach(id => {
+            var node = this.inputNode[id],
+                base = this.core.getBase(node),
+                type = this.core.getAttribute(base, 'name'),
+                name = this._nameFor[id];
+
+            if (type === 'boolean') {
+                hasBool = true;
+                sections.deserializerFor[name] = 'toboolean';
+            } else if (type === 'number') {
+                sections.deserializerFor[name] = 'tonumber';
+            } else if (type === 'string') {
+                sections.deserializerFor[name] = 'tostring';
+            }
+        });
+
+        if (hasBool) {
+            sections.deserializers += '\n' + TOBOOLEAN;
+        }
+
+        return sections;
+    };
+
+    CliExporter.main = function (sections, staticInputs) {
+        var code = [];
+
+        // Update deserializers for cli input
+        this.deserializersFromString(sections);
+
+        // Define all the operations, pipelines, etc
+        // 'getAllDefinitions' is provided as part of the public api
+        code.push(this.getAllDefinitions(sections));
+
+        // Command line specific stuff
+        var files = {},
+            main,
+            args,
+            staticNames = staticInputs.map(input => input.name),
+            varDefs,
+            index = 1;
+
+        // Create some names for the inputs
+        args = sections.pipelineInputNames.map(name => `${sections.deserializerFor[name]}(${name})`);
+
+        main = `local outputs = ${sections.pipelineName}(${args.join(', ')})`;
+
+        // Grab the args from the cli
+        code.push(sections.pipelineInputNames.map((name, index) => {
+            return `local ${name} = arg[${index + 1}]`;
+        }).join('\n'));
+
+        // Add the hash for each of the static inputs and reference them
+        staticInputs.forEach(input => {
+            files[`res/${input.name}`] = input.hash;
+        });
+
+        varDefs = staticNames.map(name => {
+            return `local ${name} = './res/${name}'`;
+        });
+
+        // Grab the remaining args from the cli
+        varDefs = varDefs.concat(sections.pipelineInputNames.map(name => {
+            if (!staticNames.includes(name)) {
+                return `local ${name} = arg[${index++}]`;
+            }
+        }));
+
+        // Add the main fn
+        code.push(varDefs.join('\n'));
+        code.push(main);
+
+        // Save outputs to disk
+        code.push(sections.serializeOutputs);
+
+        files['init.lua'] = code.join('\n\n');
+
+        // if no extra assets, just return the main file
+        return staticInputs.length ? files : files['init.lua'];
+    };
+
+    return CliExporter;
+});

src/plugins/Export/metadata.json

@@ -1,7 +1,7 @@
 {
-  "id": "GenerateExecFile",
-  "name": "Generate Execution File",
-  "version": "0.1.0",
+  "id": "Export",
+  "name": "Export",
+  "version": "1.0.0",
   "description": "",
   "icon": {
     "class": "glyphicon glyphicon-cog",

src/plugins/Export/toboolean.lua

@@ -0,0 +1,7 @@
+local function toboolean(str)
+    if str == 'true' then
+        return true
+    elseif str == 'false' then
+        return false
+    end
+end

src/plugins/GenerateArchitecture/GenerateArchitecture.js

@@ -6,13 +6,15 @@ define([
     'SimpleNodes/Constants',
     'deepforge/layer-args',
     'deepforge/utils',
+    'deepforge/Constants',
     'underscore',
     'text!./metadata.json'
 ], function (
     PluginBase,
-    Constants,
+    SimpleNodeConstants,
     createLayerDict,
     utils,
+    Constants,
     _,
     metadata
 ) {
@@ -47,6 +49,11 @@ define([
         this.LayerDict = createLayerDict(this.core, this.META);
         this.uniqueId = 2;
         this.varnames = {net: true};
+        this.definitions = [
+            'require \'nn\'',
+            'require \'rnn\''
+        ];
+
         return PluginBase.prototype.main.apply(this, arguments);
     };
 
@@ -65,15 +72,10 @@ define([
     };
 
     GenerateArchitecture.prototype.createOutputFiles = function (tree) {
-        var layers = tree[Constants.CHILDREN],
+        var layers = tree[SimpleNodeConstants.CHILDREN],
             result = {},
             code = '';
 
-        this.definitions = [
-            'require \'nn\'',
-            'require \'rnn\''
-        ];
-
         // Add an index to each layer
         layers.forEach((l, index) => l[INDEX] = index);
 
@@ -125,12 +127,61 @@ define([
     };
 
     GenerateArchitecture.prototype.createLayer = function (layer) {
-        var args = this.createArgString(layer);
-        return `nn.${layer.name}${args}`;
+        var args = this.createArgString(layer),
+            def = `nn.${layer.name}${args}`,
+            type = layer.base.base.name,
+            memberIds,
+            node,
+            name,
+            children,
+            id;
+
+        // Check if it is a container and has the 'addLayers' set
+        // If so, it should sort them by their registry 'index' and add
+        // each nested architecture's code to the given container
+        if (type === 'Container') {
+            // Get the members of the 'addLayers' set
+            memberIds = {};
+            id = layer[SimpleNodeConstants.NODE_PATH];
+            node = this._nodeCache[id];
+            this.core.getMemberPaths(node, Constants.CONTAINED_LAYER_SET)
+                .forEach(id => memberIds[id] = true);
+
+            // Get the (sorted) children
+            children = layer[SimpleNodeConstants.CHILDREN]
+                .map(child => {  // get (child, index) tuples
+                    var index = null;
+
+                    id = child[SimpleNodeConstants.NODE_PATH];
+                    if (memberIds[id]) {
+                        index = this.core.getMemberRegistry(node,
+                            Constants.CONTAINED_LAYER_SET, id, Constants.CONTAINED_LAYER_INDEX);
+                    }
+                    return [child, index];
+                })
+                .filter(pair => pair[1] !== null)  // remove non-members
+                .sort((a, b) => a[1] < b[1] ? -1 : 1)  // sort by 'index'
+                .map(pair => pair[0]);
+
+
+            var addedLayerDefs = '',
+                firstLayer;
+
+            for (var i = 0; i < children.length; i++) {
+                id = children[i][SimpleNodeConstants.NODE_PATH];
+                // Get the children!
+                firstLayer = children[i][SimpleNodeConstants.CHILDREN][0];
+                name = this.getVarName(utils.abbr(layer.name + '_' + i));
+                addedLayerDefs += this.createSequential(firstLayer, name).code;
+                def += `:add(${name})`;
+            }
+            this.hoist(addedLayerDefs);
+        }
+        return def;
     };
 
     GenerateArchitecture.prototype.createSequential = function (layer, name) {
-        var next = layer[Constants.NEXT][0],
+        var next = layer[SimpleNodeConstants.NEXT][0],
             args,
             snippet,
             snippets,
@@ -142,7 +193,7 @@ define([
 
         while (layer) {
             // if there is only one successor, just add the given layer
-            if (layer[Constants.PREV].length > 1) {  // sequential layers are over
+            if (layer[SimpleNodeConstants.PREV].length > 1) {  // sequential layers are over
                 next = layer;  // the given layer will be added by the caller
                 break;
             } else {  // add the given layer
@@ -151,11 +202,11 @@ define([
 
             }
 
-            while (layer && layer[Constants.NEXT].length > 1) {  // concat/parallel
+            while (layer && layer[SimpleNodeConstants.NEXT].length > 1) {  // concat/parallel
                 // if there is a fork, recurse and add a concat layer
 
-                this.logger.debug(`detected fork of size ${layer[Constants.NEXT].length}`);
-                snippets = layer[Constants.NEXT].map(nlayer =>
+                this.logger.debug(`detected fork of size ${layer[SimpleNodeConstants.NEXT].length}`);
+                snippets = layer[SimpleNodeConstants.NEXT].map(nlayer =>
                     this.createSequential(nlayer, this.getVarName('net')));
                 code += '\n' + snippets.map(snippet => snippet.code).join('\n');
 
@@ -183,7 +234,7 @@ define([
                                 `concat_${layer[INDEX]}:add(${snippet.name})`)
                             .join('\n') + `\n\n${name}:add(concat_${layer[INDEX]})`;
                         
-                        next = layer[Constants.NEXT][0];
+                        next = layer[SimpleNodeConstants.NEXT][0];
                     } else {
                         next = null;  // no next layers
                     }
@@ -203,7 +254,7 @@ define([
             }
 
             layer = next;
-            next = layer && layer[Constants.NEXT][0];
+            next = layer && layer[SimpleNodeConstants.NEXT][0];
         }
 
         return {
@@ -218,14 +269,14 @@ define([
         var content = layer[arg];
 
         if (typeof content === 'object') {  // layer as arg
-            if (content[Constants.CHILDREN].length) {
+            if (content[SimpleNodeConstants.CHILDREN].length) {
                 // Generate the code for the children of layer[arg]
                 var name = this.getVarName(utils.abbr(arg)),
                     layers;
 
                 this.logger.debug(`Adding layer arg for ${arg} (${layer.name})`);
                 try {
-                    layers = this.genRawArchCode(layer[arg][Constants.CHILDREN], name);
+                    layers = this.genRawArchCode(layer[arg][SimpleNodeConstants.CHILDREN], name);
                 } catch (e) {
                     this.logger.error(`Layer arg creation failed: ${e}`);
                     return null;
@@ -244,7 +295,7 @@ define([
     GenerateArchitecture.prototype.createArgString = function (layer) {
         var setters = this.LayerDict[layer.name].setters,
             setterNames = Object.keys(this.LayerDict[layer.name].setters),
-            base = layer[Constants.BASE],
+            base = layer[SimpleNodeConstants.BASE],
             desc,
             fn,
             layerCode,

src/plugins/GenerateExecFile/GenerateExecFile.js

@@ -1,344 +0,0 @@
-/*globals define, _*/
-/*jshint node:true, browser:true*/
-
-/**
- * Generated by PluginGenerator 1.7.0 from webgme on Sat Jun 04 2016 18:01:54 GMT-0500 (CDT).
- * A plugin that inherits from the PluginBase. To see source code documentation about available
- * properties and methods visit %host%/docs/source/PluginBase.html.
- */
-
-define([
-    'text!./metadata.json',
-    'plugin/PluginBase',
-    'deepforge/plugin/PtrCodeGen',
-    'q'
-], function (
-    pluginMetadata,
-    PluginBase,
-    PtrCodeGen,
-    Q
-) {
-    'use strict';
-
-    pluginMetadata = JSON.parse(pluginMetadata);
-    var HEADER_LENGTH = 60;
-
-    /**
-     * Initializes a new instance of GenerateExecFile.
-     * @class
-     * @augments {PluginBase}
-     * @classdesc This class represents the plugin GenerateExecFile.
-     * @constructor
-     */
-    var GenerateExecFile = function () {
-        // Call base class' constructor.
-        PluginBase.call(this);
-        this.pluginMetadata = pluginMetadata;
-
-        this._srcIdFor = {};  // input path -> output data node path
-
-        this._nameFor = {};  // input path -> opname
-        this._dataNameFor = {};  
-        this._opNames = {};
-
-        // topo sort stuff
-        this._nextOps = {};
-        this._incomingCnts = {};
-
-        this._operations = {};
-        this.activeNodeId = null;
-        this.activeNodeDepth = null;
-    };
-
-    /**
-     * Metadata associated with the plugin. Contains id, name, version, description, icon, configStructue etc.
-     * This is also available at the instance at this.pluginMetadata.
-     * @type {object}
-     */
-    GenerateExecFile.metadata = pluginMetadata;
-
-    // Prototypical inheritance from PluginBase.
-    GenerateExecFile.prototype = Object.create(PluginBase.prototype);
-    GenerateExecFile.prototype.constructor = GenerateExecFile;
-
-    /**
-     * Main function for the plugin to execute. This will perform the execution.
-     * Notes:
-     * - Always log with the provided logger.[error,warning,info,debug].
-     * - Do NOT put any user interaction logic UI, etc. inside this method.
-     * - callback always has to be called even if error happened.
-     *
-     * @param {function(string, plugin.PluginResult)} callback - the result callback
-     */
-    GenerateExecFile.prototype.main = function (callback) {
-        // Get all the children and call generate exec file
-        this.activeNodeId = this.core.getPath(this.activeNode);
-        this.activeNodeDepth = this.activeNodeId.split('/').length + 1;
-
-        if (this.isMetaTypeOf(this.activeNode, this.META.Execution)) {
-            this.activeNodeDepth++;
-        }
-
-        return this.core.loadChildren(this.activeNode)
-            .then(nodes => this.createExecFile(nodes))
-            .then(code => this.blobClient.putFile('init.lua', code))
-            .then(hash => {
-                this.result.addArtifact(hash);
-                this.result.setSuccess(true);
-                callback(null, this.result);
-            })
-            .fail(err => callback(err));
-    };
-
-    GenerateExecFile.prototype.createExecFile = function (children) {
-        // Convert opNodes' jobs to the nested operations
-        var opNodes,
-            nodes;
-
-        return this.unpackJobs(children)
-            .then(_nodes => {
-                nodes = _nodes;
-                opNodes = nodes
-                    .filter(node => this.isMetaTypeOf(node, this.META.Operation));
-                return Q.all(nodes.map(node => this.registerNameAndData(node)));
-            })
-            .then(() => Q.all(opNodes.map(node => this.createOperation(node))))
-            .then(operations => {
-                var nextIds = opNodes.map(n => this.core.getPath(n))
-                        .filter(id => !this._incomingCnts[id]);
-
-                operations.forEach(op => this._operations[op.id] = op);
-
-                // Toposort and concat!
-                return this.combineOpNodes(nextIds);
-            })
-            .fail(err => this.logger.error(err));
-    };
-
-    GenerateExecFile.prototype.unpackJobs = function (nodes) {
-        return Q.all(
-            nodes.map(node => {
-                if (!this.isMetaTypeOf(node, this.META.Job)) {
-                    return node;
-                }
-                return this.core.loadChildren(node)
-                    .then(children =>
-                        children.find(c => this.isMetaTypeOf(c, this.META.Operation))
-                    );
-            })
-        );
-    };
-
-    GenerateExecFile.prototype.combineOpNodes = function (opIds) {
-        var nextIds = [],
-            dstIds,
-            code,
-            id;
-
-        // Combine all nodes with incoming cnts of 0
-        code = opIds.map(id => this._operations[id].code).join('\n');
-
-        // Decrement all next ops
-        dstIds = opIds.map(id => this._nextOps[id])
-            .reduce((l1, l2) => l1.concat(l2), []);
-        for (var i = dstIds.length; i--;) {
-            id = dstIds[i];
-            if (--this._incomingCnts[id] === 0) {
-                nextIds.push(id);
-            }
-        }
-
-        // append
-        return [
-            code,
-            nextIds.length ? this.combineOpNodes(nextIds) : ''
-        ].join('\n');
-    };
-
-    GenerateExecFile.prototype.registerNameAndData = function (node) {
-        var name = this.core.getAttribute(node, 'name'),
-            id = this.core.getPath(node),
-            basename = name,
-            i = 2;
-
-        if (this.isMetaTypeOf(node, this.META.Operation)) {
-
-            // Get a unique operation name
-            while (this._opNames[name]) {
-                name = basename + '_' + i;
-                i++;
-            }
-
-            // register the unique name
-            this._opNames[name] = true;
-            this._nameFor[id] = name;
-
-            // For operations, register all output data node names by path
-            return this.core.loadChildren(node)
-                .then(cntrs => {
-                    var cntr = cntrs.find(n => this.isMetaTypeOf(n, this.META.Outputs));
-                    return this.core.loadChildren(cntr);
-                })
-                .then(outputs => {
-                    outputs.forEach(output => {
-                        var dataId = this.core.getPath(output);
-
-                        name = this.core.getAttribute(output, 'name');
-                        this._dataNameFor[dataId] = name;
-                    });
-                });
-
-        // For each input data node, register the associated output id
-        } else if (this.isMetaTypeOf(node, this.META.Transporter)) {
-            var outputData = this.core.getPointerPath(node, 'src'),
-                inputData = this.core.getPointerPath(node, 'dst'),
-                srcOpId = this.getOpIdFor(outputData),
-                dstOpId = this.getOpIdFor(inputData);
-
-            this._srcIdFor[inputData] = outputData;
-
-            // Store the next operation ids for the op id
-            if (!this._nextOps[srcOpId]) {
-                this._nextOps[srcOpId] = [];
-            }
-            this._nextOps[srcOpId].push(dstOpId);
-
-            // Increment the incoming counts for each dst op
-            this._incomingCnts[dstOpId] = this._incomingCnts[dstOpId] || 0;
-            this._incomingCnts[dstOpId]++;
-        }
-    };
-
-    GenerateExecFile.prototype.getOpIdFor = function (dataId) {
-        var ids = dataId.split('/'),
-            depth = ids.length;
-
-        ids.splice(this.activeNodeDepth - depth);
-        return ids.join('/');
-    };
-
-    // For each operation...
-    //   - unpack the inputs from prev ops
-    //   - add the attributes table (if used)
-    //     - check for '\<attributes\>' in code
-    //   - add the references
-    //     - generate the code
-    //     - replace the `return <thing>` w/ `<ref-name> = <thing>`
-    GenerateExecFile.prototype.createOperation = function (node) {
-        var id = this.core.getPath(node),
-            operation = {};
-
-        operation.name = this._nameFor[id];
-        operation.id = id;
-        operation.code = this.core.getAttribute(node, 'code');
-
-        // Update the 'code' attribute
-        // Change the last return statement to assign the results to a table
-        operation.code = this.assignResultToVar(operation.code,
-            `${operation.name}_results`);
-
-        // Get all the input names (and sources)
-        return this.core.loadChildren(node)
-            .then(containers => {
-                var inputs;
-
-                inputs = containers
-                    .find(cntr => this.isMetaTypeOf(cntr, this.META.Inputs));
-
-                this.logger.info(`${name} has ${containers.length} cntrs`);
-                return this.core.loadChildren(inputs);
-            })
-            .then(data => {
-                // Get the input names and sources
-                var inputNames = data.map(d => this.core.getAttribute(d, 'name')),
-                    ids = data.map(d => this.core.getPath(d)),
-                    srcIds = ids.map(id => this._srcIdFor[id]);
-
-                operation.inputs = inputNames.map((name, i) => {
-                    var id = srcIds[i],
-                        srcDataName = this._dataNameFor[id],
-                        srcOpId = this.getOpIdFor(id),
-                        srcOpName = this._nameFor[srcOpId];
-
-                    return `local ${name} = ${srcOpName}_results.${srcDataName}`;
-                });
-
-                return operation;
-
-            })
-            .then(operation => {
-
-                // For each reference, run the plugin and retrieve the generated code
-                operation.refNames = this.core.getPointerNames(node)
-                    .filter(name => name !== 'base');
-
-                var refs = operation.refNames
-                    .map(ref => [ref, this.core.getPointerPath(node, ref)]);
-
-                return Q.all(
-                    refs.map(pair => this.genPtrSnippet.apply(this, pair))
-                );
-            })
-            .then(codeFiles => {
-                operation.refs = codeFiles;
-                this.genOperationCode(operation);
-                return operation;
-            });
-    };
-
-    GenerateExecFile.prototype.genPtrSnippet = function (ptrName, pId) {
-        return this.getPtrCodeHash(pId)
-            .then(hash => this.blobClient.getObjectAsString(hash))
-            .then(code => this.createHeader(`creating ${ptrName}`, 40) + '\n' +
-                this.assignResultToVar(code, ptrName));
-    };
-
-    GenerateExecFile.prototype.createHeader = function (title, length) {
-        var len;
-        title = ` ${title} `;
-        length = length || HEADER_LENGTH;
-
-        len = Math.max(
-            Math.floor((length - title.length)/2),
-            2
-        );
-
-        return [
-            '',
-            title,
-            ''
-        ].join(new Array(len+1).join('-')) + '\n';
-
-    };
-
-    GenerateExecFile.prototype.genOperationCode = function (operation) {
-        var header = this.createHeader(`"${operation.name}" Operation`),
-            codeParts = [];
-
-        codeParts.push(header);
-
-        if (operation.inputs.length) {
-            codeParts.push(operation.inputs.join('\n'));
-        }
-
-        if (operation.refs.length) {
-            codeParts.push(operation.refs.join('\n'));
-        }
-
-        codeParts.push(operation.code);
-        codeParts.push('');
-        operation.code = codeParts.join('\n');
-        return operation;
-    };
-
-    GenerateExecFile.prototype.assignResultToVar = function (code, name) {
-        var i = code.lastIndexOf('return');
-        return code.substring(0, i) +
-            code.substring(i)
-                .replace('return', `local ${name} = `);
-    };
-
-    _.extend(GenerateExecFile.prototype, PtrCodeGen.prototype);
-
-    return GenerateExecFile;
-});

src/plugins/ImportArtifact/ImportArtifact.js

@@ -86,6 +86,7 @@ define([
                 });
 
                 this.core.setAttribute(dataNode, 'data', hash);
+                this.core.setAttribute(dataNode, 'createdAt', Date.now());
                 baseName = this.core.getAttribute(baseType, 'name');
 
                 var getName;

src/plugins/ImportTorch/nn.js

@@ -3,10 +3,12 @@
 define([
     'deepforge/layer-args',
     'common/util/assert',
+    'deepforge/Constants',
     'deepforge/lua'
 ], function(
     createLayerDict,
     assert,
+    Constants,
     lua
 ) {
     'use strict';
@@ -41,6 +43,20 @@ define([
             return '{' + strings.join(', ') + '}';
         };
 
+        var getAttributeString = function(value, layerType) {
+            if (value instanceof lua.types.LuaTable) {
+                if (value.get('_node')) {
+                    throw Error(`Detected unsupported varargs (composed of layers) for ${layerType}`);
+                }
+                return stringify(value);
+            } else if ((typeof value) === 'object') {
+                // special lua.js object
+                value = value.valueOf();
+            }
+
+            return value;
+        };
+
         var allConnectedTo = function(current) {
             var connectedIds = {},
                 node,
@@ -79,6 +95,7 @@ define([
                 connsFrom[id] = [];
             }
             connsFrom[id].push(conn, dst);
+            return conn;
         };
 
         // nn drawing library
@@ -104,7 +121,8 @@ define([
                 cntr,
                 layer,
                 cntrName,
-                value;
+                value,
+                i;
 
             if (this._cachedNode) {
                 // only generate a single node for each layer
@@ -117,7 +135,17 @@ define([
                 parent: parent
             });
 
-            for (var i = this._attrs.length; i--;) {
+            // merge all the last arguments into a single one (ie, assume the last
+            // attribute is varargs
+            if (this._attrs.length < this._values.length) {
+                i = this._attrs.length;
+                value = this._values.splice(i-1)
+                    .map(val => getAttributeString(val, this._base)).join(', ');
+                this._values.push(value);
+            }
+
+            // Add the attributes to the layer
+            for (i = this._attrs.length; i--;) {
                 name = this._attrs[i].name;
                 value = this._values[i];
 
@@ -169,56 +197,131 @@ define([
             return self;
         };
 
-        // Each container will have `inputs` and `outputs`
+        Layer.prototype._getAllNodes = function() {
+            return [this._node()];
+        };
+
         var Container = function() {
+            Layer.apply(this, arguments);
+            this._nestedIndex = 0;
+        };
+
+        Container.prototype = Object.create(Layer.prototype);
+
+        Container.prototype.add = function(self, tlayer) {
+            var layer = tlayer.get('_node'),
+                container = this._node(),
+                children,
+                arch;
+
+            // Add a nested 'Architecture' node
+            arch = core.createNode({
+                parent: container,
+                base: META.Architecture
+            });
+
+            // Add this node to the 'addLayers' set
+            core.addMember(container, Constants.CONTAINED_LAYER_SET, arch);
+            // Assign it an appropriate 'index' value
+            core.setMemberRegistry(
+                container,
+                Constants.CONTAINED_LAYER_SET,
+                core.getPath(arch),
+                Constants.CONTAINED_LAYER_INDEX,
+                this._nestedIndex++
+            );
+
+            // Move the added node(s)/conns to this architecture node
+            children = layer._getAllNodes();
+            for (var i = children.length; i--;) {
+                core.moveNode(children[i], arch);
+            }
+            layer._parent = arch;
+            return self;
+        };
+
+        // Implicit Containers are sequential and concat containers;
+        // these containers are visually implied in deepforge (although
+        // they are explicitly defined in torch)
+        var ImplicitContainer = function() {
             // inputs and outputs are webgme nodes
             this._inputs = [];
             this._outputs = [];
+            this._children = [];
+            this._connections = [];
         };
 
-        Container.prototype.add = function() {
+        // Implicit containers will have to record their 'children'.
+        // When an implicit container is added to an actual container,
+        // the container will set it's '_parent' value. If any additional
+        // layers are added to the implicit container after, they will
+        // need to be moved to the parent of the implicit container
+        ImplicitContainer.prototype.add = function() {
             logger.error('Add is not overridden!');
         };
 
-        var Sequential = function(/*attrs, args*/) {
-            Container.call(this);
+        ImplicitContainer.prototype._getAllNodes = function() {
+            var nodes = this._children.map(layer => layer._getAllNodes())
+                .reduce((l1, l2) => l1.concat(l2), []);
+
+            return this._connections.concat(nodes);
         };
 
-        Sequential.prototype = new Container();
+        var Sequential = function(/*attrs, args*/) {
+            ImplicitContainer.call(this);
+        };
+
+        Sequential.prototype = new ImplicitContainer();
 
         Sequential.prototype.add = function(self, tlayer) {
             var layer = tlayer.get('_node'),
-                nodes = layer._inputs;
+                nodes = layer._inputs,
+                connections = [];
 
             // If this._inputs is empty, add the layer to the inputs list
             if (this._inputs.length === 0) {  // first node
                 this._inputs = this._inputs.concat(nodes);
             } else {
                 // connect all inputs of the added node to the current outputs
+                // add the connection to the list of allNodes
                 this._outputs.forEach(src =>
-                    nodes.forEach(dst => connect(src, dst))
+                    nodes.forEach(dst => connections.push(connect(src, dst)))
                 );
             }
             this._outputs = layer._outputs;
+            this._children.push(layer);
+            this._connections = this._connections.concat(connections);
+
+            // If _parent is set, move the nodes and connection to the _parent node
+            if (this._parent) {
+                nodes = layer._getAllNodes().concat(connections);
+                for (var i = nodes.length; i--;) {
+                    core.moveNode(nodes[i], this._parent);
+                }
+            }
             return self;
         };
 
         var Concat = function(attrs, args) {
-            Container.call(this);
+            ImplicitContainer.call(this);
 
             // Create a concat node and add it to this._outputs
             var concat = new Layer('Concat', attrs, args);
             this._outputs.push(concat._node());
+            this._children.push(concat);
         };
 
-        Concat.prototype = new Container();
+        Concat.prototype = new ImplicitContainer();
 
         Concat.prototype.add = function(self, tlayer) {
             // Connect the tlayer outputs to this._outputs
             var layer = tlayer.get('_node'),
-                concatLayer = this._outputs[0];
+                concatLayer = this._outputs[0],
+                connections = [],
+                nodes;
 
-            layer._outputs.forEach(output => connect(output, concatLayer));
+            layer._outputs.forEach(output =>
+                connections.push(connect(output, concatLayer)));
 
             // Connect the incomingly connected node to tlayer
             // TODO: This might not work if adding layers after this container is
@@ -226,14 +329,23 @@ define([
 
             // Add the layer's inputs to the inputs
             this._inputs = this._inputs.concat(layer._inputs);
+            this._children.push(layer);
+            this._connections = this._connections.concat(connections);
+            if (this._parent) {
+                nodes = layer._getAllNodes().concat(connections);
+                for (var i = nodes.length; i--;) {
+                    core.moveNode(nodes[i], this._parent);
+                }
+            }
             return self;
         };
 
         // Special layers (with special functions - like 'add')
-        var LAYERS = {
-            Concat: Concat,
-            Sequential: Sequential
-        };
+        var CONTAINERS,
+            LAYERS = {
+                Concat: Concat,
+                Sequential: Sequential
+            };
 
         var getValue = function(txt) {
             if (txt === 'true') {
@@ -286,6 +398,9 @@ define([
 
             if (LAYERS[type]) {
                 node = new LAYERS[type](args, attrs);
+            } else if (CONTAINERS[type]) {
+                node = new Container(type, args, attrs);
+                res.set('add', node.add.bind(node)); // add the 'add' method
             } else {  // Call generic Layer with type name
                 node = new Layer(type, args, attrs);
             }
@@ -327,11 +442,23 @@ define([
                 return;
             }
 
-            // TODO: Create the nn object
+            // Mocking the nn layers (as defined in the metamodel)
             var nn = lua.newContext()._G,
-                names = Object.keys(LayerDict);
+                names = Object.keys(LayerDict),
+                base,
+                baseName;
+
+            // For each layer, check the name of the base type. If it is 'Container',
+            // then it should be added to the CONTAINERS dictionary. This will change how
+            // it is handled in 'CreateLayer'
+            CONTAINERS = {};
 
             for (var i = names.length; i--;) {
+                base = core.getBase(META[names[i]]);
+                baseName = core.getAttribute(base, 'name');
+                if (baseName === 'Container') {
+                    CONTAINERS[names[i]] = true;
+                }
                 nn.set(names[i], CreateLayer.bind(null, names[i]));
             }
 

src/plugins/UpdateLibrarySeed/UpdateLibrarySeed.js

@@ -84,19 +84,27 @@ define([
             .fail(err => callback(err, this.result));
     };
 
+    UpdateLibrarySeed.prototype.bumpVersion = function (rawVersion, releaseType) {
+        var vnames = ['major', 'minor', 'patch'],
+            bumpIndex = vnames.indexOf(releaseType),
+            version = rawVersion.split('.').map(num => parseInt(num));
+
+        version[bumpIndex]++;
+        // zero out the smaller numbers
+        while (++bumpIndex < version.length) {
+            version[bumpIndex] = 0;
+        }
+        return version.join('.');
+    };
+
     UpdateLibrarySeed.prototype.getLibraryVersion = function () {
         var version,
             config = this.getCurrentConfig(),
-            vnames = ['major', 'minor', 'patch'],
-            bumpIndex,
             newVersion;
 
-        version = (this.core.getAttribute(this.rootNode, 'version') || '0.0.0')
-            .split('.').map(num => parseInt(num));
-        bumpIndex = vnames.indexOf(config.releaseType);
-        version[bumpIndex]++;
+        version = (this.core.getAttribute(this.rootNode, 'version') || '0.0.0');
+        newVersion = this.bumpVersion(version, config.releaseType);
 
-        newVersion = version.join('.');
         this.core.setAttribute(this.rootNode, 'version', newVersion);
         return this.save(`Bumped version to ${newVersion}`).then(() => newVersion);
     };
@@ -116,6 +124,10 @@ define([
         this.logger.info(`Updating ${seedName} seed`);
         job.on('error', _err => {
             err = _err;
+            if (err.code === 'ENOENT') {
+                return deferred.reject('"webgme" command not found. Is webgme-cli installed on the server?');
+            }
+            return deferred.reject(err);
         });
 
         job.on('exit', code => {

src/plugins/ValidateArchitecture/ValidateArchitecture.js

@@ -0,0 +1,194 @@
+/*globals define*/
+/*jshint node:true, browser:true*/
+
+define([
+    'plugin/GenerateArchitecture/GenerateArchitecture/GenerateArchitecture',
+    'SimpleNodes/Constants',
+    'text!./metadata.json',
+    'q',
+    'fs',
+    'path',
+    'child_process',
+    'rimraf'
+], function (
+    PluginBase,
+    SimpleNodeConstants,
+    pluginMetadata,
+    Q,
+    fs,
+    path,
+    childProcess,
+    rm_rf
+) {
+    'use strict';
+
+    pluginMetadata = JSON.parse(pluginMetadata);
+
+    /**
+     * Initializes a new instance of ValidateArchitecture.
+     * @class
+     * @augments {PluginBase}
+     * @classdesc This class represents the plugin ValidateArchitecture.
+     * @constructor
+     */
+    var TMP_DIR = '/tmp',
+        spawn = childProcess.spawn,
+        GET_ARG_INDEX = /argument #([0-9]+) to/,
+        TORCH_INSTALLED = true;
+
+    var ValidateArchitecture = function () {
+        // Call base class' constructor.
+        PluginBase.call(this);
+        this.pluginMetadata = pluginMetadata;
+    };
+
+    /**
+     * Metadata associated with the plugin. Contains id, name, version, description, icon, configStructue etc.
+     * This is also available at the instance at this.pluginMetadata.
+     * @type {object}
+     */
+    ValidateArchitecture.metadata = pluginMetadata;
+
+    // Prototypical inheritance from PluginBase.
+    ValidateArchitecture.prototype = Object.create(PluginBase.prototype);
+    ValidateArchitecture.prototype.constructor = ValidateArchitecture;
+
+    ValidateArchitecture.prototype.main = function (callback) {
+        var name = this.core.getAttribute(this.activeNode, 'name');
+
+        this._callback = callback;
+        // make the tmp dir
+        this._tmpFileId = path.join(TMP_DIR, `${name}_${Date.now()}`);
+        fs.mkdir(this._tmpFileId, err => {
+            if (err) throw err;
+            return PluginBase.prototype.main.call(this, callback);
+        });
+    };
+
+    ValidateArchitecture.prototype.createOutputFiles = function (tree) {
+        var layers = tree[SimpleNodeConstants.CHILDREN],
+            tests = [],
+            id;
+
+        if (!TORCH_INSTALLED) {
+            return this.validationFinished();
+        }
+
+        // Generate code for each layer
+        this.layerName = {};
+        for (var i = layers.length; i--;) {
+            id = layers[i][SimpleNodeConstants.NODE_PATH];
+            this.layerName[id] = layers[i].name;
+            tests.push([id, this.createLayerTestCode(layers[i])]);
+        }
+
+        // Run each code snippet
+        this.validateLayers(tests)
+            .then(errors => this.validationFinished(errors))
+            .fail(err => this.logger.error(`validation failed: ${err}`));
+    };
+
+    ValidateArchitecture.prototype.validationFinished = function (errors) {
+        if (!TORCH_INSTALLED) {
+            this.logger.warn('Torch is not installed. Architecture validation is not supported.');
+        } else {
+            this.logger.info(`found ${errors.length} validation errors`);
+        }
+
+        this.createMessage(null, {
+            errors: TORCH_INSTALLED ? errors : null
+        });
+        this.result.setSuccess(true);
+        this._callback(null, this.result);
+    };
+
+    ValidateArchitecture.prototype.createLayerTestCode = function (layer) {
+        var customLayerDefs = this.genLayerDefinitions([layer]);
+
+        return this.definitions.concat([
+            customLayerDefs,
+            this.createLayer(layer)
+        ]).join('\n');
+    };
+
+    ValidateArchitecture.prototype.validateLayers = function (layerTests) {
+        return Q.all(layerTests.map(layer => this.validateLayer(layer[0], layer[1])))
+            .then(results => Q.nfcall(rm_rf, this._tmpFileId)
+                .then(() => results.filter(result => !!result))
+            );
+    };
+
+    ValidateArchitecture.prototype.validateLayer = function (id, code) {
+        var deferred = Q.defer(),
+            tmpPath = path.join(this._tmpFileId, id.replace(/[^a-zA-Z\d]+/g, '_'));
+
+        if (!TORCH_INSTALLED) {
+            deferred.resolve(null);
+        } else {
+            // Write to a temp file
+            fs.writeFile(tmpPath, code, err => {
+                var job,
+                    stderr = '',
+                    stdout = '';
+
+                if (err) {
+                    return deferred.reject(`Could not create tmp file at ${tmpPath}: ${err}`);
+                }
+                // Run the file
+                job = spawn('th', [tmpPath]);
+                job.stderr.on('data', data => stderr += data.toString());
+                job.stdout.on('data', data => stdout += data.toString());
+                job.on('error', err => {
+                    if (err.code === 'ENOENT') {
+                        TORCH_INSTALLED = false;
+                    }
+                });
+                job.on('close', code => {
+                    if (code === 0) {
+                        deferred.resolve(null);
+                    } else {
+                        // If it errored, clean the error and return it
+                        deferred.resolve(this.parseError(id, stderr));
+                    }
+                });
+            });
+        }
+
+        return deferred.promise;
+    };
+
+    ValidateArchitecture.prototype.parseError = function (id, stderr) {
+        var msg = stderr
+            .split('\n').shift()  // first line
+            .replace(/^[^:]*: /, '')  // remove the file path
+            .replace(/ at [^ ]*\)/, ')')  // remove last line number
+            .replace(/ to '\?'/, '');  // remove unknown symbol
+
+        // convert 'bad argument #[num]' to the argument name
+        if (msg.indexOf('bad argument') === 0) {
+            var layerName = this.layerName[id],
+                args = this.LayerDict[layerName].args,
+                argIndex = +(stderr.match(GET_ARG_INDEX)[1]),
+                argName = args[argIndex-1].name;
+
+            // FIXME: This is not the correct index...
+            // This is the index for the incorrect argument passed to the
+            // tensor...
+            msg = msg.replace(`#${argIndex}`, `"${argName}"`);
+        }
+
+        return {
+            id: id,
+            msg: msg
+        };
+    };
+
+    ValidateArchitecture.prototype._saveOutput = function () {};
+
+    // for testing
+    ValidateArchitecture.prototype.setTorchInstalled = function (value) {
+        TORCH_INSTALLED = !!value;
+    };
+
+    return ValidateArchitecture;
+});

src/plugins/ValidateArchitecture/metadata.json

@@ -0,0 +1,14 @@
+{
+  "id": "ValidateArchitecture",
+  "name": "ValidateArchitecture",
+  "version": "0.1.0",
+  "description": "",
+  "icon": {
+    "class": "glyphicon glyphicon-cog",
+    "src": ""
+  },
+  "disableServerSideExecution": false,
+  "disableBrowserSideExecution": true,
+  "writeAccessRequired": false,
+  "configStructure": []
+}

src/routers/ExecPulse/ExecPulse.js

@@ -0,0 +1,131 @@
+/*jshint node:true*/
+
+// This is a REST endpoint keeping track of the heartbeats of each execution. This
+// allows detection of "disconnected" executions (enabling the reconnection of the
+// executions - issue #821)
+'use strict';
+
+var express = require('express'),
+    MONGO_COLLECTION = 'ExecPulse',
+    CONSTANTS = require('../../common/Constants').PULSE,
+    mongo,
+    storage,
+    router = express.Router();
+
+/**
+ * Called when the server is created but before it starts to listening to incoming requests.
+ * N.B. gmeAuth, safeStorage and workerManager are not ready to use until the start function is called.
+ * (However inside an incoming request they are all ensured to have been initialized.)
+ *
+ * @param {object} middlewareOpts - Passed by the webgme server.
+ * @param {GmeConfig} middlewareOpts.gmeConfig - GME config parameters.
+ * @param {GmeLogger} middlewareOpts.logger - logger
+ * @param {function} middlewareOpts.ensureAuthenticated - Ensures the user is authenticated.
+ * @param {function} middlewareOpts.getUserId - If authenticated retrieves the userId from the request.
+ * @param {object} middlewareOpts.gmeAuth - Authorization module.
+ * @param {object} middlewareOpts.safeStorage - Accesses the storage and emits events (PROJECT_CREATED, COMMIT..).
+ * @param {object} middlewareOpts.workerManager - Spawns and keeps track of "worker" sub-processes.
+ */
+function initialize(middlewareOpts) {
+    var logger = middlewareOpts.logger.fork('ExecPulse'),
+        ensureAuthenticated = middlewareOpts.ensureAuthenticated,
+        STALE_THRESHOLD = 7500;
+
+    storage = require('../storage')(logger, middlewareOpts.gmeConfig);
+    logger.debug('initializing ...');
+
+    // Ensure authenticated can be used only after this rule.
+    router.use('*', function (req, res, next) {
+        res.setHeader('X-WebGME-Media-Type', 'webgme.v1');
+        next();
+    });
+
+    // Use ensureAuthenticated if the routes require authentication. (Can be set explicitly for each route.)
+    router.use('*', ensureAuthenticated);
+
+    router.get('/', function (req, res) {
+        mongo.find().toArray((err, all) => {
+            if (err) {
+                return res.status(500).send(err);
+            }
+            res.json(all.map(entry => {
+                delete entry._id;
+                return entry;
+            }));
+        });
+    });
+
+    router.get('/:hash', function (req, res) {
+        // Check if the given job is alive (has a valid heartbeat).
+        // If the data doesn't exist, then it is considered alive
+        if (!req.params.hash) {
+            return res.status(400).send('Missing hash');
+        }
+
+        logger.debug('getting pulse of ', req.params.hash);
+        mongo.findOne({hash: req.params.hash})
+            .then(job => {
+                var current = Date.now(),
+                    result = CONSTANTS.DOESNT_EXIST;
+
+                if (job) {
+                    result = (current - job.timestamp) < STALE_THRESHOLD ?
+                        CONSTANTS.ALIVE : CONSTANTS.DEAD;
+                }
+                return res.status(200).send(result.toString());
+            });
+    });
+
+    router.post('/:hash', function (req, res) {
+        var timestamp = Date.now(),
+            job = {
+                hash: req.params.hash,
+                timestamp: timestamp
+            };
+
+        // Validate the input
+        logger.debug('Received heartbeat for ', job.hash);
+        if (!job.hash) {
+            return res.status(400).send('Missing hash');
+        }
+
+        // Delete the given job from the database
+        mongo.update({hash: job.hash}, job, {upsert: true})
+            .then(() => res.sendStatus(201));
+    });
+
+    router.delete('/:hash', function (req, res) {
+        // Delete the given job from the database
+        return mongo.findOneAndDelete({hash: req.params.hash})
+            .then(() => res.sendStatus(204));
+    });
+
+    logger.debug('ready');
+}
+
+/**
+ * Called before the server starts listening.
+ * @param {function} callback
+ */
+function start(callback) {
+    storage.then(db => {
+        mongo = db.collection(MONGO_COLLECTION);
+        callback();
+    });
+}
+
+/**
+ * Called after the server stopped listening.
+ * @param {function} callback
+ */
+function stop(callback) {
+    callback();
+}
+
+
+module.exports = {
+    initialize: initialize,
+    router: router,
+    start: start,
+    stop: stop
+};

src/routers/JobLogsAPI/JobLogManager.js

@@ -71,7 +71,7 @@ JobLogManager.prototype.migrate = function(migrationInfo, jobIds) {
     }
 
     // Copy the job files and evaluate each of the finish functions
-    this.logger.info('migrating from ' + migrationInfo.srcBranch + ' to '+ migrationInfo.dstBranch);
+    this.logger.debug('migrating from ' + migrationInfo.srcBranch + ' to '+ migrationInfo.dstBranch);
     return Q.all(jobIds.map(jobId => {
         src = this._getFilePath({
             project: migrationInfo.project,
@@ -112,7 +112,7 @@ JobLogManager.prototype.appendTo = function(jobInfo, logs) {
         branchDirname = path.dirname(filename),
         projDirname = path.dirname(branchDirname);
 
-    this.logger.info(`Appending content to ${filename}`);
+    this.logger.debug(`Appending content to ${filename}`);
     // Make directory if needed
     return this.mkdirIfNeeded(this.rootDir)
         .then(() => this.mkdirIfNeeded(projDirname))
@@ -142,7 +142,7 @@ JobLogManager.prototype.delete = function(jobInfo) {
                 this.logger.debug(`Removing file ${filename}`);
                 return Q.nfcall(fs.unlink, filename);
             }
-            this.logger.info(`${filename} doesn't exist. No need to delete...`);
+            this.logger.debug(`${filename} doesn't exist. No need to delete...`);
         });
 };
 

src/routers/JobLogsAPI/JobLogsAPI.js

@@ -4,7 +4,10 @@
 
 var express = require('express'),
     JobLogManager = require('./JobLogManager'),
-    router = express.Router();
+    MONGO_COLLECTION = 'JobLogsMetadata',
+    mongo,
+    router = express.Router(),
+    storage;
 
 /**
  * Called when the server is created but before it starts to listening to incoming requests.
@@ -27,6 +30,7 @@ function initialize(middlewareOpts) {
         logManager = new JobLogManager(logger, gmeConfig);
 
     logger.debug('initializing ...');
+    storage = require('../storage')(logger, gmeConfig);
 
     // Ensure authenticated can be used only after this rule.
     router.use('*', function (req, res, next) {
@@ -40,29 +44,72 @@ function initialize(middlewareOpts) {
 
     router.get('/:project/:branch/:job', function (req, res/*, next*/) {
         // Retrieve the job logs for the given job
-        logManager.getLog(req.params).then(log => {
-            res.set('Content-Type', 'text/plain');
-            res.send(log);
-        });
+        logger.info(`Requested logs for ${req.params.job} in ${req.params.project}`);
+        logManager.getLog(req.params)
+            .then(log => {
+                res.set('Content-Type', 'text/plain');
+                res.send(log);
+            })
+            .catch(err => logger.error(`Log retrieval failed: ${err}`));
+    });
+
+    router.get('/metadata/:project/:branch/:job', function (req, res/*, next*/) {
+        logger.info(`Requested metadata for ${req.params.job} in ${req.params.project}`);
+        return mongo.findOne(req.params)
+            .then(info => {
+                var lineCount = info ? info.lineCount : -1;
+
+                return res.json({
+                    lineCount: lineCount
+                });
+            })
+            .catch(err => logger.error(`Metadata retrieval failed: ${err}`));
     });
 
     router.patch('/:project/:branch/:job', function (req, res/*, next*/) {
         var logs = req.body.patch;
         logger.info(`Received append request for ${req.params.job} in ${req.params.project}`);
-        logManager.appendTo(req.params, logs)
-            .then(() => res.send('Append successful'))
+        return logManager.appendTo(req.params, logs)
+            .then(() => {
+                if (req.body.lineCount || req.body.cmdCount || req.body.createdIds) {
+                    var info = {
+                        project: req.params.project,
+                        branch: req.params.branch,
+                        job: req.params.job,
+                        lineCount: req.body.lineCount || -1,
+                        createdIds: req.body.createdIds || [],
+                        cmdCount: req.body.cmdCount || 0
+                    };
+                    logger.debug('metadata is', info);
+                    return mongo.update(req.params, info, {upsert: true})
+                        .then(() => res.send('Append successful'));
+                } else {
+                    res.send('Append successful');
+                }
+            })
             .catch(err => logger.error(`Append failed: ${err}`));
     });
 
     router.delete('/:project/:branch/:job', function (req, res/*, next*/) {
-        logManager.delete(req.params).then(() => res.send('delete successful'));
+        logger.info(`Request to delete logs for ${req.params.job} in ${req.params.project}`);
+        logManager.delete(req.params)
+            .then(() => mongo.findOneAndDelete(req.params))
+            .then(() => {
+                logger.info('Job log deletion successful!');
+                res.status(204).send('delete successful');
+            })
+            .catch(err => logger.error(`Job log deletion failed: ${err}`));
     });
 
     router.post('/migrate/:project/:srcBranch/:dstBranch', function (req, res/*, next*/) {
         var jobs = req.body.jobs;
+        logger.info(`Migrating logs from ${req.params.srcBranch} to ${req.params.dstBranch} in ${req.params.project}`);
         logManager.migrate(req.params, jobs)
-            .then(() => res.send('migration successful'))
-            .fail(err => logger.error(err));
+            .then(() => {
+                logger.info('Log migration successful!');
+                res.send('migration successful');
+            })
+            .fail(err => logger.error(`migration failed: ${err}`));
     });
 
     logger.debug('ready');
@@ -73,7 +120,11 @@ function initialize(middlewareOpts) {
  * @param {function} callback
  */
 function start(callback) {
-    callback();
+    storage.then(db => {
+        mongo = db.collection(MONGO_COLLECTION);
+        callback();
+    });
+
 }
 
 /**

src/routers/JobOriginAPI/JobOriginAPI.js

@@ -0,0 +1,162 @@
+/*jshint node:true*/
+
+'use strict';
+
+var express = require('express'),
+    MONGO_COLLECTION = 'JobOrigins',
+    utils = require('../utils'),
+    mongo,
+    router = express.Router(),
+    storage;
+
+/**
+ * Called when the server is created but before it starts to listening to incoming requests.
+ * N.B. gmeAuth, safeStorage and workerManager are not ready to use until the start function is called.
+ * (However inside an incoming request they are all ensured to have been initialized.)
+ *
+ * @param {object} middlewareOpts - Passed by the webgme server.
+ * @param {GmeConfig} middlewareOpts.gmeConfig - GME config parameters.
+ * @param {GmeLogger} middlewareOpts.logger - logger
+ * @param {function} middlewareOpts.ensureAuthenticated - Ensures the user is authenticated.
+ * @param {function} middlewareOpts.getUserId - If authenticated retrieves the userId from the request.
+ * @param {object} middlewareOpts.gmeAuth - Authorization module.
+ * @param {object} middlewareOpts.safeStorage - Accesses the storage and emits events (PROJECT_CREATED, COMMIT..).
+ * @param {object} middlewareOpts.workerManager - Spawns and keeps track of "worker" sub-processes.
+ */
+// When testing, use in memory storage...
+function initialize(middlewareOpts) {
+    var logger = middlewareOpts.logger.fork('JobOriginAPI'),
+        gmeConfig = middlewareOpts.gmeConfig,
+        ensureAuthenticated = middlewareOpts.ensureAuthenticated,
+        REQUIRED_FIELDS = ['hash', 'project', 'execution', 'job', 'nodeId', 'branch'];
+
+    storage = require('../storage')(logger, gmeConfig);
+
+    logger.debug('initializing ...');
+    // Ensure authenticated can be used only after this rule.
+    router.use('*', function (req, res, next) {
+        // This header ensures that any failures with authentication won't redirect.
+        res.setHeader('X-WebGME-Media-Type', 'webgme.v1');
+        next();
+    });
+
+    // Use ensureAuthenticated if the routes require authentication. (Can be set explicitly for each route.)
+    router.use('*', ensureAuthenticated);
+
+    // Connect to mongo...
+
+    router.get('/', function (req, res) {
+        mongo.find().toArray((err, all) => {
+            if (err) {
+                return res.status(500).send(err);
+            }
+            res.json(all.map(entry => {
+                delete entry._id;
+                return entry;
+            }));
+        });
+    });
+
+    router.get('/:jobHash', function (req, res/*, next*/) {
+        var hash = req.params.jobHash,
+            jobInfo = {};
+
+        mongo.findOne({hash: hash})
+            .then(result => {
+                if (result) {
+                    // Filter the result object
+                    for (var i = REQUIRED_FIELDS.length; i--;) {
+                        jobInfo[REQUIRED_FIELDS[i]] = result[REQUIRED_FIELDS[i]];
+                    }
+                    return res.json(jobInfo);
+                }
+                res.sendStatus(404);
+            })
+            .catch(err => {
+                logger.error(`Storing job info failed: ${err}`);
+                res.status(500).send(err);
+            });
+    });
+
+    router.post('/:jobHash', function (req, res/*, next*/) {
+        var hash = req.params.jobHash,
+            jobInfo = {
+                hash: hash,
+                project: req.body.project,
+                execution: req.body.execution,
+                branch: req.body.branch,
+                job: req.body.job,  // job name
+                nodeId: req.body.nodeId
+            };
+
+        // Check that none of the fields are undefined
+        var missing = utils.getMissingField(jobInfo, REQUIRED_FIELDS);
+        if (missing) {
+            return res.status(400).send(`Missing required field: ${missing}`);
+        }
+
+        logger.debug(`Storing job info for ${hash}`);
+        return mongo.insertOne(jobInfo)
+            .then(() => res.sendStatus(201))
+            .catch(err => {
+                logger.error(`Storing job info failed: ${err}`);
+                res.status(500).send(err.toString());
+            });
+    });
+
+    router.delete('/:jobHash', function (req, res/*, next*/) {
+        var hash = req.params.jobHash;
+
+        mongo.findOneAndDelete({hash: hash})
+            .then(() => res.sendStatus(204));
+    });
+
+    // on fork
+    router.patch('/:jobHash', function (req, res) {
+        var hash = req.params.jobHash;
+
+        if (!req.body.branch) {
+            return res.status(400).send('Missing "branch" field');
+        }
+
+        return mongo.findOneAndUpdate({hash: hash}, {$set: {branch: req.body.branch}})
+            .then(() => {
+                logger.debug('Finished updateOne!');
+                res.sendStatus(200);
+            })
+            .catch(err => {
+                logger.error(`Job update failed: ${err}`);
+                res.status(500).send(err);
+            });
+    });
+
+    logger.debug('ready');
+}
+
+/**
+ * Called before the server starts listening.
+ * @param {function} callback
+ */
+function start(callback) {
+    storage.then(db => {
+        mongo = db.collection(MONGO_COLLECTION);
+        callback();
+    });
+
+}
+
+/**
+ * Called after the server stopped listening.
+ * @param {function} callback
+ */
+function stop(callback) {
+    callback();
+}
+
+
+module.exports = {
+    initialize: initialize,
+    router: router,
+    start: start,
+    stop: stop
+};

src/routers/storage.js

@@ -0,0 +1,19 @@
+// Get a mongodb connection
+
+var mongodb = require('mongodb'),
+    connection;
+
+module.exports = function(logger, gmeConfig) {
+    if (!connection) {
+        connection = mongodb.MongoClient.connect(gmeConfig.mongo.uri, gmeConfig.mongo.options)
+            .then(db => {
+                logger.debug('Connected to mongo!');
+                return db;
+            })
+            .catch(err => {
+                logger.error(`Could not connect to mongo: ${err}`);
+                throw err;
+            });
+    }
+    return connection;
+};

src/routers/utils.js

@@ -0,0 +1,10 @@
+module.exports = {
+    getMissingField: function(array, fields) {
+        for (var i = fields.length; i--;) {
+            if (!array[fields[i]]) {
+                return fields[i];
+            }
+        }
+        return null;
+    }
+};

src/seeds/nn/version.txt

@@ -1 +1 @@
-0.2.0
+0.5.0

src/seeds/pipeline/version.txt

@@ -1 +1 @@
-0.3.0
+0.6.0

src/visualizers/Visualizers.json

@@ -31,8 +31,8 @@
   },
   {
     "id": "RootViz",
-    "title": "MainView",
-    "panel": "panels/MainView/MainViewPanel",
+    "title": "ForwardViz",
+    "panel": "panels/ForwardViz/ForwardVizPanel",
     "DEBUG_ONLY": false
   },
   {
@@ -71,6 +71,30 @@
     "panel": "panels/PipelineIndex/PipelineIndexPanel",
     "DEBUG_ONLY": false
   },
+  {
+    "id": "ArchitectureIndex",
+    "title": "ArchitectureIndex",
+    "panel": "panels/ArchIndex/ArchIndexPanel",
+    "DEBUG_ONLY": false
+  },
+  {
+    "id": "OperationIndex",
+    "title": "OperationIndex",
+    "panel": "js/Panels/ModelEditor/ModelEditorPanel",
+    "DEBUG_ONLY": false
+  },
+  {
+    "id": "LayerIndex",
+    "title": "LayerIndex",
+    "panel": "js/Panels/ModelEditor/ModelEditorPanel",
+    "DEBUG_ONLY": false
+  },
+  {
+    "id": "DataTypeIndex",
+    "title": "DataTypeIndex",
+    "panel": "js/Panels/ModelEditor/ModelEditorPanel",
+    "DEBUG_ONLY": false
+  },
   {
     "id": "JobEditor",
     "title": "JobEditor",
@@ -100,5 +124,29 @@
     "title": "ExecutionIndex",
     "panel": "panels/ExecutionIndex/ExecutionIndexPanel",
     "DEBUG_ONLY": false
+  },
+  {
+    "id": "WorkerHeader",
+    "title": "WorkerHeader",
+    "panel": "panels/WorkerHeader/WorkerHeaderPanel",
+    "DEBUG_ONLY": false
+  },
+  {
+    "id": "ArtifactIndex",
+    "title": "ArtifactIndex",
+    "panel": "panels/ArtifactIndex/ArtifactIndexPanel",
+    "DEBUG_ONLY": false
+  },
+  {
+    "id": "ArchIndex",
+    "title": "ArchIndex",
+    "panel": "panels/ArchIndex/ArchIndexPanel",
+    "DEBUG_ONLY": false
+  },
+  {
+    "id": "ForwardViz",
+    "title": "ForwardViz",
+    "panel": "panels/ForwardViz/ForwardVizPanel",
+    "DEBUG_ONLY": false
   }
-]
+]

src/visualizers/panels/ArchEditor/ArchEditorControl.js

@@ -4,27 +4,30 @@
 define([
     'deepforge/Constants',
     'deepforge/globals',
-    'panels/EasyDAG/EasyDAGControl',
+    'deepforge/viz/panels/ThumbnailControl',
     'js/NodePropertyNames',
     'js/Utils/ComponentSettings',
-    'underscore'
+    'underscore',
+    'q'
 ], function (
     Constants,
     DeepForge,
-    EasyDAGControl,
+    ThumbnailControl,
     nodePropertyNames,
     ComponentSettings,
-    _
+    _,
+    Q
 ) {
 
     'use strict';
 
     var ArchEditorControl,
         DEFAULT_CONFIG = {
-            DefaultColor: '#ffb74d',
+            DefaultColor: '#80cbc4',
             LayerColors: {
-                Containers: '#ffb74d',
-                Convolution: '#2196f3',
+                Container: '#ffb74d',
+                NestedContainer: '#ffe0b2',
+                Convolution: '#42a5f5',
                 Simple: '#ff9100',
                 Transfer: '#80deea',
                 Misc: '#ce93d8'
@@ -32,12 +35,13 @@ define([
         };
 
     ArchEditorControl = function (options) {
-        EasyDAGControl.call(this, options);
+        ThumbnailControl.call(this, options);
         this._config = DEFAULT_CONFIG;
         ComponentSettings.resolveWithWebGMEGlobal(this._config, this.getComponentId());
+        this.validateLayers = _.debounce(() => this.validateArchitecture(), 500);
     };
 
-    _.extend(ArchEditorControl.prototype, EasyDAGControl.prototype);
+    _.extend(ArchEditorControl.prototype, ThumbnailControl.prototype);
 
     ArchEditorControl.prototype.TERRITORY_RULE = {children: 1};
     ArchEditorControl.prototype.DEFAULT_DECORATOR = 'LayerDecorator';
@@ -46,7 +50,9 @@ define([
     };
 
     ArchEditorControl.prototype.selectedObjectChanged = function(id) {
-        EasyDAGControl.prototype.selectedObjectChanged.call(this, id);
+        this.nestedLevel = typeof id === 'string' ?
+            Math.floor(id.split('/').length/2) % 2 : 0;
+        ThumbnailControl.prototype.selectedObjectChanged.call(this, id);
 
         DeepForge.last.Architecture = id;
         if (typeof id === 'string') {
@@ -56,7 +62,8 @@ define([
     };
 
     ArchEditorControl.prototype._getObjectDescriptor = function(id) {
-        var desc = EasyDAGControl.prototype._getObjectDescriptor.call(this, id);
+        var node = this._client.getNode(id),
+            desc = ThumbnailControl.prototype._getObjectDescriptor.call(this, id);
 
         // Filter attributes
         if (!desc.isConnection) {
@@ -78,7 +85,7 @@ define([
 
             for (i = names.length; i--;) {
                 // check if it is a setter
-                schema = this._client.getAttributeSchema(id, names[i]);
+                schema = node.getAttributeMeta(names[i]);
                 if (names[i] === 'name' || schema.setterType) {
                     desc.attributes[names[i]] = allAttrs[names[i]];
                 }
@@ -87,8 +94,7 @@ define([
             // Add layer type (base class's base class)
             desc.layerType = null;
             if (desc.baseName) {
-                var node = this._client.getNode(id),
-                    base = this._client.getNode(node.getMetaTypeId()),
+                var base = this._client.getNode(node.getMetaTypeId()),
                     layerType = this._client.getNode(base.getBaseId()),
                     color;
 
@@ -97,11 +103,32 @@ define([
                     desc.layerType = layerType.getAttribute(nodePropertyNames.Attributes.name);
 
                     color = this._config.LayerColors[desc.layerType];
+                    if (desc.layerType === 'Container' && this.nestedLevel) {
+                        color = this._config.LayerColors.NestedContainer;
+                    }
                     if (!color) {
                         this._logger.warn(`No color found for ${desc.layerType}`);
                         color = this._config.DefaultColor;
                     }
                     desc.color = color;
+
+                    if (desc.layerType === 'Container') {
+                        desc.containedLayers = node.getMemberIds(Constants.CONTAINED_LAYER_SET)
+                            .map(layerId => {
+                                var index = node.getMemberRegistry(
+                                    Constants.CONTAINED_LAYER_SET,
+                                    layerId,
+                                    Constants.CONTAINED_LAYER_INDEX
+                                );
+                                return [layerId, index];
+                            })
+                            .sort((a, b) => a[1] < b[1] ? -1 : 1)
+                            .map(tuple => tuple[0]);
+
+                        // Set the decorator to ContainerLayerDecorator
+                        desc.Decorator = this._client.decoratorManager
+                            .getDecoratorForWidget('ContainerLayerDecorator', 'EasyDAG');
+                    }
                 }
             }
         }
@@ -109,30 +136,19 @@ define([
     };
 
     ////////////////////////// Layer Selection Logic //////////////////////////
-    ArchEditorControl.prototype._getValidInitialNodes = function() {
-        return this._client.getChildrenMeta(this._currentNodeId).items
-            // For now, anything is possible!
-            // FIXME
-            .map(info => this._getAllDescendentIds(info.id))
-            .reduce((prev, curr) => prev.concat(curr))
-            // Filter all abstract nodes
-            .filter(nodeId => {
-                return !this._client.getNode(nodeId).isAbstract();
-            })
-            .map(id => this._getObjectDescriptor(id))
-            .filter(obj => !obj.isConnection && obj.name !== 'Connection')
-            .filter(layer => layer.layerType !== 'Criterion');
-    };
-
-    ArchEditorControl.prototype._getValidSuccessorNodes =
+    ArchEditorControl.prototype.getValidSuccessors =
     ArchEditorControl.prototype._getValidInitialNodes =
     ArchEditorControl.prototype.getNonCriterionLayers = function() {
         // Return all (non-criterion) layer types
         var metanodes = this._client.getAllMetaNodes(),
             layerId,
+            connId,
+            conn,
             criterionId,
-            allLayerIds = [],
+            allLayers = [],
             layers = [],
+            tgts,
+            j,
             i;
 
         for (i = metanodes.length; i--;) {
@@ -142,24 +158,40 @@ define([
             }
         }
 
+        // Remove all criterion layers and abstract layers
         for (i = metanodes.length; i--;) {
             if (layerId) {
-                if (!metanodes[i].isAbstract() &&
-                    this._client.isTypeOf(metanodes[i].getId(), layerId)) {
+                if (!metanodes[i].isAbstract() && metanodes[i].isTypeOf(layerId)) {
 
                     if (metanodes[i].getAttribute('name') === 'Criterion') {
                         criterionId = metanodes[i].getId();
                     } else {
-                        allLayerIds.push(metanodes[i].getId());
+                        allLayers.push(metanodes[i]);
+                    }
+                } else if (!connId && metanodes[i].getAttribute('name') === 'Connection') {  // Detect the layer connection type...
+                    tgts = this._client.getPointerMeta(metanodes[i].getId(), 'src').items;
+                    for (j = tgts.length; j--;) {
+                        if (tgts[j].id === layerId) {
+                            connId = metanodes[i].getId();
+                        }
                     }
                 }
             }
         }
 
+        if (!connId) {
+            this._logger.warn('Could not find a layer connector');
+            return [];
+        }
+        // Convert the layers into the correct format
+        conn = this._getObjectDescriptor(connId);
         // Remove all criterion layers and abstract layers
-        for (i = allLayerIds.length; i--;) {
-            if (!this._client.isTypeOf(allLayerIds[i], criterionId)) {
-                layers.push({node: this._getObjectDescriptor(allLayerIds[i])});
+        for (i = allLayers.length; i--;) {
+            if (!allLayers[i].isTypeOf(criterionId)) {
+                layers.push({
+                    node: this._getObjectDescriptor(allLayers[i].getId()),
+                    conn: conn
+                });
             }
         }
 
@@ -172,8 +204,9 @@ define([
 
     // Widget extensions
     ArchEditorControl.prototype._initWidgetEventHandlers = function() {
-        EasyDAGControl.prototype._initWidgetEventHandlers.call(this);
+        ThumbnailControl.prototype._initWidgetEventHandlers.call(this);
         this._widget.getCreateNewDecorator = this.getCreateNewDecorator.bind(this);
+        this._widget.insertLayer = this.insertLayer.bind(this);
     };
 
     ArchEditorControl.prototype.getCreateNewDecorator = function() {
@@ -183,5 +216,63 @@ define([
         );
     };
 
+    ArchEditorControl.prototype.insertLayer = function(layerBaseId, connId) {
+        var conn = this._client.getNode(connId),
+            parentId = conn.getParentId(),
+            layerId,
+            nextLayerId = conn.getPointer('dst').to,
+            connBaseId = conn.getBaseId(),
+            newConnId,
+
+            baseName = this._client.getNode(layerBaseId).getAttribute('name'),
+            prevLayerId = conn.getPointer('src').to,
+            srcName = this._client.getNode(prevLayerId).getAttribute('name'),
+            dstName = this._client.getNode(nextLayerId).getAttribute('name'),
+            msg = `Inserting ${baseName} layer between ${srcName} and ${dstName}`;
+
+        this._client.startTransaction(msg);
+        // Create the new layer
+        layerId = this._client.createNode({
+            parentId: parentId,
+            baseId: layerBaseId
+        });
+
+        // Connect the new layer to the previous dst of 'connId'
+        newConnId = this._client.createNode({
+            parentId: parentId,
+            baseId: connBaseId
+        });
+        this._client.setPointer(newConnId, 'src', layerId);
+        this._client.setPointer(newConnId, 'dst', nextLayerId);
+
+        // Change the dst of 'connId' to the new layer
+        this._client.setPointer(connId, 'dst', layerId);
+
+        this._client.completeTransaction();
+    };
+
+    ArchEditorControl.prototype._eventCallback = function() {
+        ThumbnailControl.prototype._eventCallback.apply(this, arguments);
+        this.validateLayers();
+    };
+
+    ArchEditorControl.prototype.validateArchitecture = function() {
+        var pluginId = 'ValidateArchitecture',
+            context = this._client.getCurrentPluginContext(pluginId);
+
+        this._logger.info('about to validate arch');
+        // Run the plugin in the browser (set namespace)
+        context.managerConfig.namespace = 'nn';
+        context.pluginConfig = {};
+        Q.ninvoke(this._client, 'runServerPlugin', pluginId, context)
+            .then(res => {
+                var results = res.messages[0].message;
+                if (results.errors !== null) {
+                    this._widget.displayErrors(results.errors);
+                }
+            })
+            .fail(err => this._logger.warn(`Validation failed: ${err}`));
+    };
+
     return ArchEditorControl;
 });

src/visualizers/panels/ArchIndex/ArchIndexControl.js

@@ -0,0 +1,55 @@
+/* globals define */
+define([
+    'panels/PipelineIndex/PipelineIndexControl'
+], function(
+    PipelineIndexControl
+) {
+    var ArchIndexControl = function() {
+        PipelineIndexControl.apply(this, arguments);
+    };
+
+    ArchIndexControl.prototype = Object.create(PipelineIndexControl.prototype);
+    ArchIndexControl.prototype._getObjectDescriptor = function (nodeId) {
+        var node = this._client.getNode(nodeId),
+            base,
+            desc;
+
+        if (node) {
+            base = this._client.getNode(node.getBaseId());
+            desc = {
+                id: node.getId(),
+                name: node.getAttribute('name'),
+                parentId: node.getParentId(),
+                thumbnail: node.getAttribute('thumbnail'),
+                type: base.getAttribute('name')
+            };
+        }
+
+        return desc;
+    };
+
+    ArchIndexControl.prototype._initWidgetEventHandlers = function () {
+        this._widget.deletePipeline = id => {
+            var node = this._client.getNode(id),
+                name = node.getAttribute('name'),
+                msg = `Deleted "${name}" architecture`;
+
+
+            this._client.startTransaction(msg);
+            this._client.deleteNode(id);
+            this._client.completeTransaction();
+        };
+
+        this._widget.setName = (id, name) => {
+            var oldName = this._client.getNode(id).getAttribute('name'),
+                msg = `Renaming architecture: "${oldName}" -> "${name}"`;
+
+            if (oldName !== name && !/^\s*$/.test(name)) {
+                this._client.startTransaction(msg);
+                this._client.setAttribute(id, 'name', name);
+                this._client.completeTransaction();
+            }
+        };
+    };
+    return ArchIndexControl;
+});

src/visualizers/panels/ArchIndex/ArchIndexPanel.js

@@ -0,0 +1,100 @@
+/*globals define, _, WebGMEGlobal*/
+/*jshint browser: true*/
+
+define([
+    'js/PanelBase/PanelBaseWithHeader',
+    'js/PanelManager/IActivePanel',
+    'widgets/ArchIndex/ArchIndexWidget',
+    'panels/ArchIndex/ArchIndexControl'
+], function (
+    PanelBaseWithHeader,
+    IActivePanel,
+    ArchIndexWidget,
+    ArchIndexControl
+) {
+    'use strict';
+
+    var ArchIndexPanel;
+
+    ArchIndexPanel = function (layoutManager, params) {
+        var options = {};
+        //set properties from options
+        options[PanelBaseWithHeader.OPTIONS.LOGGER_INSTANCE_NAME] = 'ArchIndexPanel';
+        options[PanelBaseWithHeader.OPTIONS.FLOATING_TITLE] = true;
+
+        //call parent's constructor
+        PanelBaseWithHeader.apply(this, [options, layoutManager]);
+
+        this._client = params.client;
+        this._embedded = params.embedded;
+
+        //initialize UI
+        this._initialize();
+
+        this.logger.debug('ctor finished');
+    };
+
+    //inherit from PanelBaseWithHeader
+    _.extend(ArchIndexPanel.prototype, PanelBaseWithHeader.prototype);
+    _.extend(ArchIndexPanel.prototype, IActivePanel.prototype);
+
+    ArchIndexPanel.prototype._initialize = function () {
+        var self = this;
+
+        //set Widget title
+        this.setTitle('');
+
+        this.widget = new ArchIndexWidget(this.logger, this.$el);
+
+        this.widget.setTitle = function (title) {
+            self.setTitle(title);
+        };
+
+        this.control = new ArchIndexControl({
+            logger: this.logger,
+            client: this._client,
+            embedded: this._embedded,
+            widget: this.widget
+        });
+
+        this.onActivate();
+    };
+
+    /* OVERRIDE FROM WIDGET-WITH-HEADER */
+    /* METHOD CALLED WHEN THE WIDGET'S READ-ONLY PROPERTY CHANGES */
+    ArchIndexPanel.prototype.onReadOnlyChanged = function (isReadOnly) {
+        //apply parent's onReadOnlyChanged
+        PanelBaseWithHeader.prototype.onReadOnlyChanged.call(this, isReadOnly);
+
+    };
+
+    ArchIndexPanel.prototype.onResize = function (width, height) {
+        this.logger.debug('onResize --> width: ' + width + ', height: ' + height);
+    };
+
+    /* * * * * * * * Visualizer life cycle callbacks * * * * * * * */
+    ArchIndexPanel.prototype.destroy = function () {
+        this.control.destroy();
+        this.widget.destroy();
+
+        PanelBaseWithHeader.prototype.destroy.call(this);
+        WebGMEGlobal.KeyboardManager.setListener(undefined);
+        WebGMEGlobal.Toolbar.refresh();
+    };
+
+    ArchIndexPanel.prototype.onActivate = function () {
+        this.widget.onActivate();
+        this.control.onActivate();
+        WebGMEGlobal.KeyboardManager.setListener(this.widget);
+        WebGMEGlobal.Toolbar.refresh();
+    };
+
+    ArchIndexPanel.prototype.onDeactivate = function () {
+        this.widget.onDeactivate();
+        this.control.onDeactivate();
+        WebGMEGlobal.KeyboardManager.setListener(undefined);
+        WebGMEGlobal.Toolbar.refresh();
+    };
+
+    return ArchIndexPanel;
+});

src/visualizers/panels/ArtifactIndex/ArtifactIndexControl.js

@@ -0,0 +1,203 @@
+/*globals define, WebGMEGlobal*/
+/*jshint browser: true*/
+
+define([
+    'blob/BlobClient',
+    'js/Constants'
+], function (
+    BlobClient,
+    CONSTANTS
+) {
+
+    'use strict';
+
+    var ArtifactIndexControl;
+
+    ArtifactIndexControl = function (options) {
+
+        this._logger = options.logger.fork('Control');
+        this.blobClient = new BlobClient({
+            logger: this._logger.fork('BlobClient')
+        });
+
+        this._client = options.client;
+
+        // Initialize core collections and variables
+        this._widget = options.widget;
+
+        this._currentNodeId = null;
+        this._initWidgetEventHandlers();
+
+        this._logger.debug('ctor finished');
+    };
+
+    ArtifactIndexControl.prototype._initWidgetEventHandlers = function () {
+        this._widget.onNodeClick = (/*id*/) => {
+            // Change the current active object
+            // This is currently disabled as there are not any good
+            // visualizers for the data types
+            // WebGMEGlobal.State.registerActiveObject(id);
+        };
+
+        this._widget.onNodeDeleteClicked = id => {
+            var name = this._client.getNode(id).getAttribute('name'),
+                msg = `Deleted "${name}" artifact (${id}) --`;
+
+            this._client.startTransaction(msg);
+            this._client.deleteNode(id);
+            this._client.completeTransaction();
+        };
+    };
+
+    /* * * * * * * * Visualizer content update callbacks * * * * * * * */
+    // One major concept here is with managing the territory. The territory
+    // defines the parts of the project that the visualizer is interested in
+    // (this allows the browser to then only load those relevant parts).
+    ArtifactIndexControl.prototype.selectedObjectChanged = function (nodeId) {
+        this._logger.debug('activeObject nodeId \'' + nodeId + '\'');
+
+        // Remove current territory patterns
+        if (this._currentNodeId) {
+            this._client.removeUI(this._territoryId);
+        }
+
+        this._currentNodeId = nodeId;
+
+        if (typeof this._currentNodeId === 'string') {
+            // Put new node's info into territory rules
+            this._widget.currentNode = this._currentNodeId;
+            this._selfPatterns = {};
+
+            this._territoryId = this._client.addUI(this, events => {
+                this._eventCallback(events);
+            });
+
+            this._selfPatterns[nodeId] = {children: 1};
+            this._client.updateTerritory(this._territoryId, this._selfPatterns);
+        }
+    };
+
+    // This next function retrieves the relevant node information for the widget
+    ArtifactIndexControl.prototype._getObjectDescriptor = function (nodeId) {
+        var node = this._client.getNode(nodeId),
+            type,
+            hash,
+            objDescriptor;
+
+        if (node) {
+            type = this._client.getNode(node.getMetaTypeId());
+            hash = node.getAttribute('data');
+            objDescriptor = {
+                id: node.getId(),
+                type: type ? type.getAttribute('name') : 'n/a',
+                name: node.getAttribute('name'),
+                createdAt: node.getAttribute('createdAt'),
+                dataURL: this.blobClient.getDownloadURL(hash),
+                parentId: node.getParentId()
+            };
+        }
+
+        return this.blobClient.getMetadata(hash)
+            .then(metadata => {
+                objDescriptor.size = this._humanFileSize(metadata.size);
+                return objDescriptor;
+            });
+    };
+
+    ArtifactIndexControl.prototype._humanFileSize = function (bytes, si) {
+        var thresh = si ? 1000 : 1024,
+            units = si ?
+                ['kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'] :
+                ['KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'],
+            u = -1;
+
+        if (bytes < thresh) {
+            return bytes + ' B';
+        }
+
+        do {
+            bytes = bytes / thresh;
+            u += 1;
+        } while (bytes >= thresh);
+
+        return bytes.toFixed(1) + ' ' + units[u];
+    };
+
+    /* * * * * * * * Node Event Handling * * * * * * * */
+    ArtifactIndexControl.prototype._eventCallback = function (events) {
+        var i = events ? events.length : 0,
+            event;
+
+        this._logger.debug('_eventCallback \'' + i + '\' items');
+
+        while (i--) {
+            event = events[i];
+            switch (event.etype) {
+
+            case CONSTANTS.TERRITORY_EVENT_LOAD:
+                this._onLoad(event.eid);
+                break;
+            case CONSTANTS.TERRITORY_EVENT_UPDATE:
+                this._onUpdate(event.eid);
+                break;
+            case CONSTANTS.TERRITORY_EVENT_UNLOAD:
+                this._onUnload(event.eid);
+                break;
+            default:
+                break;
+            }
+        }
+
+        this._logger.debug('_eventCallback \'' + events.length + '\' items - DONE');
+    };
+
+    ArtifactIndexControl.prototype._onLoad = function (gmeId) {
+        this._getObjectDescriptor(gmeId).then(desc => this._widget.addNode(desc));
+    };
+
+    ArtifactIndexControl.prototype._onUpdate = function (gmeId) {
+        this._getObjectDescriptor(gmeId).then(desc => this._widget.updateNode(desc));
+    };
+
+    ArtifactIndexControl.prototype._onUnload = function (gmeId) {
+        this._widget.removeNode(gmeId);
+    };
+
+    ArtifactIndexControl.prototype._stateActiveObjectChanged = function (model, activeObjectId) {
+        if (this._currentNodeId === activeObjectId) {
+            // The same node selected as before - do not trigger
+        } else {
+            this.selectedObjectChanged(activeObjectId);
+        }
+    };
+
+    /* * * * * * * * Visualizer life cycle callbacks * * * * * * * */
+    ArtifactIndexControl.prototype.destroy = function () {
+        this._detachClientEventListeners();
+    };
+
+    ArtifactIndexControl.prototype._attachClientEventListeners = function () {
+        this._detachClientEventListeners();
+        WebGMEGlobal.State.on('change:' + CONSTANTS.STATE_ACTIVE_OBJECT, this._stateActiveObjectChanged, this);
+    };
+
+    ArtifactIndexControl.prototype._detachClientEventListeners = function () {
+        WebGMEGlobal.State.off('change:' + CONSTANTS.STATE_ACTIVE_OBJECT, this._stateActiveObjectChanged);
+    };
+
+    ArtifactIndexControl.prototype.onActivate = function () {
+        this._attachClientEventListeners();
+
+        if (typeof this._currentNodeId === 'string') {
+            WebGMEGlobal.State.registerSuppressVisualizerFromNode(true);
+            WebGMEGlobal.State.registerActiveObject(this._currentNodeId);
+            WebGMEGlobal.State.registerSuppressVisualizerFromNode(false);
+        }
+    };
+
+    ArtifactIndexControl.prototype.onDeactivate = function () {
+        this._detachClientEventListeners();
+    };
+
+    return ArtifactIndexControl;
+});

src/visualizers/panels/ArtifactIndex/ArtifactIndexPanel.js

@@ -0,0 +1,99 @@
+/*globals define, _, WebGMEGlobal*/
+/*jshint browser: true*/
+
+define([
+    'js/PanelBase/PanelBaseWithHeader',
+    'js/PanelManager/IActivePanel',
+    'widgets/ArtifactIndex/ArtifactIndexWidget',
+    './ArtifactIndexControl'
+], function (
+    PanelBaseWithHeader,
+    IActivePanel,
+    ArtifactIndexWidget,
+    ArtifactIndexControl
+) {
+    'use strict';
+
+    var ArtifactIndexPanel;
+
+    ArtifactIndexPanel = function (layoutManager, params) {
+        var options = {};
+        //set properties from options
+        options[PanelBaseWithHeader.OPTIONS.LOGGER_INSTANCE_NAME] = 'ArtifactIndexPanel';
+        options[PanelBaseWithHeader.OPTIONS.FLOATING_TITLE] = true;
+
+        //call parent's constructor
+        PanelBaseWithHeader.apply(this, [options, layoutManager]);
+
+        this._client = params.client;
+
+        //initialize UI
+        this._initialize();
+
+        this.logger.debug('ctor finished');
+    };
+
+    //inherit from PanelBaseWithHeader
+    _.extend(ArtifactIndexPanel.prototype, PanelBaseWithHeader.prototype);
+    _.extend(ArtifactIndexPanel.prototype, IActivePanel.prototype);
+
+    ArtifactIndexPanel.prototype._initialize = function () {
+        var self = this;
+
+        //set Widget title
+        this.setTitle('');
+
+        this.widget = new ArtifactIndexWidget(this.logger, this.$el);
+
+        this.widget.setTitle = function (title) {
+            self.setTitle(title);
+        };
+
+        this.control = new ArtifactIndexControl({
+            logger: this.logger,
+            client: this._client,
+            widget: this.widget
+        });
+
+        this.onActivate();
+    };
+
+    /* OVERRIDE FROM WIDGET-WITH-HEADER */
+    /* METHOD CALLED WHEN THE WIDGET'S READ-ONLY PROPERTY CHANGES */
+    ArtifactIndexPanel.prototype.onReadOnlyChanged = function (isReadOnly) {
+        //apply parent's onReadOnlyChanged
+        PanelBaseWithHeader.prototype.onReadOnlyChanged.call(this, isReadOnly);
+
+    };
+
+    ArtifactIndexPanel.prototype.onResize = function (width, height) {
+        this.logger.debug('onResize --> width: ' + width + ', height: ' + height);
+        this.widget.onWidgetContainerResize(width, height);
+    };
+
+    /* * * * * * * * Visualizer life cycle callbacks * * * * * * * */
+    ArtifactIndexPanel.prototype.destroy = function () {
+        this.control.destroy();
+        this.widget.destroy();
+
+        PanelBaseWithHeader.prototype.destroy.call(this);
+        WebGMEGlobal.KeyboardManager.setListener(undefined);
+        WebGMEGlobal.Toolbar.refresh();
+    };
+
+    ArtifactIndexPanel.prototype.onActivate = function () {
+        this.widget.onActivate();
+        this.control.onActivate();
+        WebGMEGlobal.KeyboardManager.setListener(this.widget);
+        WebGMEGlobal.Toolbar.refresh();
+    };
+
+    ArtifactIndexPanel.prototype.onDeactivate = function () {
+        this.widget.onDeactivate();
+        this.control.onDeactivate();
+        WebGMEGlobal.KeyboardManager.setListener(undefined);
+        WebGMEGlobal.Toolbar.refresh();
+    };
+
+    return ArtifactIndexPanel;
+});

src/visualizers/panels/ClassCodeEditor/ClassCodeEditorControl.js

@@ -100,7 +100,7 @@ define([
 
         this._client.startTransaction(`Updating class "${name || nodeName}"`);
         if (name) {
-            this._client.setAttributes(id, 'name', name);
+            this._client.setAttribute(id, 'name', name);
         }
         if (basePath) {
             this._client.setBase(id, basePath);
@@ -137,7 +137,7 @@ define([
         }
 
         for (i = nameMatches.length; i--;) {
-            if (this._client.isTypeOf(nameMatches[i].getId(), classNode.getId())) {
+            if (nameMatches[i].isTypeOf(classNode.getId())) {
                 return nameMatches[i].getId();
             }
         }

src/visualizers/panels/ExecutionView/ExecutionViewControl.js

@@ -6,12 +6,14 @@ define([
     'deepforge/Constants',
     'panels/EasyDAG/EasyDAGControl',
     'deepforge/viz/PipelineControl',
+    'deepforge/viz/Execute',
     'underscore'
 ], function (
     GME_CONSTANTS,
     CONSTANTS,
     EasyDAGControl,
     PipelineControl,
+    Execute,
     _
 ) {
 
@@ -21,15 +23,18 @@ define([
 
     ExecutionViewControl = function (options) {
         EasyDAGControl.call(this, options);
+        Execute.call(this, this._client, this._logger);
         this.addedNodes = {};
         this.originTerritory = {};
         this.originTerritoryId = null;
+        this.readOnly = false;
     };
 
     _.extend(
         ExecutionViewControl.prototype,
         EasyDAGControl.prototype,
-        PipelineControl.prototype
+        PipelineControl.prototype,
+        Execute.prototype
     );
 
     /* * * * * * * * Visualizer content update callbacks * * * * * * * */
@@ -76,6 +81,9 @@ define([
                     this._client.removeUI(this.originTerritoryId);
                 }
             }
+            if (!this.readOnly) {
+                this.checkPipelineExecution(this._client.getNode(id));
+            }
         }
     };
 
@@ -108,6 +116,7 @@ define([
                     allAttrs = {},
                     hiddenAttrs = [
                         CONSTANTS.LINE_OFFSET,
+                        CONSTANTS.DISPLAY_COLOR,
                         'code',
                         'name'
                     ],

src/visualizers/panels/ExecutionView/ExecutionViewPanel.js

@@ -73,7 +73,7 @@ define([
     ExecutionViewPanel.prototype.onReadOnlyChanged = function (isReadOnly) {
         //apply parent's onReadOnlyChanged
         PanelBaseWithHeader.prototype.onReadOnlyChanged.call(this, isReadOnly);
-
+        this.control.readOnly = isReadOnly;
     };
 
     ExecutionViewPanel.prototype.onResize = function (width, height) {

src/visualizers/panels/Footer/FooterPanel.js

@@ -1,19 +1,15 @@
-/*globals define, _, WebGMEGlobal, $ */
+/*globals define, _, $ */
 /*jshint browser: true*/
 
 define([
     'js/PanelBase/PanelBase',
     'js/Widgets/NetworkStatus/NetworkStatusWidget',
     'js/Widgets/BranchStatus/BranchStatusWidget',
-    'js/Widgets/BranchSelector/BranchSelectorWidget',
-    'js/Widgets/KeyboardManager/KeyboardManagerWidget',
     './FilteredNotificationWidget'
 ], function (
     PanelBase,
     NetworkStatusWidget,
     BranchStatusWidget,
-    BranchSelectorWidget,
-    KeyboardManagerWidget,
     NotificationWidget
 ) {
 
@@ -47,7 +43,6 @@ define([
             navBarInner = $('<div/>', {class: 'navbar-inner'}),
             separator = $('<div class="spacer pull-right"></div>'),
             widgetPlaceHolder = $('<div class="pull-right"></div>'),
-            keyBoardManagerEl,
             networkStatusEl,
             branchStatusEl,
             notificationEl;
@@ -58,13 +53,6 @@ define([
         //padding from screen right edge
         navBarInner.append(separator.clone());
 
-        //keyboard enable/disbale widget (NOTE: only on non touch device)
-        if (WebGMEGlobal.SUPPORTS_TOUCH !== true) {
-            keyBoardManagerEl = widgetPlaceHolder.clone();
-            new KeyboardManagerWidget(keyBoardManagerEl);
-            navBarInner.append(keyBoardManagerEl).append(separator.clone());
-        }
-
         networkStatusEl = widgetPlaceHolder.clone();
         new NetworkStatusWidget(networkStatusEl, this._client);
         navBarInner.append(networkStatusEl).append(separator.clone());

src/visualizers/panels/ForgeActionButton/Actions.js

@@ -5,7 +5,8 @@ define([
     'panel/FloatingActionButton/styles/Materialize',
     'q',
     'js/RegistryKeys',
-    'deepforge/globals'
+    'deepforge/globals',
+    'deepforge/Constants'
 ], function(
     Materialize,
     Q,
@@ -264,6 +265,26 @@ define([
                 action: function() {
                     this.addOperation();
                 }
+            },
+            {
+                name: 'Export Pipeline',
+                icon: 'launch',
+                priority: -1,
+                action: function() {
+                    this.exportPipeline()
+                        .then(result => {
+                            Materialize.toast('Export successful!', 2000);
+                            // Download the result!
+                            this.downloadFromBlob(result.artifacts[0]);
+                            result.__unread = true;
+                            this.results.push(result);
+                            this._updatePluginBtns();
+                        })
+                        .fail(err => {
+                            this.logger.warn('Pipeline export failed:', err);
+                            Materialize.toast(`Export failed: ${err}`, 4000);
+                        });
+                }
             }
         ],
         Architecture: [

src/visualizers/panels/ForgeActionButton/ConfigDialog.css

@@ -0,0 +1,5 @@
+.config-section-header {
+  margin-top: 0;
+  color: #888;
+  font-style: italic;
+}

src/visualizers/panels/ForgeActionButton/ConfigDialog.js

@@ -0,0 +1,222 @@
+/* globals define, $*/
+define([
+    'js/Dialogs/PluginConfig/PluginConfigDialog',
+    'text!js/Dialogs/PluginConfig/templates/PluginConfigDialog.html',
+    'plugin/Export/Export/format',
+    'css!./ConfigDialog.css'
+], function(
+    PluginConfigDialog,
+    pluginConfigDialogTemplate,
+    ExportFormats
+) {
+    var SECTION_DATA_KEY = 'section',
+        ATTRIBUTE_DATA_KEY = 'attribute',
+    //jscs:disable maximumLineLength
+        PLUGIN_CONFIG_SECTION_BASE = $('<div><fieldset><form class="form-horizontal" role="form"></form><fieldset></div>'),
+        ENTRY_BASE = $('<div class="form-group"><div class="row"><label class="col-sm-4 control-label">NAME</label><div class="col-sm-8 controls"></div></div><div class="row description"><div class="col-sm-4"></div></div></div>'),
+    //jscs:enable maximumLineLength
+        DESCRIPTION_BASE = $('<div class="desc muted col-sm-8"></div>'),
+        SECTION_HEADER = $('<h6 class="config-section-header">');
+
+    var ConfigDialog = function(client, nodeId) {
+        PluginConfigDialog.call(this, {client: client});
+        this._widgets = {};
+        this._node = this._client.getNode(nodeId);
+    };
+
+    ConfigDialog.prototype = Object.create(PluginConfigDialog.prototype);
+
+    ConfigDialog.prototype.getFormatOptions = function() {
+        this._exportFormats = Object.keys(ExportFormats);
+
+        this._formatOptions = [];
+        if (this._exportFormats.length > 1) {
+            this._formatOptions.push({  // format options
+                name: 'exportFormat',
+                displayName: 'Export Format',
+                valueType: 'string',
+                value: this._exportFormats[0],
+                valueItems: this._exportFormats,
+                readOnly: false
+            });
+        }
+    };
+
+    ConfigDialog.prototype.show = function(pluginMetadata, callback) {
+        this._pluginMetadata = pluginMetadata;
+
+        this.getFormatOptions();
+        this._initDialog();
+
+        this._dialog.on('shown', () => {
+            this._dialog.find('input').first().focus();
+        });
+
+        this._btnSave.on('click', event => {
+            this.submit(callback);
+            event.stopPropagation();
+            event.preventDefault();
+        });
+
+        //save&run on CTRL + Enter
+        this._dialog.on('keydown.PluginConfigDialog', event => {
+            if (event.keyCode === 13 && (event.ctrlKey || event.metaKey)) {
+                event.stopPropagation();
+                event.preventDefault();
+                this.submit(callback);
+            }
+        });
+        this._dialog.modal('show');
+    };
+
+    ConfigDialog.prototype._initDialog = function() {
+        this._dialog = $(pluginConfigDialogTemplate);
+
+        this._btnSave = this._dialog.find('.btn-save');
+        this._divContainer = this._dialog.find('.modal-body');
+        this._saveConfigurationCb = this._dialog.find('.save-configuration');
+        this._modalHeader = this._dialog.find('.modal-header');
+
+        // Create the header
+        var iconEl = $('<i/>', {
+            class: this._pluginMetadata.icon.class || 'glyphicon glyphicon-cog'
+        });
+        iconEl.addClass('plugin-icon pull-left');
+        this._modalHeader.prepend(iconEl);
+        this._title = this._modalHeader.find('.modal-title');
+        this._title.text(this._pluginMetadata.id + ' ' + 'v' + this._pluginMetadata.version);
+
+        // Generate the config options
+        var sectionHeader = SECTION_HEADER.clone();
+
+        sectionHeader.text('Static Artifacts');
+        this._divContainer.append(sectionHeader);
+        this.generateConfigSection(this._pluginMetadata);
+
+        if (this._exportFormats.length > 1) {
+            this._divContainer.append($('<hr class="extension-config-divider">'));
+            sectionHeader = SECTION_HEADER.clone();
+            sectionHeader.text('Export Options');
+            this._divContainer.append(sectionHeader);
+
+            this.generateConfigSection({
+                id: 'FormatOptions',
+                configStructure: this._formatOptions
+            });
+            this._widgets.FormatOptions.exportFormat.el.find('select').on('change', event => {
+                var format = event.target.value;
+                // Update the ext config
+                this.updateExtConfig(format);
+            });
+        }
+
+        this.updateExtConfig(this._exportFormats[0]);
+    };
+
+    ConfigDialog.prototype.submit = function (callback) {
+        var config = this._getAllConfigValues();
+        this._dialog.modal('hide');
+
+        if (this._exportFormats.length === 1) {
+            config.FormatOptions = {
+                exportFormat: this._exportFormats[0]
+            };
+        }
+        return callback(config);
+    };
+
+    ConfigDialog.prototype._getAllConfigValues = function () {
+        var settings = {};
+
+        Object.keys(this._widgets).forEach(namespace => {
+            settings[namespace] = {};
+
+            Object.keys(this._widgets[namespace]).forEach(name => {
+                settings[namespace][name] = this._widgets[namespace][name].getValue();
+            });
+        });
+
+        return settings;
+    };
+
+    ConfigDialog.prototype.updateExtConfig = function (format) {
+        var extConfig = {
+            id: 'extensionConfig',
+            class: 'extension-config',
+            configStructure: ExportFormats[format].getConfigStructure ?
+                ExportFormats[format].getConfigStructure(this._node, this._client) : []
+        };
+        this._divContainer.find('.extension-config').remove();
+
+        if (extConfig.configStructure.length) {
+            this.generateConfigSection(extConfig);
+        }
+    };
+
+    ConfigDialog.prototype.generateConfigSection = function (metadata) {
+        var len = metadata.configStructure.length,
+            i,
+            el,
+            pluginConfigEntry,
+            widget,
+            descEl,
+            containerEl,
+            pluginSectionEl = PLUGIN_CONFIG_SECTION_BASE.clone();
+
+        pluginSectionEl.data(SECTION_DATA_KEY, metadata.id);
+        this._divContainer.append(pluginSectionEl);
+        containerEl = pluginSectionEl.find('.form-horizontal');
+
+        if (metadata.class) {
+            pluginSectionEl.addClass(metadata.class);
+        }
+
+        this._widgets[metadata.id] = {};
+        for (i = 0; i < len; i += 1) {
+            pluginConfigEntry = metadata.configStructure[i];
+            descEl = undefined;
+
+            // Make sure not modify the global metadata.
+            pluginConfigEntry = JSON.parse(JSON.stringify(pluginConfigEntry));
+            if (this._client.getProjectAccess().write === false && pluginConfigEntry.writeAccessRequired === true) {
+                pluginConfigEntry.readOnly = true;
+            }
+
+            widget = this._propertyGridWidgetManager.getWidgetForProperty(pluginConfigEntry);
+            this._widgets[metadata.id][pluginConfigEntry.name] = widget;
+
+            el = ENTRY_BASE.clone();
+            el.data(ATTRIBUTE_DATA_KEY, pluginConfigEntry.name);
+
+            el.find('label.control-label').text(pluginConfigEntry.displayName);
+
+            if (pluginConfigEntry.description && pluginConfigEntry.description !== '') {
+                descEl = descEl || DESCRIPTION_BASE.clone();
+                descEl.text(pluginConfigEntry.description);
+            }
+
+            if (pluginConfigEntry.minValue !== undefined &&
+                pluginConfigEntry.minValue !== null &&
+                pluginConfigEntry.minValue !== '') {
+                descEl = descEl || DESCRIPTION_BASE.clone();
+                descEl.append(' The minimum value is: ' + pluginConfigEntry.minValue + '.');
+            }
+
+            if (pluginConfigEntry.maxValue !== undefined &&
+                pluginConfigEntry.maxValue !== null &&
+                pluginConfigEntry.maxValue !== '') {
+                descEl = descEl || DESCRIPTION_BASE.clone();
+                descEl.append(' The maximum value is: ' + pluginConfigEntry.maxValue + '.');
+            }
+
+            el.find('.controls').append(widget.el);
+            if (descEl) {
+                el.find('.description').append(descEl);
+            }
+
+            containerEl.append(el);
+        }
+    };
+
+    return ConfigDialog;
+});

src/visualizers/panels/ForgeActionButton/ForgeActionButton.js

@@ -1,8 +1,10 @@
-/*globals $, window, define, _ */
+/*globals $, window, define, _, WebGMEGlobal */
 /*jshint browser: true*/
 
 define([
     'blob/BlobClient',
+    'js/Utils/SaveToDisk',
+    './ConfigDialog',
     'js/Constants',
     'panel/FloatingActionButton/FloatingActionButton',
     'deepforge/viz/PipelineControl',
@@ -13,10 +15,14 @@ define([
     'js/RegistryKeys',
     'js/Panels/MetaEditor/MetaEditorConstants',
     'q',
-    'deepforge/globals'
+    'deepforge/globals',
+    'deepforge/Constants',
+    'plugin/Export/Export/format'
 ], function (
     BlobClient,
-    CONSTANTS,
+    SaveToDisk,
+    ConfigDialog,
+    GME_CONSTANTS,
     PluginButton,
     PipelineControl,
     NodePrompter,
@@ -26,7 +32,9 @@ define([
     REGISTRY_KEYS,
     META_CONSTANTS,
     Q,
-    DeepForge
+    DeepForge,
+    Constants,
+    ExportFormatDict
 ) {
     'use strict';
 
@@ -140,19 +148,19 @@ define([
 
     // Helper functions REMOVE! FIXME
     ForgeActionButton.prototype.addToMetaSheet = function(nodeId, metasheetName) {
-        var root = this.client.getNode(CONSTANTS.PROJECT_ROOT_ID),
+        var root = this.client.getNode(GME_CONSTANTS.PROJECT_ROOT_ID),
             metatabs = root.getRegistry(REGISTRY_KEYS.META_SHEETS),
             metatab = metatabs.find(tab => tab.title === metasheetName) || metatabs[0],
             metatabId = metatab.SetID;
 
         // Add to the general meta
         this.client.addMember(
-            CONSTANTS.PROJECT_ROOT_ID,
+            GME_CONSTANTS.PROJECT_ROOT_ID,
             nodeId,
             META_CONSTANTS.META_ASPECT_SET_NAME
         );
         this.client.setMemberRegistry(
-            CONSTANTS.PROJECT_ROOT_ID,
+            GME_CONSTANTS.PROJECT_ROOT_ID,
             nodeId,
             META_CONSTANTS.META_ASPECT_SET_NAME,
             REGISTRY_KEYS.POSITION,
@@ -163,9 +171,9 @@ define([
         );
 
         // Add to the specific sheet
-        this.client.addMember(CONSTANTS.PROJECT_ROOT_ID, nodeId, metatabId);
+        this.client.addMember(GME_CONSTANTS.PROJECT_ROOT_ID, nodeId, metatabId);
         this.client.setMemberRegistry(
-            CONSTANTS.PROJECT_ROOT_ID,
+            GME_CONSTANTS.PROJECT_ROOT_ID,
             nodeId,
             metatabId,
             REGISTRY_KEYS.POSITION,
@@ -178,7 +186,7 @@ define([
 
     ForgeActionButton.prototype.createNamedNode = function(baseId, isMeta) {
         var parentId = this._currentNodeId,
-            newId = this.client.createChild({parentId, baseId}),
+            newId = this.client.createNode({parentId, baseId}),
             basename = 'New' + this.client.getNode(baseId).getAttribute('name'),
             newName = this.getUniqueName(parentId, basename);
 
@@ -186,7 +194,7 @@ define([
         if (!isMeta) {
             newName = newName.substring(0, 1).toLowerCase() + newName.substring(1);
         }
-        this.client.setAttributes(newId, 'name', newName);
+        this.client.setAttribute(newId, 'name', newName);
         return newId;
     };
 
@@ -355,10 +363,137 @@ define([
         var nodeId = this._currentNodeId;
         if (nodeId) {
             this.client.startTransaction(msg);
-            this.client.delMoreNodes([nodeId]);
-            this.client.completeTransaction(msg);
+            this.client.deleteNode(nodeId);
+            this.client.completeTransaction();
         }
     };
 
+    ForgeActionButton.prototype.downloadFromBlob = function(hash) {
+        this._blobClient.getMetadata(hash)
+            .then(metadata => {
+                var url = this._blobClient.getDownloadURL(hash),
+                    name = metadata.name,
+                    save = document.createElement('a');
+
+                save.href = url;
+                save.target = '_self';
+                save.download = name;
+
+                save.click();
+                (window.URL || window.webkitURL).revokeObjectURL(save.href);
+            })
+            .fail(err => this.logger.error(`Blob download failed: ${err}`));
+    };
+
+    /// Export Pipeline Support
+    ForgeActionButton.prototype.exportPipeline = function() {
+        var deferred = Q.defer(),
+            pluginId = 'Export',
+            metadata = WebGMEGlobal.allPluginsMetadata[pluginId],
+            id = this._currentNodeId,
+            node = this.client.getNode(id),
+            inputData,
+            inputNames;
+
+        inputData = node.getChildrenIds()
+            .map(id => this.client.getNode(id))
+            .filter(node => {
+                var typeId = node.getMetaTypeId(),
+                    type = this.client.getNode(typeId).getAttribute('name');
+
+                return type === Constants.OP.INPUT;
+            })
+            .map(input => {
+                var outputCntr,
+                    outputIds;
+
+                outputCntr = input.getChildrenIds()
+                    .map(id => this.client.getNode(id))
+                    .find(node => {
+                        var typeId = node.getMetaTypeId(),
+                            type = this.client.getNode(typeId).getAttribute('name');
+                        return type === 'Outputs';
+                    });
+
+                // input operations only have a single output
+                outputIds = outputCntr.getChildrenIds();
+
+                if (outputIds.length === 1) {
+                    return outputIds[0];
+                } else if (outputIds.length > 1) {
+                    this.logger.warn(`Found multiple ids for input op: ${outputIds.join(', ')}`);
+                    return;
+                }
+            })
+            .filter(outputId => !!outputId)
+            .map(id => this.client.getNode(id))
+            .filter(output => output.getAttribute('data'));
+
+        // get the name of node referenced from the input op
+        inputNames = inputData
+            .map(node => {
+                var cntrId = node.getParentId(),
+                    opId = this._client.getNode(cntrId).getParentId(),
+                    inputOp = this._client.getNode(opId),
+                    targetNodeId = inputOp.getPointer('artifact').to;
+
+                return this._client.getNode(targetNodeId).getAttribute('name');
+            })
+            .sort();
+
+        // create config options from inputs
+        var inputOpts = inputNames.map((input, index) => {
+            return {
+                name: inputData[index].getId(),
+                displayName: input,
+                description: `Export ${input} as static (non-input) content`,
+                value: false,
+                valueType: 'boolean',
+                readOnly: false
+            };
+        });
+
+        var exportFormats = Object.keys(ExportFormatDict),
+            configDialog = new ConfigDialog(this.client, this._currentNodeId),
+            inputConfig = _.extend({}, metadata);
+
+        inputConfig.configStructure = inputOpts;
+
+        // Try to get the extension options
+        if (inputOpts.length || exportFormats.length > 1) {
+            configDialog.show(inputConfig, (allConfigs) => {
+                var context = this.client.getCurrentPluginContext(pluginId),
+                    exportFormat = allConfigs.FormatOptions.exportFormat,
+                    staticInputs = Object.keys(allConfigs[pluginId]).filter(input => allConfigs[pluginId][input]);
+
+                this.logger.debug('Exporting pipeline to format', exportFormat);
+                this.logger.debug('static inputs:', staticInputs);
+
+                context.managerConfig.namespace = 'pipeline';
+                context.pluginConfig = {
+                    format: exportFormat,
+                    staticInputs: staticInputs,
+                    extensionConfig: allConfigs.extensionConfig
+                };
+                return Q.ninvoke(this.client, 'runBrowserPlugin', pluginId, context)
+                    .then(deferred.resolve)
+                    .fail(deferred.reject);
+            });
+        } else {  // no options - just run the plugin!
+            var context = this.client.getCurrentPluginContext(pluginId);
+
+            this.logger.debug('Exporting pipeline to format', exportFormats[0]);
+
+            context.managerConfig.namespace = 'pipeline';
+            context.pluginConfig = {
+                format: exportFormats[0],
+                staticInputs: []
+            };
+            return Q.ninvoke(this.client, 'runBrowserPlugin', pluginId, context);
+        }
+
+        return deferred.promise;
+    };
+
     return ForgeActionButton;
 });

src/visualizers/panels/ForwardViz/ForwardVizPanel.js

@@ -0,0 +1,67 @@
+/*globals define, _, WebGMEGlobal*/
+/*jshint browser: true*/
+
+define([
+    'js/Constants',
+    'deepforge/globals',
+    'js/PanelBase/PanelBaseWithHeader'
+], function (
+    CONSTANTS,
+    DeepForge,
+    PanelBaseWithHeader
+) {
+    'use strict';
+
+    var ForwardVizPanel;
+
+    ForwardVizPanel = function (layoutManager, params) {
+        var options = {};
+        //set properties from options
+        options[PanelBaseWithHeader.OPTIONS.LOGGER_INSTANCE_NAME] = 'ForwardViz';
+
+        //call parent's constructor
+        PanelBaseWithHeader.apply(this, [options, layoutManager]);
+
+        this._client = params.client;
+
+        //initialize UI
+        this._initialize();
+
+        this.logger.debug('ctor finished');
+    };
+
+    //inherit from PanelBaseWithHeader
+    _.extend(ForwardVizPanel.prototype, PanelBaseWithHeader.prototype);
+
+    ForwardVizPanel.prototype._initialize = function () {
+        this.control = this;
+        this.onActivate();
+    };
+
+    ForwardVizPanel.prototype.selectedObjectChanged = function(nodeId) {
+        if (nodeId === CONSTANTS.PROJECT_ROOT_ID) {
+            DeepForge.places.MyPipelines().then(id => WebGMEGlobal.State.registerActiveObject(id));
+        }
+    };
+
+    /* OVERRIDE FROM WIDGET-WITH-HEADER */
+    /* METHOD CALLED WHEN THE WIDGET'S READ-ONLY PROPERTY CHANGES */
+        //apply parent's onReadOnlyChanged
+    ForwardVizPanel.prototype.onReadOnlyChanged = function() {
+        PanelBaseWithHeader.prototype.onReadOnlyChanged.apply(this, arguments);
+
+    };
+
+    /* * * * * * * * Visualizer life cycle callbacks * * * * * * * */
+    ForwardVizPanel.prototype.destroy = function () {
+        PanelBaseWithHeader.prototype.destroy.call(this);
+    };
+
+    ForwardVizPanel.prototype.onReadOnlyChanged =
+    ForwardVizPanel.prototype.onResize =
+    ForwardVizPanel.prototype.onActivate =
+    ForwardVizPanel.prototype.onDeactivate = function () {
+    };
+
+    return ForwardVizPanel;
+});

src/visualizers/panels/JobEditor/JobEditorPanel.js

@@ -10,11 +10,13 @@ define([
     'panels/TilingViz/TilingVizPanel',
     'panels/OutputViewer/OutputViewerPanel',
     'panels/OperationCodeEditor/OperationCodeEditorPanel',
+    'deepforge/viz/Execute',
     'js/Constants'
 ], function (
     TilingViz,
     OutputViewer,
     OperationCodeEditor,
+    Execute,
     CONSTANTS
 ) {
     'use strict';
@@ -23,11 +25,16 @@ define([
 
     JobEditorPanel = function (layoutManager, params) {
         TilingViz.call(this, layoutManager, params);
+        Execute.call(this, this._client, this.logger);
         this.readOnly = false;
     };
 
     //inherit from PanelBaseWithHeader
-    _.extend(JobEditorPanel.prototype, TilingViz.prototype);
+    _.extend(
+        JobEditorPanel.prototype,
+        Execute.prototype,
+        TilingViz.prototype
+    );
 
     JobEditorPanel.prototype.getPanels = function () {
         if (this.readOnly) {
@@ -88,6 +95,10 @@ define([
             // update the OutputViewer controller
             var i = this._panels.length;
             this._panels[i-1].control.selectedObjectChanged(nodeId);
+            // Check if the job needs to be reconnected
+            if (!this.isReadOnly()) {
+                this.checkJobExecution(node);
+            }
         }
     };
 

src/visualizers/panels/LayerEditor/LayerEditorControl.js

@@ -94,10 +94,10 @@ define([
         this._client.startTransaction(msg);
 
         TextEditorControl.prototype.saveTextFor.call(this, id, text, true);
-        this._client.setAttributes(id, 'name', layerSchema.name);
+        this._client.setAttribute(id, 'name', layerSchema.name);
 
         this._logger.debug(`Setting ctor args to ${ctorAttrs.join(',')}`);
-        this._client.setAttributes(id, Constants.CTOR_ARGS_ATTR, ctorAttrs.join(','));
+        this._client.setAttribute(id, Constants.CTOR_ARGS_ATTR, ctorAttrs.join(','));
 
         types = layerSchema.types || {};
         schema = this.getPointerMeta();
@@ -114,29 +114,29 @@ define([
 
         // Remove old pointers
         node.getPointerNames().filter(ptr => !currentPtrs[ptr])
-            .forEach(ptr => this._client.deleteMetaPointer(id, ptr));
+            .forEach(ptr => this._client.delPointerMeta(id, ptr));
 
         // Remove old attributes
         setterNames = Object.keys(layerSchema.setters);
         _.difference(currentAttrs, ctorAttrs, setterNames)
-            .forEach(attr => this._client.removeAttributeSchema(id, attr));
+            .forEach(attr => this._client.delAttributeMeta(id, attr));
 
         // Add setters
         for (i = setterNames.length; i--;) {
             schema = utils.getSetterSchema(setterNames[i], layerSchema.setters, layerSchema.defaults);
             // Get setter attr schema
             if (schema.hasOwnProperty('default')) {
-                this._client.setAttributes(id, setterNames[i], schema.default);
+                this._client.setAttribute(id, setterNames[i], schema.default);
                 delete schema.default;
             }
             if (types[setterNames[i]]) {
                 schema.type = types[setterNames[i]];
             }
-            this._client.setAttributeSchema(id, setterNames[i], schema);
+            this._client.setAttributeMeta(id, setterNames[i], schema);
         }
 
         ctorAttrs.forEach(attr =>
-            this._client.setAttributeSchema(id, attr, {
+            this._client.setAttributeMeta(id, attr, {
                 type: types[attr] || 'string'
             })
         );

src/visualizers/panels/LineGraph/LineGraphControl.js

@@ -99,6 +99,9 @@ define([
 
                 desc.type = 'line';
                 desc.points = points;
+            } else {
+                desc.xlabel = node.getAttribute('xlabel');
+                desc.ylabel = node.getAttribute('ylabel');
             }
         }
 

src/visualizers/panels/LogViewer/LogViewerControl.js

@@ -5,7 +5,7 @@
 // if the job is running, get the logs from the log-storage
 define([
     'q',
-    'deepforge/JobLogsClient',
+    'deepforge/api/JobLogsClient',
     'js/Constants',
     'deepforge/Constants',
     'panels/TextEditor/TextEditorControl'
@@ -82,7 +82,14 @@ define([
     };
 
     LogViewerControl.prototype._getRunningLogs = function (id) {
-        var logManager = new JobLogsClient({
+        var logManager;
+
+        if (!this._client.getActiveBranchName() || !this._client.getActiveProjectId()) {
+            // Logs are only stored for a given branch
+            return Q().then(() => '');
+        }
+
+        logManager = new JobLogsClient({
             logger: this._logger,
             projectId: this._client.getActiveProjectId(),
             branchName: this._client.getActiveBranchName()