v1.2.0

* Only retrieve member registry from nodes in set. Fixes #998

* WIP #998 Added test case

.codeclimate.yml

@@ -31,6 +31,8 @@ exclude_paths:
 - test/
 - src/common/lua.js
 - src/common/js-yaml.min.js
+- src/visualizers/widgets/Sidebar/lib/
 - src/visualizers/widgets/TextEditor/lib/
 - src/visualizers/widgets/PipelineIndex/styles/PipelineIndex.css
 - src/visualizers/widgets/LineGraph/lib/
+- src/visualizers/widgets/PipelineEditor/klay.js

.npmignore

@@ -36,5 +36,5 @@ blob-local-storage/
 src/seeds/nn/hash.txt
 src/seeds/pipeline/hash.txt
 
-# npm specific things
+# docs
 images/

README.md

@@ -1,13 +1,22 @@
 [![Release State](https://img.shields.io/badge/state-beta-yellow.svg)](https://img.shields.io/badge/state-beta-yellow.svg)
 [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](./LICENSE)
-[![Build Status](https://travis-ci.org/dfst/deepforge.svg?branch=master)](https://travis-ci.org/dfst/deepforge)
-[![Join the chat at https://gitter.im/dfst/deepforge](https://badges.gitter.im/dfst/deepforge.svg)](https://gitter.im/dfst/deepforge?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
-[![Stories in Ready](https://badge.waffle.io/dfst/deepforge.png?label=ready&title=Ready)](https://waffle.io/dfst/deepforge)
-
-**Notice**: DeepForge is still a work in progress and is also lacking significant documentation! That being said, any contributions and/or feedback is greatly appreciated (and feel free to always ask any questions on the gitter)!
+[![Build Status](https://travis-ci.org/deepforge-dev/deepforge.svg?branch=master)](https://travis-ci.org/deepforge-dev/deepforge)
+[![Join the chat at https://gitter.im/deepforge-dev/deepforge](https://badges.gitter.im/deepforge-dev/deepforge.svg)](https://gitter.im/deepforge-dev/deepforge?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+[![Stories in Ready](https://badge.waffle.io/deepforge-dev/deepforge.png?label=ready&title=Ready)](https://waffle.io/deepforge-dev/deepforge)
 
 # DeepForge
-DeepForge is an open-source visual development environment for deep learning. Currently, it supports Convolutional Neural Networks but we are planning on supporting additional deep learning classifiers such as RNNs and LSTMs. Additional features include real-time collaborative editing and version control.
+DeepForge is an open-source visual development environment for deep learning providing end-to-end support for creating deep learning models. This is achieved through providing the ability to design **architectures**, create training **pipelines**, and then execute these pipelines over a cluster. Using a notebook-esque api, users can get real-time feedback about the status of any of their **executions** including compare them side-by-side in real-time.
+
+![overview](images/overview.png "")
+
+Additional features include:
+- Graphical architecture editor
+- Training/testing pipeline creation
+- Distributed pipeline execution
+- Real-time pipeline feedback
+- Collaborative editing
+- Automatic version control.
+- Facilitates defining custom layers 
 
 ## Quick Start
 Simply run the following command to install deepforge with its dependencies:
@@ -22,13 +31,22 @@ Or, if you already have NodeJS (v6) installed, simply run
 npm install -g deepforge
 ```
 
-Next, start deepforge with `deepforge start`!
+Finally, start deepforge with `deepforge start`and navigate to [http://localhost:8888](http://localhost:8888) to start using DeepForge! For more, detailed instructions, check out the [wiki](https://github.com/dfst/deepforge/wiki/Installation-Guide).
 
-Finally, navigate to [http://localhost:8888](http://localhost:8888) to start using DeepForge! For more, detailed instructions, check out the [wiki](https://github.com/dfst/deepforge/wiki/Installation-Guide).
+**Note**: running deepforge w/ `deepforge start` will also require [MongoDB](https://www.mongodb.com/download-center?jmp=nav#community) to be installed locally.
 
 Also, be sure to check out the other available features of the `deepforge` cli; it can be used to update, manage your torch installation, uninstall deepforge and run individual components!
 
+## Additional Resources
+- [Intro to DeepForge Slides](https://docs.google.com/presentation/d/10_y5O3gHXSATfjHVLJg7dOdrz-tAXNWjlxhJ5SlA0ic/edit?usp=sharing)
+- [wiki](https://github.com/deepforge-dev/deepforge/wiki) containing overview, installation, configuration and developer information
+- [Starter Kit](https://github.com/deepforge-dev/examples/tree/master/starterkit) containing example pipelines demonstrating various deepforge features
+- [Examples](https://github.com/deepforge-dev/examples)
+
+- [Datamodel Developer Slides](https://docs.google.com/presentation/d/1hd3IyUlzW_TIPnzCnE-1pdz00Pw8WaIxYiOW_Hyog-M/edit#slide=id.p)
+
 ## Interested in contributing?
-Contributions are welcome! Either fork the project and submit some PR's or shoot me an email about getting more involved!
+Contributions are welcome! Either fork the project and submit some PR's or shoot me an email about getting more involved! If you have any questions, check out the [wiki](https://github.com/dfst/deepforge/wiki/) or drop me a line on the gitter!
+
 
 Sponsored by [Digital Reasoning](http://www.digitalreasoning.com/)

_config.yml

@@ -0,0 +1 @@
+theme: jekyll-theme-cayman

app.js

@@ -4,11 +4,17 @@
 var gmeConfig = require('./config'),
     webgme = require('webgme'),
     path = require('path'),
+    fs = require('fs'),
     rm_rf = require('rimraf'),
+    gracefulFs = require('graceful-fs'),
     myServer;
 
+process.chdir(__dirname);
 webgme.addToRequireJsPaths(gmeConfig);
 
+// Patch the 'fs' module to fix 'too many files open' error
+gracefulFs.gracefulify(fs);
+
 // Clear seed hash info
 ['nn', 'pipeline'].map(lib => path.join(__dirname, 'src', 'seeds', lib, 'hash.txt'))
     .forEach(file => rm_rf.sync(file));

bin/deepforge

@@ -1,6 +1,7 @@
 #!/usr/bin/env node
 
 var Command = require('commander').Command,
+    tcpPortUsed = require('tcp-port-used'),
     program = new Command(),
     childProcess = require('child_process'),
     rawSpawn = childProcess.spawn,
@@ -8,7 +9,8 @@ var Command = require('commander').Command,
     execSync = childProcess.execSync,
     path = require('path'),
     fs = require('fs'),
-    version = require('../package.json').version,
+    pkgJson = require('../package.json'),
+    version = pkgJson.version,
     exists = require('exists-file'),
     DEFAULT_CONFIG = require('./config.json'),
     merge = require('lodash.merge'),
@@ -117,32 +119,35 @@ var isLocalUri = function(protocol, uri) {
         uri.indexOf(protocol + '://127.0.0.1') === 0;
 };
 
-var checkMongo = function(args, notSilent) {
+var checkMongo = function(args, notSilent, mongoUri) {
     // check the webgme config
-    var gmeConfig = require('../config'),
-        mongoUri = gmeConfig.mongo.uri;
+    var gmeConfig = require('../config');
+
+    mongoUri = mongoUri || gmeConfig.mongo.uri;
 
     if (isLocalUri('mongodb', mongoUri)) {
+        var match = mongoUri.match(/:([0-9]+)/),
+            port = '80';
+
+        if (match) {
+            port = match[1];
+        }
+
         // Make sure mongo is running locally (using pgrep)
         try {
             execSync('pgrep mongod').toString();
             console.log('MongoDB is already running!');
         } catch (e) {  // no pIds
             console.log('Starting MongoDB...');
-            var match = mongoUri.match(/:([0-9]+)/),
-                port = '80';
-
-            if (match) {
-                port = match[1];
-            }
-
             startMongo(args, port, !notSilent);
         }
+        return tcpPortUsed.waitUntilUsed(+port, 100, 1000);
     } else if (notSilent) {
         console.log(`Cannot start remote mongo locally: ${mongoUri}`);
     } else {
         console.log(`Using remote mongo: ${mongoUri}`);
     }
+    return Q();
 };
 
 var startMongo = function(args, port, silent) {
@@ -208,28 +213,34 @@ var installTorch = function() {
         args = `clone https://github.com/torch/distro.git ${tgtDir} --recursive`.split(' ');
 
         return spawn('git', args)
-            .then(code => {
-                if (code !== 0) {
-                    if (code === 128) {
-                        console.error(`${tgtDir} is not empty. ` +
-                            'Please empty it or change the torch directory:\n' +
-                            '\n    deepforge config torch.dir NEW/TORCH/PATH\n');
+            .catch(result => {
+                var error = result.error || result.stderr ||
+                    `Torch install failed with exit code ${result.code}`;
 
-                    }
+                if (result.stderr.includes('unable to access')) {
+                    error = `Could not access the torch repository. Are you ` +
+                        `connected to the internet?\n`;
+                } else if (result.code === 128) {
+                    error = `${tgtDir} is not empty. ` +
+                        'Please empty it or change the torch directory:\n' +
+                        '\n    deepforge config torch.dir NEW/TORCH/PATH\n';
 
-                    throw `Torch install Failed with exit code ${code}`;
-                } else {  // continue installation
-                    process.chdir(tgtDir);
-                    return spawn('bash', ['install-deps'])
-                        .then(() => spawn('bash', ['install.sh'], true))
-                        .then(() => {
-                            storeConfig('torch.dir', tgtDir);
-                            console.log('Installed torch. Please close and ' +
-                                're-open your terminal to use DeepForge w/ ' +
-                                'torch support!');
-                            process.exit(0);
-                        });
                 }
+
+                console.error(error);
+                process.exit(result.code);
+            })
+            .then((code, stderr) => {
+                process.chdir(tgtDir);
+                return spawn('bash', ['install-deps'])
+                    .then(() => spawn('bash', ['install.sh'], true))
+                    .then(() => {
+                        storeConfig('torch.dir', tgtDir);
+                        console.log('Installed torch. Please close and ' +
+                            're-open your terminal to use DeepForge w/ ' +
+                            'torch support!');
+                        process.exit(0);
+                    });
             });
     } else {
         return Q();
@@ -242,21 +253,32 @@ var spawn = function(cmd, args, opts) {
         spawnOpts = typeof opts === 'object' ? opts : null,
         forwardStdin = opts === true,
         isOpen = true,
+        stderr = '',
         err;
 
     args = args || [];
     job = spawnOpts ? rawSpawn(cmd, args, spawnOpts) : rawSpawn(cmd, args);
     job.stdout.on('data', data => process.stdout.write(data));
-    job.stderr.on('data', data => process.stderr.write(data));
+    job.stderr.on('data', data => {
+        stderr += data;
+        process.stderr.write(data);
+    });
+
     job.on('close', code => {
         isOpen = false;
-        if (err) {
-            deferred.reject(err, code);
+        if (err || code !== 0) {
+            deferred.reject({
+                code: code,
+                stderr: stderr,
+                error: err
+            });
         } else {
             deferred.resolve(code);
         }
     });
-    job.on('error', e => err = e);
+    job.on('error', e => {
+        err = e;
+    });
 
     if (forwardStdin) {
         process.stdin.on('data', data => {
@@ -276,42 +298,49 @@ program.command('start')
     .option('-w, --worker [url]', 'start a worker and connect to given url. Defaults to local deepforge')
     .option('-m, --mongo', 'start MongoDB')
     .action(args => {
-        var main = path.join(__dirname, 'start-local.js');
+        var main = path.join(__dirname, 'start-local.js'),
+            current = Q();
 
         if (args.port) {
             process.env.PORT = args.port;
         }
 
+        if (args.mongo) {
+            current = current.then(() => checkMongo(args, true));
+        }
+
         if (args.server) {
-            checkMongo(args);
-            main = path.join(__dirname, '..', 'app.js');
-            spawn('node', [main]);
+            current = current
+                .then(() => checkMongo(args))
+                .then(() => {
+                    main = path.join(__dirname, '..', 'app.js');
+                    return spawn('node', [main]);
+                });
         }
 
         if (args.worker) {
             if (hasTorch()) {
-                installTorchExtras().then(() => {
-                    main = path.join(__dirname, 'start-worker.js');
-                    if (args.worker !== true) {
-                        spawn('node', [main, args.worker]);
-                    } else {
-                        spawn('node', [main]);
-                    }
-                });
+                current
+                    .then(() => installTorchExtras())
+                    .then(() => {
+                        main = path.join(__dirname, 'start-worker.js');
+                        if (args.worker !== true) {
+                            spawn('node', [main, args.worker]);
+                        } else {
+                            spawn('node', [main]);
+                        }
+                    });
             } else {
                 installTorch();
             }
         }
 
-        if (args.mongo) {
-            checkMongo(args, true);
-        }
-
         if (!args.server && !args.worker && !args.mongo) {
             // Starting everything
-            checkMongo(args);
+            current = current.then(() => checkMongo(args));
             if (hasTorch()) {
-                installTorchExtras().then(() => spawn('node', [main]));
+                current.then(() => installTorchExtras())
+                    .then(() => spawn('node', [main]));
             } else {
                 installTorch();
             }
@@ -334,7 +363,7 @@ program
         if (!args.torch || args.server) {
 
             if (args.git) {
-                pkg = 'dfst/deepforge';
+                pkg = pkgJson.repository.url;
             } else {
                 // Check the version
                 try {
@@ -447,12 +476,17 @@ program
         }
     });
 
+// extensions
+program
+    .command('extensions <command>', 'Manage deepforge extensions');
+
 module.exports = function(cmd) {
     var cmds = cmd.split(/\s+/).filter(w => !!w);
     cmds.unshift('./bin/deepforge');
     cmds.unshift('node');
     program.parse(cmds);
 };
+module.exports.checkMongo = checkMongo;
 
 if (require.main === module) {
     program.parse(process.argv);

bin/deepforge-extensions

@@ -0,0 +1,67 @@
+#!/usr/bin/env node
+
+var Command = require('commander').Command,
+    program = new Command(),
+    extender = require('../utils/extender');
+
+// Supported commands
+//  - add
+//  - remove
+//  - list
+//  - update
+program
+    .command('add <project>')
+    .description('Add an extension to deepforge')
+    .option('-n, --name <name>', 'Project name (if different from <project>)')
+    .action(project => {
+        console.log('loading extension from: ' + project);
+        extender.install(project)
+            .then(extConfig => 
+                console.log(`The ${extConfig.name} extension has been added to deepforge.`))
+            .fail(err => {
+                console.error('Could not install extension:\n');
+                console.error(err);
+                process.exit(1);
+            });
+    });
+
+program
+    .command('remove <name>').alias('rm')
+    .description('Remove an extension from deepforge')
+    .action(name => {
+        try {
+            extender.uninstall(name);
+            console.log(`${name} has been successfully removed!`);
+        } catch (e) {
+            console.error('Could not remove extension:');
+            console.error(e);
+            process.exit(1);
+        }
+    });
+
+program
+    .command('list').alias('ls')
+    .description('List installed deepforge extensions')
+    .action(() => {
+        var allExtConfigs = extender.getExtensionsConfig(),
+            types = Object.keys(allExtConfigs),
+            hasContents = false,
+            names;
+
+        for (var i = types.length; i--;) {
+            names = Object.keys(allExtConfigs[types[i]]);
+            if (names.length) {
+                hasContents = true;
+                console.log(types[i]);
+                for (var j = names.length; j--;) {
+                    console.log(`  ${names[j]}`);
+                }
+            }
+        }
+
+        if (!hasContents) {
+            console.log('No installed extensions');
+        }
+    });
+
+program.parse(process.argv);

bin/start-local.js

@@ -13,8 +13,8 @@ if (gmeConfig.blob.type === 'FS') {
 }
 
 process.env.NODE_ENV = 'local';
-execJob = spawn('npm', [
-    'start'
+execJob = spawn('node', [
+    path.join(__dirname, '..', 'app.js')
 ], env);
 execJob.stdout.pipe(process.stdout);
 execJob.stderr.pipe(process.stderr);

bin/start-worker.js

@@ -6,8 +6,7 @@ var path = require('path'),
     spawn = childProcess.spawn,
     rm_rf = require('rimraf'),
     projectConfig = require(__dirname + '/../config'),
-    executorSrc = path.join(__dirname, '..', 'node_modules', 'webgme', 'src',
-        'server', 'middleware', 'executor', 'worker'),
+    executorSrc = path.join(__dirname, '..', 'node_modules', '.bin', 'webgme-executor-worker'),
     id = Date.now(),
     workerRootPath = process.env.DEEPFORGE_WORKER_DIR || path.join(__dirname, '..', 'src', 'worker'),
     workerPath = path.join(workerRootPath, `worker_${id}`),
@@ -60,7 +59,7 @@ var startExecutor = function() {
 
     // Start the executor
     var execJob = spawn('node', [
-        'node_worker.js',
+        executorSrc,
         workerConfigPath,
         workerTmp
     ]);
@@ -74,20 +73,13 @@ var createConfigJson = function() {
 
     if (process.argv.length > 2) {
         address = process.argv[2];
+        if (!/^https?:\/\//.test(address)) {
+            address = 'http://' + address;
+        }
     }
 
     config[address] = {};
     fs.writeFile(workerConfigPath, JSON.stringify(config), startExecutor);
 };
 
-process.chdir(executorSrc);
-
-fs.mkdir(workerTmp, function() {
-    // npm install in this directory
-    var npmInstall = spawn('npm', ['install']);
-    npmInstall.stdout.pipe(process.stdout);
-    npmInstall.stderr.pipe(process.stderr);
-    npmInstall.on('close', function() {
-        createConfigJson();
-    });
-});
+fs.mkdir(workerTmp, createConfigJson);

config/components.json

@@ -2,13 +2,29 @@
     "AutoViz": {
         "preloadIds": [
             "ArchEditor",
+            "ArchIndex",
+            "PipelineIndex",
             "PipelineEditor",
             "OperationEditor",
             "ExecutionView"
-        ]
+        ],
+        "visualizerOverrides": {
+            "": "ForwardViz",
+            "MyArtifacts": "ArtifactIndex",
+            "MyArchitectures": "ArchIndex",
+            "MyExecutions": "ExecutionIndex",
+            "MyPipelines": "PipelineIndex"
+        }
+    },
+    "PipelineEditor": {
+        "itemName": "operation"
+    },
+    "ExecutionView": {
+        "itemName": "job"
     },
     "ArchEditor": {
         "hotkeys": "none",
+        "itemName": "layer",
         "LayerColors": {}
     },
     "BreadcrumbHeader": {
@@ -32,10 +48,6 @@
             "ImportTorch": {
                 "icon": "import_export",
                 "priority": -1
-            },
-            "GenerateExecFile": {
-                "icon": "play_for_work",
-                "priority": -1
             }
         }
     },
@@ -43,11 +55,11 @@
         "rootMenuClass": "deepforge-logo",
         "rootDisplayName": "DeepForge"
     },
-    "CHFLayout": {
+    "SidebarLayout": {
         "panels": [
             {
-                "id": "Header",
-                "panel": "BreadcrumbHeader/BreadcrumbHeaderPanel",
+                "id": "WorkerHeader",
+                "panel": "WorkerHeader/WorkerHeaderPanel",
                 "container": "header",
                 "DEBUG_ONLY": false
             },
@@ -63,6 +75,12 @@
                 "container": "center",
                 "DEBUG_ONLY": false
             },
+            {
+                "id": "Sidebar",
+                "panel": "Sidebar/SidebarPanel",
+                "container": "sidebar",
+                "DEBUG_ONLY": false
+            },
             {
                 "id": "ForgeActionButton",
                 "panel": "ForgeActionButton/ForgeActionButton",

config/config.webgme.js

@@ -9,6 +9,7 @@ var config = require('webgme/config/config.default'),
 // The paths can be loaded from the webgme-setup.json
 config.plugin.basePaths.push(__dirname + '/../src/plugins');
 config.plugin.basePaths.push(__dirname + '/../node_modules/webgme-simple-nodes/src/plugins');
+config.visualization.layout.basePaths.push(__dirname + '/../src/layouts');
 config.visualization.layout.basePaths.push(__dirname + '/../node_modules/webgme-chflayout/src/layouts');
 config.visualization.decoratorPaths.push(__dirname + '/../src/decorators');
 config.visualization.decoratorPaths.push(__dirname + '/../node_modules/webgme-easydag/src/decorators');
@@ -20,6 +21,7 @@ config.seedProjects.basePaths.push(__dirname + '/../src/seeds/devPipelineTests')
 config.seedProjects.basePaths.push(__dirname + '/../src/seeds/project');
 config.seedProjects.basePaths.push(__dirname + '/../src/seeds/cifar10');
 config.seedProjects.basePaths.push(__dirname + '/../src/seeds/xor');
+config.seedProjects.basePaths.push(__dirname + '/../src/seeds/devProject');
 
 
 
@@ -31,6 +33,8 @@ config.visualization.panelPaths.push(__dirname + '/../src/visualizers/panels');
 
 
 config.rest.components['execution/logs'] = __dirname + '/../src/routers/JobLogsAPI/JobLogsAPI.js';
+config.rest.components['job/origins'] = __dirname + '/../src/routers/JobOriginAPI/JobOriginAPI.js';
+config.rest.components['execution/pulse'] = __dirname + '/../src/routers/ExecPulse/ExecPulse.js';
 
 // Visualizer descriptors
 config.visualization.visualizerDescriptors.push(__dirname + '/../src/visualizers/Visualizers.json');
@@ -55,7 +59,7 @@ config.requirejsPaths = {
   'widgets/FloatingActionButton': './node_modules/webgme-fab/src/visualizers/widgets/FloatingActionButton'
 };
 
-config.visualization.layout.default = 'CHFLayout';
+config.visualization.layout.default = 'SidebarLayout';
 config.mongo.uri = 'mongodb://127.0.0.1:27017/deepforge';
 validateConfig(config);
 module.exports = config;

install.sh

@@ -42,10 +42,10 @@ detect_profile() {
 detect_profile
 
 set_node_version() {
-    # Install nodejs v6.2.0
-    echo "Installing NodeJS v6.2.0"
-    nvm install v6.2.0
-    nvm alias default v6.2.0
+    # Install nodejs v6.2.1
+    echo "Installing NodeJS v6.2.1"
+    nvm install v6.2.1
+    nvm alias default v6.2.1
 
     # Install npm@2
     npm install npm@2 -g

package.json

@@ -1,33 +1,44 @@
 {
   "name": "deepforge",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/deepforge-dev/deepforge.git"
+  },
   "bin": {
     "deepforge": "./bin/deepforge"
   },
   "scripts": {
-    "start": "node app.js",
+    "start": "./bin/deepforge start",
+    "postinstall": "node utils/reinstall-extensions.js",
     "start-dev": "NODE_ENV=dev node app.js",
     "local": "node ./bin/start-local.js",
     "worker": "node ./bin/start-worker.js",
-    "test": "mkdir ./test-tmp; node ./node_modules/mocha/bin/mocha --recursive test",
-    "watch-test": "./node_modules/nodemon/bin/nodemon.js --exec 'node ./node_modules/mocha/bin/mocha --recursive test'",
+    "test": "mkdir ./test-tmp; mocha --recursive test",
+    "watch-test": "nodemon --exec 'mocha --recursive test'",
     "build-nn": "node ./utils/nn-parser.js"
   },
-  "version": "0.15.0",
+  "version": "1.2.0",
   "dependencies": {
     "commander": "^2.9.0",
     "dotenv": "^2.0.0",
     "exists-file": "^2.1.0",
     "express": "^4.14.0",
+    "graceful-fs": "^4.1.10",
     "lodash.difference": "^4.1.2",
     "lodash.merge": "^4.5.1",
+    "lodash.template": "^4.4.0",
+    "mongodb": "^2.2.10",
     "nodemon": "^1.9.2",
+    "npm": "^4.0.5",
     "q": "1.4.1",
     "rimraf": "^2.4.0",
-    "webgme": "^2.0.0",
-    "webgme-autoviz": "dfst/webgme-autoviz",
+    "tcp-port-used": "^0.1.2",
+    "webgme": "^2.7.1",
+    "webgme-autoviz": "^2.2.0",
     "webgme-breadcrumbheader": "^2.1.1",
     "webgme-chflayout": "^2.0.0",
     "webgme-easydag": "dfst/webgme-easydag",
+    "webgme-executor-worker": "^1.0.1",
     "webgme-fab": "dfst/webgme-fab",
     "webgme-simple-nodes": "^2.1.0"
   },

src/common/Constants.js

@@ -1,31 +1,58 @@
 /* globals define */
-define({
-    LINE_OFFSET: 'lineOffset',
+(function(root, factory){
+    if(typeof define === 'function' && define.amd) {
+        define([], function(){
+            return factory();
+        });
+    } else if(typeof module === 'object' && module.exports) {
+        module.exports = factory();
+    } else {
+        root.CONSTANTS = factory();
+    }
+}(this, function() {
+    return {
+        CONTAINED_LAYER_SET: 'addLayers',
+        CONTAINED_LAYER_INDEX: 'index',
 
-    // DeepForge metadata creation in dist execution
-    START_CMD: 'deepforge-cmd',
+        LINE_OFFSET: 'lineOffset',
+        DISPLAY_COLOR: 'displayColor',
 
-    IMAGE: {  // all prefixed w/ 'IMG' for simple upload detection
-        PREFIX: 'IMG',
-        BASIC: 'IMG-B',
-        CREATE: 'IMG-C',
-        UPDATE: 'IMG-U',
-        NAME: 'IMAGE-N'  // No upload required
-    },
+        // DeepForge metadata creation in dist execution
+        START_CMD: 'deepforge-cmd',
 
-    GRAPH_CREATE: 'GRAPH',
-    GRAPH_PLOT: 'PLOT',
-    GRAPH_CREATE_LINE: 'LINE',
+        IMAGE: {  // all prefixed w/ 'IMG' for simple upload detection
+            PREFIX: 'IMG',
+            BASIC: 'IMG-B',
+            CREATE: 'IMG-C',
+            UPDATE: 'IMG-U',
+            NAME: 'IMAGE-N'  // No upload required
+        },
 
-    // Code Generation Constants
-    CTOR_ARGS_ATTR: 'ctor_arg_order',
+        GRAPH_CREATE: 'GRAPH',
+        GRAPH_PLOT: 'PLOT',
+        GRAPH_CREATE_LINE: 'LINE',
+        GRAPH_LABEL_AXIS: {
+            X: 'X',
+            Y: 'Y'
+        },
 
-    // Operation types
-    OP: {
-        INPUT: 'Input',
-        OUTPUT: 'Output'
-    },
+        // Code Generation Constants
+        CTOR_ARGS_ATTR: 'ctor_arg_order',
 
-    // Job stdout update
-    STDOUT_UPDATE: 'stdout_update'
-});
+        // Operation types
+        OP: {
+            INPUT: 'Input',
+            OUTPUT: 'Output'
+        },
+
+        // Heartbeat constants (ExecPulse router)
+        PULSE: {
+            DEAD: 0,
+            ALIVE: 1,
+            DOESNT_EXIST: 2
+        },
+
+        // Job stdout update
+        STDOUT_UPDATE: 'stdout_update'
+    };
+}));

src/common/LayerParser.js

@@ -265,6 +265,7 @@
     var findTorchClass = function(ast){ 
         var torchClassArgs,  // args for `torch.class(...)`
             name = '',
+            alias,
             baseType,
             params,
             setters = {},
@@ -283,6 +284,7 @@
                     name = torchClassArgs[0];
                     if(name !== ''){
                         name = name.replace('nn.', '');
+                        alias = func.names[0] || name;
                         if (torchClassArgs.length > 1) {
                             baseType = torchClassArgs[1].replace('nn.', '');
                         }
@@ -302,7 +304,7 @@
                 attrName;
 
             // Record the setter functions
-            if (isSetterMethod(curr, parent, name)) {
+            if (isSetterMethod(curr, parent, alias)) {
                 firstLine = curr.block.stats[0];
                 // just use the attribute attrName for now...
                 attrName = getSettingAttrName(firstLine);
@@ -316,7 +318,7 @@
                 } else {
                     setters[attrName] = schema;
                 }
-            } else if (isInitFn(curr, name)) {  // Record the defaults
+            } else if (isInitFn(curr, alias)) {  // Record the defaults
                 paramDefs = getAttrsAndVals(curr);
                 attrDefs = getClassAttrDefs(curr);
                 types = inferParamTypes(curr, paramDefs);

src/common/api/APIClient.js

@@ -0,0 +1,75 @@
+/*globals define*/
+define([
+    'q',
+    'superagent'
+], function(
+    Q,
+    superagent
+) {
+    'use strict';
+
+    // Wrap the ability to read, update, and delete logs using the JobLogsAPI
+    var APIClient = function(params) {
+        params = params || {};
+
+        this.logger = this.logger || params.logger.fork('APIClient');
+
+        // Get the server url
+        this.token = params.token;
+        this.origin = this._getServerUrl(params);
+        this.relativeUrl = this.relativeUrl || '';
+        this.url = this.origin + this.relativeUrl;
+
+        this.logger.debug(`Setting url to ${this.url}`);
+
+        this.branch = params.branchName;
+        this.project = params.projectId;
+        this._modifiedJobs = [];
+
+        this.logger.debug(`Using <project>:<branch>: "${this.project}"/"${this.branch}"`);
+        this.logger.info('ctor finished');
+    };
+
+    APIClient.prototype._getServerUrl = function(params) {
+        if (typeof window !== 'undefined') {
+            return window.location.origin;
+        }
+
+        // If not in browser, set using the params
+        var server = params.server || '127.0.0.1',
+            port = params.port || '80',
+            protocol = params.httpsecure ? 'https' : 'http';  // default is http
+
+        return params.origin || `${protocol}://${server}:${port}`;
+    };
+
+    APIClient.prototype.getUrl = function() {
+        return this.url;
+    };
+
+    APIClient.prototype._request = function(method, jobId, content) {
+        var deferred = Q.defer(),
+            req = superagent[method](this.getUrl(jobId));
+
+        this.logger.debug(`sending ${method} request to ${this.getUrl(jobId)}`);
+        if (this.token) {
+            req.set('Authorization', 'Bearer ' + this.token);
+        }
+
+        if (content) {
+            req = req.send(content);
+        }
+
+        req.end((err, res) => {
+            if (err || res.status > 399) {
+                return deferred.reject(res || err);
+            }
+
+            return deferred.resolve(res);
+        });
+
+        return deferred.promise;
+    };
+
+    return APIClient;
+});

src/common/api/ExecPulseClient.js

@@ -0,0 +1,44 @@
+/* globals define */
+define([
+    './APIClient'
+], function(
+    APIClient
+) {
+    'use strict';
+
+    var ExecPulseClient = function(params) {
+        this.relativeUrl = '/execution/pulse/';
+        this.logger = params.logger.fork('ExecPulseClient');
+        APIClient.call(this, params);
+    };
+
+    ExecPulseClient.prototype = Object.create(APIClient.prototype);
+
+    ExecPulseClient.prototype.getUrl = function(hash) {
+        return this.url + hash;
+    };
+
+    // - update the heartbeat
+    // - check the heartbeat
+    // - delete the heartbeat
+    ExecPulseClient.prototype.update = function(hash) {
+        return this._request('post', hash)
+            .catch(err => {
+                throw err.text || err;
+            });
+    };
+
+    ExecPulseClient.prototype.check = function(hash) {
+        return this._request('get', hash)
+            .then(res => JSON.parse(res.text))
+            .catch(err => {
+                throw err.text || err;
+            });
+    };
+
+    ExecPulseClient.prototype.clear = function(hash) {
+        return this._request('delete', hash);
+    };
+
+    return ExecPulseClient;
+});

src/common/api/JobLogsClient.js

@@ -1,51 +1,38 @@
 /* globals define */
 define([
+    './APIClient',
     'q',
     'superagent'
 ], function(
+    APIClient,
     Q,
     superagent
 ) {
     'use strict';
 
     // Wrap the ability to read, update, and delete logs using the JobLogsAPI
+    var METADATA_FIELDS = [
+        'lineCount'
+    ];
     var JobLogsClient = function(params) {
         params = params || {};
 
-        this.logger = params.logger.fork('JobLogsClient');
-
-        // Get the server url
-        this.token = params.token;
-        this.origin = this._getServerUrl(params);
         this.relativeUrl = '/execution/logs';
-        this.url = this.origin + this.relativeUrl;
-
-        this.logger.debug(`Setting url to ${this.url}`);
+        this.logger = params.logger.fork('JobLogsClient');
+        APIClient.call(this, params);
 
         // Get the project, branch name
         if (!(params.branchName && params.projectId)) {
             throw Error('"branchName" and "projectId" required');
         }
-        this.branch = params.branchName;
-        this.project = params.projectId;
+
         this._modifiedJobs = [];
 
         this.logger.debug(`Using <project>:<branch>: "${this.project}"/"${this.branch}"`);
         this.logger.info('ctor finished');
     };
 
-    JobLogsClient.prototype._getServerUrl = function(params) {
-        if (typeof window !== 'undefined') {
-            return window.location.origin;
-        }
-
-        // If not in browser, set using the params
-        var server = params.server || '127.0.0.1',
-            port = params.port || '80',
-            protocol = params.httpsecure ? 'https' : 'http';  // default is http
-
-        return params.origin || `${protocol}://${server}:${port}`;
-    };
+    JobLogsClient.prototype = Object.create(APIClient.prototype);
 
     // This method could be optimized - it could make a log of requests
     JobLogsClient.prototype.fork = function(forkName) {
@@ -79,53 +66,54 @@ define([
     };
 
     JobLogsClient.prototype.getUrl = function(jobId) {
+        var url = this.url;
+            
+        if (typeof jobId !== 'string') {
+            url = this.url + jobId.route;
+            jobId = jobId.jobId;
+        }
+
         return [
-            this.url,
+            url,
             encodeURIComponent(this.project),
             encodeURIComponent(this.branch),
             encodeURIComponent(jobId)
         ].join('/');
     };
 
-    JobLogsClient.prototype._logRequest = function(method, jobId, content) {
-        var deferred = Q.defer(),
-            req = superagent[method](this.getUrl(jobId));
-
-        this.logger.info(`sending ${method} request to ${this.getUrl(jobId)}`);
-        if (this.token) {
-            req.set('Authorization', 'Bearer ' + this.token);
-        }
-
-        if (content) {
-            req = req.send(content);
-        }
-
-        req.end((err, res) => {
-            if (err || res.status > 399) {
-                return deferred.reject(err || res.status);
-            }
-
-            return deferred.resolve(res);
-        });
-
-        return deferred.promise;
+    var hasRequiredFields = function(md) {
+        return METADATA_FIELDS.reduce((passing, nextField) => {
+            return passing && md.hasOwnProperty(nextField);
+        }, true);
     };
 
-    JobLogsClient.prototype.appendTo = function(jobId, text) {
+    JobLogsClient.prototype.appendTo = function(jobId, text, metadata) {
         this._modifiedJobs.push(jobId);
         this.logger.info(`Appending logs to ${jobId}`);
-        return this._logRequest('patch', jobId, {patch: text});
+
+        if (metadata && !hasRequiredFields(metadata)) {
+            throw Error(`Required metadata fields: ${METADATA_FIELDS.join(', ')}`);
+        }
+        metadata = metadata || {};
+        metadata.patch = text;
+        return this._request('patch', jobId, metadata);
     };
 
     JobLogsClient.prototype.getLog = function(jobId) {
         this.logger.info(`Getting logs for ${jobId}`);
-        return this._logRequest('get', jobId)
+        return this._request('get', jobId)
             .then(res => res.text);
     };
 
     JobLogsClient.prototype.deleteLog = function(jobId) {
         this.logger.info(`Deleting logs for ${jobId}`);
-        return this._logRequest('delete', jobId);
+        return this._request('delete', jobId);
+    };
+
+    JobLogsClient.prototype.getMetadata = function(jobId) {
+        this.logger.info(`Getting line count for ${jobId}`);
+        return this._request('get', {jobId: jobId, route: '/metadata'})
+            .then(res => JSON.parse(res.text));
     };
 
     return JobLogsClient;

src/common/api/JobOriginClient.js

@@ -0,0 +1,60 @@
+/* globals define */
+define([
+    './APIClient'
+], function(
+    APIClient
+) {
+    'use strict';
+
+    var JobOriginClient = function(params) {
+        this.relativeUrl = '/job/origins/';
+        this.logger = params.logger.fork('JobOriginClient');
+        APIClient.call(this, params);
+    };
+
+    JobOriginClient.prototype = Object.create(APIClient.prototype);
+
+    // - Record the origin
+    // - Look up the origin
+    // - Delete record
+    JobOriginClient.prototype.getUrl = function(hash) {
+        return this.url + hash;
+    };
+
+    JobOriginClient.prototype.record = function(hash, info) {
+        var jobInfo = {
+            hash: hash,
+            nodeId: info.nodeId,
+            job: info.job,
+            project: info.project || this.project,
+            branch: info.branch || this.branch,
+            execution: info.execution
+        };
+
+        return this._request('post', hash, jobInfo)
+            .catch(err => {
+                throw err.text || err;
+            });
+    };
+
+    JobOriginClient.prototype.getOrigin = function(hash) {
+        return this._request('get', hash)
+            .then(res => JSON.parse(res.text))
+            .catch(res => {
+                if (res.status && res.status === 404) {
+                    return null;
+                }
+                throw res;
+            });
+    };
+
+    JobOriginClient.prototype.fork = function(hash, forkName) {
+        return this._request('patch', hash, {branch: forkName});
+    };
+
+    JobOriginClient.prototype.deleteRecord = function(hash) {
+        return this._request('delete', hash);
+    };
+
+    return JobOriginClient;
+});

src/common/globals.js

@@ -57,7 +57,7 @@ define([
     };
 
     var createNamedNode = function(baseId, parentId, isMeta) {
-        var newId = client.createChild({parentId, baseId}),
+        var newId = client.createNode({parentId, baseId}),
             baseNode = client.getNode(baseId),
             basename = 'New' + baseNode.getAttribute('name'),
             newName = getUniqueName(parentId, basename);
@@ -72,7 +72,7 @@ define([
             client.setRegistry(newId, 'isAbstract', false);
         }
 
-        client.setAttributes(newId, 'name', newName);
+        client.setAttribute(newId, 'name', newName);
         return newId;
     };
 
@@ -83,7 +83,9 @@ define([
             exists = {},
             i = 2;
 
-        children.forEach(child => exists[child.getAttribute('name')] = true);
+        children
+            .filter(child => child !== null)
+            .forEach(child => exists[child.getAttribute('name')] = true);
 
         while (exists[name]) {
             name = basename + '_' + i;
@@ -191,7 +193,7 @@ define([
                 .getId();
 
         // Look up the parent container
-        DeepForge.places[placeName]().then(parentId => {
+        return DeepForge.places[placeName]().then(parentId => {
 
             client.startTransaction(msg);
             newId = createNamedNode(baseId, parentId, !!metasheetName);
@@ -265,14 +267,12 @@ define([
         }
 
         dataBaseId = dataBase.getId();
-        dataTypes = metanodes.filter(n => client.isTypeOf(n.getId(), dataBaseId))
+        dataTypes = metanodes.filter(n => n.isTypeOf(dataBaseId))
             .filter(n => !n.getRegistry('isAbstract'))
             .map(node => node.getAttribute('name'));
 
-        //this.logger.info(`Found ${dataTypes.length} data types`);
-
-        // Add the target type to the pluginMetadata... hacky :/
-        var metadata = WebGMEGlobal.allPluginsMetadata[UPLOAD_PLUGIN], 
+        // Add the target type to the pluginMetadata...
+        var metadata = WebGMEGlobal.allPluginsMetadata[UPLOAD_PLUGIN],
             config = metadata.configStructure
                 .find(opt => opt.name === DATA_TYPE_CONFIG.name);
 
@@ -297,7 +297,8 @@ define([
     };
 
     DeepForge.last = {};
-    DeepForge.create  = {};
+    DeepForge.create = {};
+    DeepForge.register = {};
     instances.forEach(type => {
         DeepForge.create[type] = function() {
             return createNew.call(null, type);
@@ -308,6 +309,10 @@ define([
         DeepForge.create[type] = function() {
             return createNew.call(null, type, type);
         };
+        DeepForge.register[type] = function(id) {
+            // Add the given element to the metasheet!
+            return addToMetaSheet(id, type);
+        };
     });
 
     DeepForge.create.Layer = createCustomLayer;

src/common/lua.js

@@ -2565,7 +2565,7 @@ function LuaContext(){
             case 'number':
                 return c;
             case 'string':
-                return parseInt(s) || dummy0;
+                return parseInt(c) || dummy0;
             default:
                 if (c == dummy0){
                     return c;

src/common/plugin/LocalExecutor.js

@@ -20,13 +20,13 @@ define([
                 var output = cntrs
                     .find(cntr => {
                         var metaNode = this.core.getMetaType(cntr),
-                            metaName = this.core.getAttribute(metaNode, 'name');
+                            metaName = this.getAttribute(metaNode, 'name');
                         return metaName === 'Outputs';
                     });
                 return this.core.loadChildren(output);
             })
             .then(dataNodes => {
-                hash = this.core.getAttribute(dataNodes[0], 'data');
+                hash = this.getAttribute(dataNodes[0], 'data');
                 return this.getOutputs(node);
             })
             .then(outputTuples => {
@@ -48,7 +48,7 @@ define([
         var hash,
             typeId = this.core.getPointerPath(node, 'type'),
             type,
-            artifactName = this.core.getAttribute(node, 'artifactName');
+            artifactName = this.getAttribute(node, 'artifactName');
 
         return this.core.loadByPath(this.rootNode, typeId)
             .then(_type => {
@@ -58,25 +58,20 @@ define([
             .then(saveDir => this.core.loadChildren(saveDir))
             .then(artifacts => {
                 return artifacts.find(artifact =>
-                    this.core.getAttribute(artifact, 'name') === artifactName &&
+                    this.getAttribute(artifact, 'name') === artifactName &&
                         this.isMetaTypeOf(artifact, type));
             })
             .then(matchingArtifact => {
-                hash = matchingArtifact && this.core.getAttribute(matchingArtifact, 'data');
+                hash = matchingArtifact && this.getAttribute(matchingArtifact, 'data');
                 // If no hash, just continue (the subsequent ops will receive 'nil')
                 if (!hash) {
                     return this.onOperationComplete(node);
                 } else {
                     return this.getOutputs(node)
                         .then(outputPairs => {
-                            var outputs = outputPairs.map(pair => pair[2]),
-                                paths;
-
-                            paths = outputs.map(output => this.core.getPath(output));
+                            var outputs = outputPairs.map(pair => pair[2]);
                             // Get the 'data' hash and store it in the output data ports
-                            this.logger.info(`Loading blob data (${hash}) to ${paths.map(p => `"${p}"`)}`);
-
-                            outputs.forEach(output => this.core.setAttribute(output, 'data', hash));
+                            outputs.forEach(output => this.setAttribute(output, 'data', hash));
 
                             this.onOperationComplete(node);
                         });
@@ -99,7 +94,7 @@ define([
 
                 if (containers.length > 1) {
                     saveDir = containers.find(c =>
-                        this.core.getAttribute(c, 'name').toLowerCase().indexOf('artifacts') > -1
+                        this.getAttribute(c, 'name').toLowerCase().indexOf('artifacts') > -1
                     ) || containers[0];
                 }
 
@@ -121,8 +116,8 @@ define([
             .then(artifacts => {
                 currNameHashPairs = artifacts
                     .map(node => [
-                        this.core.getAttribute(node, 'name'),
-                        this.core.getAttribute(node, 'data')
+                        this.getAttribute(node, 'name'),
+                        this.getAttribute(node, 'data')
                     ]);
                 return this.getInputs(node);
             })
@@ -142,9 +137,9 @@ define([
 
                 // Remove nodes that already exist
                 dataNodes = allDataNodes.filter(dataNode => {
-                    var hash = this.core.getAttribute(dataNode, 'data'),
+                    var hash = this.getAttribute(dataNode, 'data'),
                         name = this.core.getOwnAttribute(node, 'saveName') ||
-                            this.core.getAttribute(dataNode, 'name');
+                            this.getAttribute(dataNode, 'name');
 
                     return !(currNameHashPairs
                         .find(pair => pair[0] === name && pair[1] === hash));
@@ -153,13 +148,16 @@ define([
                 // get the input node
                 if (dataNodes.length !== 0) {
                     var newNodes = this.core.copyNodes(dataNodes, parentNode),
-                        newName = this.core.getOwnAttribute(node, 'saveName');
+                        newName = this.core.getOwnAttribute(node, 'saveName'),
+                        createdAt = Date.now();
+
                     if (newName) {
-                        newNodes.forEach(node =>
-                            this.core.setAttribute(node, 'name', newName)
-                        );
+                        newNodes.forEach(node => {
+                            this.setAttribute(node, 'name', newName);
+                            this.setAttribute(node, 'createdAt', createdAt);
+                        });
                     }
-                    var hashes = dataNodes.map(n => this.core.getAttribute(n, 'data'));
+                    var hashes = dataNodes.map(n => this.getAttribute(n, 'data'));
                     this.logger.info(`saving hashes: ${hashes.map(h => `"${h}"`)}`);
                 } else if (allDataNodes.length === 0) {
                     this.logger.warn('No data nodes found!');

src/common/viz/Buttons.js

@@ -1,8 +1,10 @@
 /*globals define, WebGMEGlobal*/
 define([
+    'deepforge/globals',
     'widgets/EasyDAG/Buttons',
     'widgets/EasyDAG/Icons'
 ], function(
+    DeepForge,
     EasyDAGButtons,
     Icons
 ) {
@@ -57,9 +59,79 @@ define([
         return n && n.getBaseId();
     };
 
+    var CloneAndEdit = function(params) {
+        GoToBase.call(this, params);
+    };
+
+    CloneAndEdit.prototype = Object.create(GoToBase.prototype);
+    CloneAndEdit.prototype.BTN_CLASS = 'clone-and-edit';
+
+    CloneAndEdit.prototype._render = function() {
+        var lineRadius = GoToBase.SIZE - GoToBase.BORDER,
+            btnColor = '#a5d6a7';
+
+        if (this.disabled) {
+            btnColor = '#e0e0e0';
+        }
+
+        this.$el
+            .append('circle')
+            .attr('r', GoToBase.SIZE)
+            .attr('fill', btnColor);
+
+        // Show the 'code' icon
+        Icons.addIcon('code', this.$el, {
+            radius: lineRadius
+        });
+    };
+
+    CloneAndEdit.prototype._onClick = function(item) {
+        var node = client.getNode(item.id),
+            baseId = node && node.getBaseId(),
+            base = baseId && client.getNode(baseId),
+            typeId = base && base.getBaseId(),
+            type = typeId && client.getNode(typeId),
+            ctrName,
+            typeName,
+            name,
+            newId;
+
+        // Clone the given node's base and change to it
+        if (type) {
+            typeName = type.getAttribute('name');
+            ctrName = `My${typeName}s`;
+            if (DeepForge.places[ctrName]) {
+                DeepForge.places[ctrName]().then(ctrId => {
+                    type = base.getAttribute('name');
+                    client.startTransaction(`Creating new ${typeName} from ${item.name}`);
+                    newId = client.copyNode(baseId, ctrId);
+                    name = node.getAttribute('name');
+                    client.setAttribute(newId, 'name', 'Copy of ' + name);
+                    DeepForge.register[typeName](newId);
+
+                    client.completeTransaction();
+                    WebGMEGlobal.State.registerActiveObject(newId);
+                });
+            }
+        } else {
+            this._logger.warn('Could not find the base node!');
+        }
+    };
+
+    var Insert = function(params) {
+        EasyDAGButtons.ButtonBase.call(this, params);
+    };
+
+    Insert.prototype = Object.create(EasyDAGButtons.Add.prototype);
+    Insert.prototype._onClick = function(item) {
+        this.onInsertButtonClicked(item);
+    };
+
     return {
         DeleteOne: EasyDAGButtons.DeleteOne,
-        GoToBase: GoToBase
+        GoToBase: GoToBase,
+        CloneAndEdit: CloneAndEdit,
+        Insert: Insert
     };
 });
 

src/common/viz/Execute.js

@@ -3,21 +3,31 @@
 define([
     'q',
     'executor/ExecutorClient',
+    'deepforge/api/ExecPulseClient',
+    'deepforge/api/JobOriginClient',
+    'deepforge/Constants',
     'panel/FloatingActionButton/styles/Materialize'
 ], function(
     Q,
     ExecutorClient,
+    ExecPulseClient,
+    JobOriginClient,
+    CONSTANTS,
     Materialize
 ) {
 
     var Execute = function(client, logger) {
         this.client = this.client || client;
         this.logger = this.logger || logger;
+        this.pulseClient = new ExecPulseClient({
+            logger: this.logger
+        });
         this._executor = new ExecutorClient({
             logger: this.logger.fork('ExecutorClient'),
             serverPort: WebGMEGlobal.gmeConfig.server.port,
             httpsecure: window.location.protocol === 'https:'
         });
+        this.originManager = new JobOriginClient({logger: this.logger});
     };
 
     Execute.prototype.executeJob = function(node) {
@@ -102,6 +112,21 @@ define([
             .fail(err => this.logger.error(`Job cancel failed: ${err}`));
     };
 
+    Execute.prototype._setJobStopped = function(jobId, silent) {
+        if (!silent) {
+            var name = this.client.getNode(jobId).getAttribute('name');
+            this.client.startTransaction(`Stopping "${name}" job`);
+        }
+
+        this.client.delAttribute(jobId, 'jobId');
+        this.client.delAttribute(jobId, 'secret');
+        this.client.setAttribute(jobId, 'status', 'canceled');
+
+        if (!silent) {
+            this.client.completeTransaction();
+        }
+    };
+
     Execute.prototype.stopJob = function(job, silent) {
         var jobId;
 
@@ -109,18 +134,7 @@ define([
         jobId = job.getId();
 
         this.silentStopJob(job);
-
-        if (!silent) {
-            this.client.startTransaction(`Stopping "${name}" job`);
-        }
-
-        this.client.delAttributes(jobId, 'jobId');
-        this.client.delAttributes(jobId, 'secret');
-        this.client.setAttributes(jobId, 'status', 'canceled');
-
-        if (!silent) {
-            this.client.completeTransaction();
-        }
+        this._setJobStopped(jobId, silent);
     };
 
 
@@ -165,14 +179,17 @@ define([
     };
 
     Execute.prototype._stopExecution = function(execNode, inTransaction) {
-        var msg = `Canceling ${execNode.getAttribute('name')} execution`;
+        var msg = `Canceling ${execNode.getAttribute('name')} execution`,
+            jobIds;
 
         if (!inTransaction) {
             this.client.startTransaction(msg);
         }
 
-        this._silentStopExecution(execNode);
-        this.client.setAttributes(execNode.getId(), 'status', 'canceled');
+        jobIds = this._silentStopExecution(execNode);
+
+        this.client.setAttribute(execNode.getId(), 'status', 'canceled');
+        jobIds.forEach(jobId => this._setJobStopped(jobId, true));
 
         if (!inTransaction) {
             this.client.completeTransaction();
@@ -180,11 +197,88 @@ define([
     };
 
     Execute.prototype._silentStopExecution = function(execNode) {
-        var jobIds = execNode.getChildrenIds();
+        var runningJobIds = execNode.getChildrenIds()
+            .map(id => this.client.getNode(id))
+            .filter(job => this.isRunning(job));  // get running jobs
 
-        jobIds.map(id => this.client.getNode(id))
-            .filter(job => this.isRunning(job))  // get running jobs
-            .forEach(job => this.silentStopJob(job));  // stop them
+        runningJobIds.forEach(job => this.silentStopJob(job));  // stop them
+
+        return runningJobIds;
+    };
+
+    // Resuming Executions
+    Execute.prototype.checkJobExecution= function (job) {
+        var pipelineId = job.getParentId(),
+            pipeline = this.client.getNode(pipelineId);
+
+        // First check the parent execution. If it doesn't exist, then check the job
+        return this.checkPipelineExecution(pipeline)
+            .then(tryToStartJob => {
+                if (tryToStartJob) {
+                    return this._checkJobExecution(job);
+                }
+            });
+    };
+
+    Execute.prototype._checkJobExecution = function (job) {
+        var jobId = job.getAttribute('jobId'),
+            status = job.getAttribute('status');
+
+        if (status === 'running' && jobId) {
+            return this.pulseClient.check(jobId)
+                .then(status => {
+                    if (status !== CONSTANTS.PULSE.DOESNT_EXIST) {
+                        return this._onOriginBranch(jobId).then(onBranch => {
+                            if (onBranch) {
+                                this.runExecutionPlugin('ExecuteJob', {
+                                    node: job
+                                });
+                            }
+                        });
+                    } else {
+                        this.logger.warn(`Could not restart job: ${job.getId()}`);
+                    }
+                });
+        }
+        return Q();
+    };
+
+    Execute.prototype._onOriginBranch = function (hash) {
+        return this.originManager.getOrigin(hash)
+            .then(origin => {
+                var currentBranch = this.client.getActiveBranchName();
+                if (origin && origin.branch) {
+                    return origin.branch === currentBranch;
+                }
+                return false;
+            });
+    };
+
+    Execute.prototype.checkPipelineExecution = function (pipeline) {
+        var runId = pipeline.getAttribute('runId'),
+            status = pipeline.getAttribute('status'),
+            tryToStartJob = true;
+
+        if (status === 'running' && runId) {
+            return this.pulseClient.check(runId)
+                .then(status => {
+                    if (status === CONSTANTS.PULSE.DEAD) {
+                        // Check the origin branch
+                        return this._onOriginBranch(runId).then(onBranch => {
+                            if (onBranch) {
+                                this.runExecutionPlugin('ExecutePipeline', {
+                                    node: pipeline
+                                });
+                            }
+                        });
+                    }
+                    // only try to start if the pulse info doesn't exist
+                    tryToStartJob = status === CONSTANTS.PULSE.DOESNT_EXIST;
+                    return tryToStartJob;
+                });
+        } else {
+            return Q().then(() => tryToStartJob);
+        }
     };
 
     return Execute;

src/common/viz/NodePrompter.js

@@ -4,9 +4,11 @@
 // adding a "plus" button for creating new objects in line
 
 define([
+    'deepforge/Constants',
     'q',
     'css!./NodePrompter.css'
 ], function(
+    Constants,
     Q
 ) {
 
@@ -271,12 +273,14 @@ define([
     };
 
     var Container = function(svg, node) {  // used for positioning
+        var colorAttr = node.attributes[Constants.DISPLAY_COLOR];
         this.$el = svg.append('g');
         this.x = 0;
         this.y = 0;
         this.node = node;
         this.decorator = new node.Decorator({
             node: node,
+            color: colorAttr && colorAttr.value,
             parentEl: this.$el
         });
     };

src/common/viz/PipelineControl.js

@@ -65,7 +65,7 @@ define([
 
     PipelineControl.prototype.createNode = function(baseId) {
         var parentId = this._currentNodeId,
-            newNodeId = this._client.createChild({parentId, baseId});
+            newNodeId = this._client.createNode({parentId, baseId});
 
         return newNodeId;
     };
@@ -103,6 +103,10 @@ define([
                 delete desc.attributes.code;
             }
 
+            // Handle the display color
+            desc.displayColor = desc.attributes[CONSTANTS.DISPLAY_COLOR].value;
+            delete desc.attributes[CONSTANTS.DISPLAY_COLOR];
+
         } else if (desc.isConnection) {
             // Set src, dst to siblings and add srcPort, dstPort
             desc.srcPort = desc.src;

src/common/viz/RenameablePanel.js

@@ -49,7 +49,7 @@ define([
 
                 if (!/^\s*$/.test(newValue)) {
                     this._client.startTransaction(msg);
-                    this._client.setAttributes(nodeId, 'name', newValue);
+                    this._client.setAttribute(nodeId, 'name', newValue);
                     this._client.completeTransaction();
                 }
             }

src/common/viz/panels/ThumbnailControl.js

@@ -0,0 +1,41 @@
+/* globals define */
+define([
+    'panels/EasyDAG/EasyDAGControl'
+], function(
+    EasyDAGControl
+) {
+    var ThumbnailControl = function() {
+        EasyDAGControl.apply(this, arguments);
+    };
+
+    ThumbnailControl.prototype = Object.create(EasyDAGControl.prototype);
+
+    ThumbnailControl.prototype._initWidgetEventHandlers = function () {
+        EasyDAGControl.prototype._initWidgetEventHandlers.call(this);
+        this._widget.updateThumbnail = this.updateThumbnail.bind(this);
+    };
+
+    ThumbnailControl.prototype.updateThumbnail = function (svg) {
+        var node = this._client.getNode(this._currentNodeId),
+            name,
+            attrs,
+            currentThumbnail,
+            attrName = 'thumbnail',
+            msg;
+
+        if (node) {  // may have been deleted
+            name = node.getAttribute('name');
+            attrs = node.getValidAttributeNames();
+            currentThumbnail = node.getAttribute(attrName);
+            msg = `Updating pipeline thumbnail for "${name}"`;
+
+            if (attrs.indexOf(attrName) > -1 && currentThumbnail !== svg) {
+                this._client.startTransaction(msg);
+                this._client.setAttribute(this._currentNodeId, attrName, svg);
+                this._client.completeTransaction();
+            }
+        }
+    };
+
+    return ThumbnailControl;
+});

src/common/viz/widgets/Thumbnail.js

@@ -0,0 +1,84 @@
+/* globals define, $, _ */
+define([
+    'widgets/EasyDAG/EasyDAGWidget'
+], function(
+    EasyDAGWidget
+) {
+
+    var ThumbnailWidget = function() {
+        EasyDAGWidget.apply(this, arguments);
+    };
+
+    ThumbnailWidget.prototype = Object.create(EasyDAGWidget.prototype);
+
+    ThumbnailWidget.prototype.addNode = function() {
+        var result = EasyDAGWidget.prototype.addNode.apply(this, arguments);
+
+        this.refreshThumbnail();
+        return result;
+    };
+
+    ThumbnailWidget.prototype.removeNode = function() {
+        var result = EasyDAGWidget.prototype.removeNode.apply(this, arguments);
+
+        this.refreshThumbnail();
+        return result;
+    };
+
+    ThumbnailWidget.prototype._removeConnection = function() {
+        var result = EasyDAGWidget.prototype._removeConnection.apply(this, arguments);
+
+        this.refreshThumbnail();
+        return result;
+    };
+
+    ThumbnailWidget.prototype.addConnection = function() {
+        var result = EasyDAGWidget.prototype.addConnection.apply(this, arguments);
+
+        this.refreshThumbnail();
+        return result;
+    };
+
+    ////////////////////////// Thumbnail updates //////////////////////////
+    ThumbnailWidget.prototype.getSvgDistanceDim = function(dim) {
+        var maxValue = this._getMaxAlongAxis(dim),
+            nodes,
+            minValue;
+
+        nodes = this.graph.nodes().map(id => this.graph.node(id));
+        minValue = nodes.length ? Math.min.apply(null, nodes.map(node => node[dim] || 0)) : 0;
+        return maxValue-minValue;
+    };
+
+    ThumbnailWidget.prototype.getSvgWidth = function() {
+        return this.getSvgDistanceDim('x');
+    };
+
+    ThumbnailWidget.prototype.getSvgHeight = function() {
+        return this.height - 25;
+    };
+
+    ThumbnailWidget.prototype.getViewBox = function() {
+        var maxX = this.getSvgWidth('x'),
+            maxY = this.getSvgHeight('y');
+
+        return `0 0 ${maxX} ${maxY}`;
+    };
+
+    ThumbnailWidget.prototype.refreshThumbnail = _.debounce(function() {
+        // Get the svg...
+        var svg = document.createElement('svg'),
+            group = this.$svg.node(),
+            child;
+
+        svg.setAttribute('viewBox', this.getViewBox());
+        for (var i = 0; i < group.children.length; i++) {
+            child = $(group.children[i]);
+            svg.appendChild(child.clone()[0]);
+        }
+
+        this.updateThumbnail(svg.outerHTML);
+    }, 1000);
+
+    return ThumbnailWidget;
+});

src/decorators/ArtifactOpDecorator/EasyDAG/ArtifactOpDecorator.EasyDAGWidget.js

@@ -40,10 +40,40 @@ define([
     ArtifactOpDecorator.prototype.DECORATOR_ID = DECORATOR_ID;
 
     ArtifactOpDecorator.prototype.getTargetFilterFnFor = function() {
+        var currentNode,
+            pId,
+            peerIds,
+            currentDataId,
+            targetTypeIds = [];
+
+        // Get all connections from this node's data
+        if (this._node.outputs[0]) {
+            currentNode = this.client.getNode(this._node.id);
+            pId = currentNode.getParentId();
+            peerIds = this.client.getNode(pId).getChildrenIds();
+            currentDataId = this._node.outputs[0].id;
+
+            targetTypeIds = peerIds.map(id => this.client.getNode(id))
+                .filter(node => {
+                    var ptr = node.getPointer('src');
+                    return ptr.to === currentDataId;
+                })
+
+                // Get the target data types
+                .map(node => this.client.getNode(node.getPointer('dst').to).getMetaTypeId());
+        }
+
         return id => {
             var node = this.client.getNode(id),
-                isMetaTgt = node.getId() === node.getMetaTypeId();
-            return isMetaTgt === this.castOpts.metaTgt;
+                isMetaTgt = node.getId() === node.getMetaTypeId(),
+                isValidType = true;
+
+            // make sure it is a type of the target types
+            isValidType = targetTypeIds.reduce((passing, typeId) => {
+                return passing && node.isTypeOf(typeId);
+            }, true);
+
+            return isValidType && isMetaTgt === this.castOpts.metaTgt;
         };
     };
 
@@ -57,13 +87,13 @@ define([
             this.client.startTransaction(`Removing output of ${this.name}`);
             this.client.delPointer(this._node.id, name);
             if (outputId) {
-                this.client.delAttributes(outputId, 'data');
+                this.client.delAttribute(outputId, 'data');
             }
             this.client.completeTransaction();
         } else if (name === this.castOpts.ptr) {  // set the casted value
             this.client.startTransaction(`Setting output of ${this.name} to ${to}`);
             this.castOutputType(to);
-            this.client.makePointer(this._node.id, name, to);
+            this.client.setPointer(this._node.id, name, to);
             this.client.completeTransaction();
         } else {
             DecoratorBase.prototype.savePointer.call(this, name, to);

src/decorators/ContainerLayerDecorator/ContainerLayerDecorator.js

@@ -0,0 +1,39 @@
+/*globals define, _*/
+/*jshint browser: true, camelcase: false*/
+
+define([
+    'js/Decorators/DecoratorBase',
+    './EasyDAG/ContainerLayerDecorator.EasyDAGWidget'
+], function (
+    DecoratorBase,
+    ContainerLayerDecoratorEasyDAGWidget
+) {
+
+    'use strict';
+
+    var ContainerLayerDecorator,
+        __parent__ = DecoratorBase,
+        __parent_proto__ = DecoratorBase.prototype,
+        DECORATOR_ID = 'ContainerLayerDecorator';
+
+    ContainerLayerDecorator = function (params) {
+        var opts = _.extend({loggerName: this.DECORATORID}, params);
+
+        __parent__.apply(this, [opts]);
+
+        this.logger.debug('ContainerLayerDecorator ctor');
+    };
+
+    _.extend(ContainerLayerDecorator.prototype, __parent_proto__);
+    ContainerLayerDecorator.prototype.DECORATORID = DECORATOR_ID;
+
+    /*********************** OVERRIDE DecoratorBase MEMBERS **************************/
+
+    ContainerLayerDecorator.prototype.initializeSupportedWidgetMap = function () {
+        this.supportedWidgetMap = {
+            EasyDAG: ContainerLayerDecoratorEasyDAGWidget
+        };
+    };
+
+    return ContainerLayerDecorator;
+});

src/decorators/ContainerLayerDecorator/EasyDAG/ContainerLayerDecorator.EasyDAGWidget.css

@@ -0,0 +1,25 @@
+.condense .nested-layers {
+  display: none;
+}
+
+.nested-layers .hover-box {
+  stroke: black;
+  stroke-dasharray: 4, 4;
+  stroke-opacity: 0;
+}
+
+.nested-layers .unhovered .button {
+  opacity: 0;
+}
+
+.nested-layers .hovered .button {
+  opacity: 1;
+}
+
+.nested-layers .unhovered .hover-box {
+  stroke-opacity: 0;
+}
+
+.nested-layers .hovered .hover-box {
+  stroke-opacity: 0;
+}

src/decorators/ContainerLayerDecorator/EasyDAG/ContainerLayerDecorator.EasyDAGWidget.js

@@ -0,0 +1,428 @@
+/*globals define, _, */
+/*jshint browser: true, camelcase: false*/
+
+define([
+    'decorators/LayerDecorator/EasyDAG/LayerDecorator.EasyDAGWidget',
+    'js/Constants',
+    'deepforge/Constants',
+    './NestedLayer',
+    'widgets/EasyDAG/Buttons',
+    'css!./ContainerLayerDecorator.EasyDAGWidget.css'
+], function (
+    LayerDecorator,
+    GME_CONSTANTS,
+    CONSTANTS,
+    NestedLayer,
+    Buttons
+) {
+
+    'use strict';
+
+    var ContainerLayerDecorator,
+        ZOOM = 0.8,
+        DECORATOR_ID = 'ContainerLayerDecorator';
+
+    // Container layer nodes need to be able to nest the containedLayers
+    // in order inside of themselves when expanded
+    ContainerLayerDecorator = function (options) {
+        this.nestedLayers = {};
+        LayerDecorator.call(this, options);
+        this.$nested = this.$el.append('g')
+            .attr('class', 'nested-layers');
+
+        // If clicked, deselect the given nested layer
+        this.$el.on('click', () => {
+            if (this.expanded) {
+                Object.keys(this.nestedLayers).forEach(id => {
+                    this.nestedLayers[id].widget.onBackgroundClick();
+                });
+            }
+        });
+        this.onNestedRefresh = _.debounce(this.updateExpand.bind(this), 50);
+
+        // Add event handlers
+        NestedLayer.prototype.addLayerBefore = function(layerId) {
+            var decorator = this._parent,
+                index = decorator._node.containedLayers.indexOf(this.id);
+            return decorator.addLayerAt(layerId, index - 1);
+        };
+
+        NestedLayer.prototype.addLayerAfter = function(layerId) {
+            var decorator = this._parent,
+                index = decorator._node.containedLayers.indexOf(this.id);
+            return decorator.addLayerAt(layerId, index + 1);
+        };
+
+        NestedLayer.prototype.isLast = function() {
+            var index = this._parent._node.containedLayers.length - 1;
+            return this._parent._node.containedLayers[index] === this.id;
+        };
+
+        NestedLayer.prototype.isFirst = function() {
+            return this._parent._node.containedLayers[0] === this.id;
+        };
+
+        NestedLayer.prototype.moveLayerForward = function() {
+            return this.moveLayer(true);
+        };
+
+        NestedLayer.prototype.moveLayerBackward = function() {
+            return this.moveLayer();
+        };
+
+        NestedLayer.prototype.moveLayer = function(forward) {
+            var decorator = this._parent,
+                index = decorator._node.containedLayers.indexOf(this.id),
+                client = decorator.client,
+                msg;
+
+            decorator._node.containedLayers.splice(index, 1);
+            if (forward) {
+                index = Math.max(0, index - 1);
+            } else {
+                index++;
+            }
+
+            decorator._node.containedLayers.splice(index, 0, this.id);
+
+            msg = `Swapping nested layers at ${index} and ${forward ? index-1 : index+1}`;
+            client.startTransaction(msg);
+            decorator._updateNestedIndices();
+            client.completeTransaction();
+        };
+
+        NestedLayer.prototype.onLastNodeRemoved = function() {
+            var decorator = this._parent,
+                index = decorator._node.containedLayers.indexOf(this.id),
+                msg = `Removing nested layer of ${decorator._node.name} at position ${index}`;
+
+            decorator.client.startTransaction(msg);
+            decorator.client.deleteNode(this.id);
+            decorator.client.completeTransaction();
+        };
+        this.updateNestedTerritory();
+    };
+
+    _.extend(ContainerLayerDecorator.prototype, LayerDecorator.prototype);
+
+    ContainerLayerDecorator.prototype.DECORATOR_ID = DECORATOR_ID;
+
+    ContainerLayerDecorator.prototype._updateNestedIndices = function() {
+        this._node.containedLayers.forEach((layerId, index) => {
+            // Set the layer's member registry to it's index
+            this.client.setMemberRegistry(
+                this._node.id,
+                layerId,
+                CONSTANTS.CONTAINED_LAYER_SET,
+                CONSTANTS.CONTAINED_LAYER_INDEX,
+                index
+            );
+        });
+    };
+
+    ContainerLayerDecorator.prototype.addLayerAt = function(baseId, index) {
+        var client = this.client,
+            parentId = this._node.id,
+            archNode,
+            newId,
+            msg;
+
+        // Get the index of the given layer
+        index = Math.max(index, 0);
+
+        archNode = client.getAllMetaNodes()
+            .find(node => node.getAttribute('name') === 'Architecture');
+
+        // Create a new Architecture node in the given node
+        msg = `Adding layer to ${this._node.name} at position ${index}`;
+        client.startTransaction(msg);
+
+        newId = client.createNode({
+            parentId: parentId,
+            baseId: archNode.getId()
+        });
+        // Create the selected layer
+        client.createNode({
+            parentId: newId,
+            baseId: baseId
+        });
+        client.addMember(parentId, newId, CONSTANTS.CONTAINED_LAYER_SET);
+        this._node.containedLayers.splice(index, 0, newId);
+        this._updateNestedIndices();
+
+        client.completeTransaction();
+    };
+
+    ContainerLayerDecorator.prototype.condense = function() {
+        // hide the nested layers
+        this.$el.attr('class', 'centering-offset condense');
+        this.removeCreateNestedBtn();
+        return LayerDecorator.prototype.condense.apply(this, arguments);
+    };
+
+    ContainerLayerDecorator.prototype.updateNestedTerritory = function() {
+        // Add the nested layers and update
+        if (!this._nestedTerritoryUI) {
+            this._nestedTerritoryUI = this.client.addUI(this, this._containedEvents.bind(this));
+        }
+        this._territory = {};
+        this._node.containedLayers.forEach(id => this._territory[id] = {children: 0});
+        this.client.updateTerritory(this._nestedTerritoryUI, this._territory);
+    };
+
+    ContainerLayerDecorator.prototype._containedEvents = function(events) {
+        for (var i = events.length; i--;) {
+            switch (events[i].etype) {
+            case GME_CONSTANTS.TERRITORY_EVENT_LOAD:
+                if (!this.nestedLayers[events[i].eid]) {
+                    this.createNestedWidget(events[i].eid);
+                }
+                break;
+
+            case GME_CONSTANTS.TERRITORY_EVENT_UNLOAD:
+                this.removeNestedWidget(events[i].eid);
+                break;
+            }
+        }
+        if (events.length > 1) {  // if more than just 'complete' event
+            this.updateExpand();
+        }
+    };
+
+    ContainerLayerDecorator.prototype.update = function(node) {
+        var attrsUpdated = false,
+            attrs = this._attributes;
+
+        this._node = node;
+        // Update the attributes
+        this.setAttributes();
+        attrsUpdated = !_.isEqual(attrs, this._attributes);
+
+        // Check for a new nested layer
+        var hasNewLayers = this._node.containedLayers
+            .filter(id => !this.nestedLayers[id])
+            .length > 0;
+
+        if (hasNewLayers) {
+            this.updateNestedTerritory();
+        } else {
+            // Update the order of the nested layers
+            if (this._selected) {
+                this.expand();
+            } else {
+                this.condense();
+            }
+        }
+        // Only reset fieldsWidth if the attribute has gotten larger
+        if (attrsUpdated) {
+            this.fieldsWidth = null;
+        }
+    };
+
+    ContainerLayerDecorator.prototype.updateExpand = function() {
+        if (this.expanded) {
+            this.expand();
+        }
+    };
+
+    ContainerLayerDecorator.prototype.createNestedWidget = function(id) {
+        if (!this.$nested) {
+            this.$nested = this.$el.append('g')
+                .attr('class', 'nested-layers');
+        }
+
+        this.nestedLayers[id] = new NestedLayer({
+            $container: this.$nested,
+            parent: this,
+            client: this.client,
+            logger: this.logger,
+            onRefresh: this.onNestedRefresh,
+            id: id
+        });
+        return this.nestedLayers[id];
+    };
+
+    ContainerLayerDecorator.prototype.removeNestedWidget = function(id) {
+        this.nestedLayers[id].destroy();
+        delete this.nestedLayers[id];
+        this.updateExpand();
+    };
+
+    ContainerLayerDecorator.prototype._renderInfo = function(top, width) {
+        var isAnUpdate = this.expanded,
+            y = top;
+
+        // Add the attribute fields
+        this.clearFields();
+        this.$attributes = this.$el.append('g')
+            .attr('fill', '#222222');
+
+        if (!isAnUpdate) {
+            this.$attributes.attr('opacity', 0);
+        }
+
+        y = this.createAttributeFields(y, width);
+        y = this.createPointerFields(y, width);
+
+        if (y !== top) {
+            y += this.ROW_HEIGHT/2;
+        }
+        return y;
+    };
+
+    ContainerLayerDecorator.prototype.expand = function() {
+        // This should be rendered with the attributes
+        var height,
+            width,
+
+            // Attributes
+            initialY = 25,
+            isAnUpdate = this.expanded,
+            NAME_MARGIN = 15,
+            nestedMargin = 15,  // minimum
+            margin = 5,
+            y = margin + initialY,
+            x = margin,
+            i;
+
+        // Shift name down
+        this.$name.attr('y', 20);
+
+        // Add the nested children
+        var ids = this._node.containedLayers.filter(id => this.nestedLayers[id]),
+            totalNestedWidth = 0,
+            maxNestedHeight = 0,
+            fieldWidth,
+            widget;
+
+        if (ids.length === 0) {
+            maxNestedHeight = CreateNestedBtn.SIZE * 2;
+        } else {
+            for (i = 0; i < ids.length; i++) {
+                widget = this.nestedLayers[ids[i]].widget;
+                totalNestedWidth += widget.getSvgWidth() * ZOOM;
+                maxNestedHeight = Math.max(widget.getSvgHeight() * ZOOM, maxNestedHeight);
+
+                // Update the buttons (in case of reorder)
+                this.nestedLayers[ids[i]].refreshButtons();
+            }
+        }
+
+        fieldWidth = this.fieldsWidth + 3 * NAME_MARGIN;
+        width = Math.max(
+            this.nameWidth + 2 * NAME_MARGIN,
+            this.size.width,
+            fieldWidth,
+            totalNestedWidth + (ids.length + 1) * nestedMargin
+        );
+
+        // Render attributes
+        y = this._renderInfo(y, fieldWidth);
+        y += nestedMargin;
+
+        // Update width, height
+        height = y + maxNestedHeight + nestedMargin;
+
+        // Equally space the nested widgets
+        nestedMargin = (width - totalNestedWidth)/(ids.length + 1);
+        x = nestedMargin - width/2;
+        for (i = 0; i < ids.length; i++) {
+            this.nestedLayers[ids[i]].$el
+                .attr('transform', `translate(${x}, ${y}) scale(${ZOOM})`);
+            x += this.nestedLayers[ids[i]].widget.getSvgWidth() * ZOOM + nestedMargin;
+        }
+
+        this.removeCreateNestedBtn();
+
+        if (ids.length === 0) {
+            // Add the 'create nested layer' button if no nested layers
+            this.$createNestedBtn = new CreateNestedBtn({
+                context: this,
+                $pEl: this.$el,
+                y: y + CreateNestedBtn.SIZE
+            });
+        }
+
+        this.$body
+            .transition()
+            .attr('x', -width/2)
+            .attr('y', 0)
+            .attr('rx', 0)
+            .attr('ry', 0)
+            .attr('width', width)
+            .attr('height', height)
+            .each('end', () => {
+                if (!isAnUpdate) {
+                    this.$attributes.attr('opacity', 1);
+                    this.$el.attr('class', 'centering-offset expand');
+                }
+            });
+
+        if (this.height !== height || this.width !== width) {
+            this.height = height;
+            this.width = width;
+            this.expanded = true;
+            this.$el
+                .attr('transform', `translate(${this.width/2}, 0)`);
+
+            this.onResize();
+        }
+    };
+
+    ContainerLayerDecorator.prototype.removeCreateNestedBtn = function() {
+        if (this.$createNestedBtn) {
+            this.$createNestedBtn.remove();
+            this.$createNestedBtn = null;
+        }
+    };
+
+    ContainerLayerDecorator.prototype.destroyNested = function() {
+        Object.keys(this.nestedLayers).forEach(id => this.nestedLayers[id].destroy());
+        this.nestedLayers = {};
+
+        if (this.$nested) {
+            this.$nested.remove();
+            this.$nested = this.$el.append('g')
+                .attr('class', 'nested-layers');
+        }
+    };
+
+    ContainerLayerDecorator.prototype.destroy = function() {
+        LayerDecorator.prototype.destroy.call(this);
+        if (this._nestedTerritoryUI) {
+            this.client.removeUI(this._nestedTerritoryUI);
+            this._nestedTerritoryUI = null;
+        }
+        this.destroyNested();
+    };
+
+    var CreateNestedBtn = function(params) {
+        params.title = 'Add nested layer';
+        Buttons.Add.call(this, params);
+    };
+
+    CreateNestedBtn.SIZE = Buttons.Add.SIZE;
+    CreateNestedBtn.prototype = Object.create(Buttons.Add.prototype);
+
+    CreateNestedBtn.prototype._onClick = function() {
+        // Call addLayerAfter and prompt for a layer
+        this.promptLayer()
+            .then(layerId => this.addLayerAt(layerId, 0));
+    };
+
+    ContainerLayerDecorator.prototype.expandAll = function() {
+        this.expand();
+        // For each of the nested layers, expand all their nodes
+        Object.keys(this.nestedLayers)
+            .forEach(id => this.nestedLayers[id].widget.expandAllNodes());
+    };
+
+    ContainerLayerDecorator.prototype.condenseAll = function() {
+        this.condense();
+        // For each of the nested layers, expand all their nodes
+        Object.keys(this.nestedLayers)
+            .forEach(id => this.nestedLayers[id].widget.expandAllNodes(true));
+    };
+
+    return ContainerLayerDecorator;
+});

src/decorators/ContainerLayerDecorator/EasyDAG/NestedLayer.js

@@ -0,0 +1,175 @@
+/*globals define, _ */
+define([
+    'panels/ArchEditor/ArchEditorControl',
+    'widgets/ArchEditor/ArchEditorWidget',
+    'widgets/EasyDAG/Buttons'
+], function(
+    ArchEditor,
+    ArchEditorWidget,
+    Buttons
+) {
+    var nop = () => {};
+    var NestedLayer = function(opts) {
+        this.$el = opts.$container.append('g')
+            .attr('class', 'nested-layer');
+
+        this.id = opts.id;
+        this._parent = opts.parent;
+        this.logger = opts.logger;
+
+        this.refreshButtons = _.debounce(this.updateButtons.bind(this), 100);
+        this.$outline = this.$el.append('rect')  // for hover detection
+            .attr('fill-opacity', 0)
+            .attr('x', 0)
+            .attr('y', 0);
+
+        this.$content = this.$el.append('g');
+        this.initHover();
+
+        this.widget = new ArchEditorWidget({
+            logger: this.logger.fork('ArchWidget'),
+            autoCenter: false,
+            svg: this.$content
+        });
+        this.widget.setTitle =
+        this.widget.updateEmptyMsg = nop;
+        this.onRefresh = opts.onRefresh;
+        this.widget.refreshExtras = this.onWidgetRefresh.bind(this);
+
+        this.control = new ArchEditor({
+            logger: this.logger.fork('ArchControl'),
+            client: opts.client,
+            embedded: true,
+            widget: this.widget
+        });
+        this.control._onUnload = () => {
+            ArchEditor.prototype._onUnload.apply(this.control, arguments);
+            // If it was the last node, remove it
+            var node = this.control._client.getNode(this.id);
+            if (node.getChildrenIds().length === 0) {
+                this.onLastNodeRemoved();
+            }
+        };
+
+        // hack :(
+        this.control.$btnModelHierarchyUp = {
+            show: nop,
+            hide: nop
+        };
+        this.widget.active = true;
+        this.control.selectedObjectChanged(this.id);
+    };
+
+    NestedLayer.prototype.initHover = function() {
+        this.$hover = this.$el.append('g')
+            .attr('class', 'hover-items');
+
+
+
+        this.$el.on('mouseenter', this.onHover.bind(this));
+        this.$el.on('mouseleave', this.onUnhover.bind(this));
+
+        // Buttons
+        this.$leftBtn = new Buttons.Add({
+            hide: true,
+            icon: this.isFirst() ? 'plus' : 'chevron-left',
+            $pEl: this.$hover
+        });
+
+        this.$rightBtn = new Buttons.Add({
+            hide: true,
+            icon: this.isLast() ? 'plus' : 'chevron-right',
+            $pEl: this.$hover
+        });
+
+        this.$deleteBtn = new Buttons.DeleteOne({
+            hide: true,
+            title: 'Delete',
+            $pEl: this.$hover
+        });
+
+        this.$leftBtn._onClick = this.clickLeft.bind(this);
+        this.$rightBtn._onClick = this.clickRight.bind(this);
+        this.$deleteBtn._onClick = () => this.onLastNodeRemoved();
+
+        this.$leftHint = this.$leftBtn.$el.append('title');
+        this.$rightHint = this.$rightBtn.$el.append('title');
+        this.refreshButtons();
+    };
+
+    NestedLayer.prototype.updateButtons = function() {
+        this.$leftBtn.icon = this.isFirst() ? 'plus' : 'chevron-left';
+        this.$rightBtn.icon = this.isLast() ? 'plus' : 'chevron-right';
+
+        this.$leftHint.text(this.isFirst() ?
+            'Add nested layer' :
+            'Move nested layer left'
+        );
+        this.$rightHint.text(this.isLast() ?
+            'Add nested layer' :
+            'Move nested layer right'
+        );
+
+        this.$leftBtn.render();
+        this.$rightBtn.render();
+    };
+
+    NestedLayer.prototype.clickLeft = function() {
+        if (this.isFirst()) {
+            this.promptLayer()
+                .then(layerId => this.addLayerBefore(layerId));
+        } else {
+            this.moveLayerForward();
+        }
+        this.onUnhover();
+    };
+
+    NestedLayer.prototype.promptLayer = function() {
+        var nodes = this.widget.getValidInitialNodes();
+
+        return this.widget.promptLayer(nodes)
+            .then(selected => selected.node.id);
+    };
+
+    NestedLayer.prototype.clickRight = function() {
+        if (this.isLast()) {
+            this.promptLayer()
+                .then(layerId => this.addLayerAfter(layerId));
+        } else {
+            this.moveLayerBackward();
+        }
+        this.onUnhover();
+    };
+
+    NestedLayer.prototype.onHover = function() {
+        this.refreshButtons();
+        this.$hover.attr('class', 'hover-items hovered');
+    };
+
+    NestedLayer.prototype.onUnhover = function() {
+        this.$hover.attr('class', 'hover-items unhovered');
+    };
+
+    NestedLayer.prototype.onWidgetRefresh = function() {
+        var width = this.widget.getSvgWidth(),
+            height = this.widget.getSvgHeight();
+
+        this.$outline
+            .attr('width', width)
+            .attr('height', height);
+
+        this.$leftBtn.$el.attr('transform', `translate(0, ${height/2})`);
+        this.$rightBtn.$el
+            .attr('transform', `translate(${width}, ${height/2})`);
+
+        this.onRefresh();
+    };
+
+    NestedLayer.prototype.destroy = function() {
+        this.control.destroy();
+        this.widget.destroy();
+        this.$el.remove();
+    };
+
+    return NestedLayer;
+});

src/decorators/DcOpDecorator/EasyDAG/DcOpDecorator.EasyDAGWidget.js

@@ -47,11 +47,11 @@ define([
             // create the outputId node
             outputId = this._createOutputNode(baseId);
         } else {
-            this.client.makePointer(outputId, CONSTANTS.POINTER_BASE, baseId);
+            this.client.setPointer(outputId, CONSTANTS.POINTER_BASE, baseId);
         }
         // Copy the data content to the output node
         hash = target.getAttribute('data');
-        this.client.setAttributes(outputId, 'data', hash);
+        this.client.setAttribute(outputId, 'data', hash);
     };
 
     DcOpDecorator.prototype._createOutputNode = function(baseId) {
@@ -69,7 +69,7 @@ define([
             return metaType.getAttribute('name') === 'Outputs';
         });
 
-        return this.client.createChild({
+        return this.client.createNode({
             baseId: baseId,
             parentId: outputCntrId
         });

src/decorators/LayerDecorator/EasyDAG/LayerDecorator.EasyDAGWidget.js

@@ -77,11 +77,11 @@ define([
         // Create a nested "architecture" node and set the ptr target to it
         baseId = base.getId();
         this.client.startTransaction(msg);
-        tgtId = this.client.createChild({
+        tgtId = this.client.createNode({
             parentId: this._node.id,
             baseId: baseId
         });
-        this.client.setAttributes(tgtId, 'name', `${ptr} (${this._node.name})`);
+        this.client.setAttribute(tgtId, 'name', `${ptr} (${this._node.name})`);
         this.savePointer(ptr, tgtId);
         this.client.completeTransaction();
         WebGMEGlobal.State.registerActiveObject(tgtId);
@@ -95,7 +95,7 @@ define([
             // If the target is contained in the current node, delete it!
             if (currentId.indexOf(this._node.id) === 0) {
                 this.client.startTransaction(`Removing layer for ${ptr} of ${name}`);
-                this.client.delMoreNodes([currentId]);
+                this.client.deleteNode(currentId);
                 this.client.completeTransaction();
                 this.logger.info(`Removed ${ptr} and deleted target (${currentId})`);
             } else {

src/decorators/OpIntDecorator/EasyDAG/OpIntDecorator.EasyDAGWidget.js

@@ -102,8 +102,8 @@ define([
             msg = `Deleting "${name}" attribute from "${opName}" operation`;
 
         this.client.startTransaction(msg);
-        this.client.removeAttributeSchema(this._node.id, name);
-        this.client.delAttributes(this._node.id, name);
+        this.client.delAttributeMeta(this._node.id, name);
+        this.client.delAttribute(this._node.id, name);
         this.client.completeTransaction();
     };
 
@@ -129,14 +129,14 @@ define([
 
         if (name !== desc.name) {  // Renaming attribute
             if (name) {
-                this.client.removeAttributeSchema(this._node.id, name);
-                this.client.delAttributes(this._node.id, name);
+                this.client.delAttributeMeta(this._node.id, name);
+                this.client.delAttribute(this._node.id, name);
             }
             name = desc.name;
         }
 
-        this.client.setAttributeSchema(this._node.id, name, schema);
-        this.client.setAttributes(this._node.id, name, desc.defaultValue);
+        this.client.setAttributeMeta(this._node.id, name, schema);
+        this.client.setAttribute(this._node.id, name, desc.defaultValue);
         this.client.completeTransaction();
     };
 

src/decorators/OperationDecorator/EasyDAG/OperationDecorator.EasyDAGWidget.js

@@ -2,9 +2,11 @@
 /*jshint browser: true, camelcase: false*/
 
 define([
+    'deepforge/Constants',
     'decorators/EllipseDecorator/EasyDAG/EllipseDecorator.EasyDAGWidget',
     'css!./OperationDecorator.EasyDAGWidget.css'
 ], function (
+    CONSTANTS,
     DecoratorBase
 ) {
 
@@ -13,6 +15,7 @@ define([
     var OperationDecorator,
         NAME_MARGIN = 25,
         DECORATOR_ID = 'OperationDecorator',
+        OPERATION_COLORS = {},
         PORT_TOOLTIP_OPTS = {
             tipJoint: 'left',
             removeElementsOnHide: true,
@@ -24,8 +27,10 @@ define([
     //     - highlight ports
     //     - unhighlight ports
     //     - report the location of specific ports
+    OPERATION_COLORS[CONSTANTS.OP.OUTPUT] = '#b0bec5';
+    OPERATION_COLORS[CONSTANTS.OP.INPUT] = '#b0bec5';
     OperationDecorator = function (options) {
-        options.color = options.color || '#78909c';
+        options.color = OPERATION_COLORS[options.node.name] || options.color || '#78909c';
         DecoratorBase.call(this, options);
 
         this.id = this._node.id;
@@ -73,14 +78,15 @@ define([
 
     OperationDecorator.prototype.showPorts = function(ids, areInputs) {
         var allPorts = areInputs ? this._node.inputs : this._node.outputs,
-            ports = ids ? allPorts.filter(port => ids.indexOf(port.id) > -1) : allPorts,
             x = -this.width/2,
-            dx = this.width/(ports.length+1),
+            dx = this.width/(allPorts.length+1),
             y = areInputs ? 0 : this.height;  // (this.height/2);
 
-        ports.forEach(port => {
+        allPorts.forEach(port => {
             x += dx;
-            this.renderPort(port, x, y, areInputs);
+            if (!ids || ids.indexOf(port.id) > -1) {
+                this.renderPort(port, x, y, areInputs);
+            }
         });
     };
 
@@ -117,7 +123,7 @@ define([
         tooltip = new Opentip(portIcon[0][0], PORT_TOOLTIP_OPTS);
         tooltip.setContent(port.name);
         portIcon.on('mouseenter', () => tooltip.show());
-        portIcon.on('mouseout', () => tooltip.hide());
+        portIcon.on('mouseleave', () => tooltip.hide());
         this.$portTooltips[port.id] = tooltip;
     };
 

src/layouts/SidebarLayout/SidebarLayout.css

@@ -0,0 +1,12 @@
+.ui-layout-center .layout-center {
+    left: 40px;
+    height: 100%;
+    width: 100%;
+    position: absolute;
+}
+
+.ui-layout-sidebar {
+    top: 64px;
+    width: 40px;
+    bottom: 27px;
+}

src/layouts/SidebarLayout/SidebarLayout.js

@@ -0,0 +1,57 @@
+/*globals define, */
+define([
+    'layout/CHFLayout/CHFLayout/CHFLayout',
+    'text!./templates/SidebarLayout.html',
+    'css!./SidebarLayout.css'
+], function(
+    CHFLayout,
+    SidebarTemplate
+) {
+    'use strict';
+    
+    var SidebarLayout = function(params) {
+        params = params || {};
+        params.template = SidebarTemplate;
+        CHFLayout.call(this, params);
+    };
+
+    SidebarLayout.prototype = Object.create(CHFLayout.prototype);
+
+    SidebarLayout.prototype.getComponentId = function () {
+        return 'SidebarLayout';
+    };
+
+    /**
+     * Initialize the html page. This example is using the jQuery Layout plugin.
+     *
+     * @return {undefined}
+     */
+    SidebarLayout.prototype.init = function() {
+        CHFLayout.prototype.init.apply(this, arguments);
+        this._sidebarPanel = this._body.find('div.ui-layout-sidebar');
+        this._centerPanel = this._body.find('div.layout-center');
+    };
+
+    /**
+     * Add a panel to a given container. This is defined in the corresponding
+     * layout config JSON file.
+     *
+     * @param {Panel} panel
+     * @param {String} container
+     * @return {undefined}
+     */
+    SidebarLayout.prototype.addToContainer = function(panel, container) {
+        if (container === 'sidebar') {
+            this._sidebarPanel.append(panel.$pEl);
+        } else {
+            CHFLayout.prototype.addToContainer.apply(this, arguments);
+        }
+    };
+
+    SidebarLayout.prototype._onCenterResize = function() {
+        var width = this._centerPanel.width() - this._sidebarPanel.width();
+        this._canvas.setSize(width, this._centerPanel.height());
+    };
+
+    return SidebarLayout;
+});

src/layouts/SidebarLayout/templates/SidebarLayout.html

@@ -0,0 +1,7 @@
+<div class="ui-layout-center" style="position: relative;">
+    <div class="layout-center"></div>
+    <div class="float"></div>
+</div>
+<div class="ui-layout-sidebar"></div>
+<div class="ui-layout-north"></div>
+<div class="ui-layout-south"></div>

src/plugins/CheckLibraries/CheckLibraries.js

@@ -92,7 +92,7 @@ define([
 
                     this.logger.info(`${tuple[0]} version info:\n${projVersion} ` +
                         `(project)\n${latest} (latest)`);
-                    return latest !== projVersion;
+                    return projVersion < latest;
                 });
 
                 return Q.all(tuples.map(tuple => this.uploadSeed.apply(this, tuple)));

src/plugins/CreateTorchMeta/schemas/nn.json

@@ -65,7 +65,14 @@
       "affine"
     ],
     "setters": {},
-    "defaults": {},
+    "types": {
+      "eps": "number",
+      "momentum": "number"
+    },
+    "defaults": {
+      "momentum": 0.1,
+      "eps": 0.00001
+    },
     "type": "Misc"
   },
   {
@@ -82,6 +89,35 @@
     "defaults": {},
     "type": "Misc"
   },
+  {
+    "name": "Bottle",
+    "baseType": "Container",
+    "params": [
+      "module",
+      "nInputDim",
+      "nOutputDim"
+    ],
+    "setters": {},
+    "types": {
+      "nInputDim": "number",
+      "module": "nn.Module"
+    },
+    "defaults": {
+      "nInputDim": 2
+    },
+    "type": "Container"
+  },
+  {
+    "name": "CAdd",
+    "baseType": "Module",
+    "params": [
+      "params"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Misc"
+  },
   {
     "name": "CAddTable",
     "baseType": "Module",
@@ -102,6 +138,24 @@
     "defaults": {},
     "type": "Misc"
   },
+  {
+    "name": "CMaxTable",
+    "baseType": "Module",
+    "params": [],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Misc"
+  },
+  {
+    "name": "CMinTable",
+    "baseType": "Module",
+    "params": [],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Misc"
+  },
   {
     "name": "CMul",
     "baseType": "Module",
@@ -175,7 +229,7 @@
     "setters": {},
     "types": {},
     "defaults": {},
-    "type": "Containers"
+    "type": "Simple"
   },
   {
     "name": "ConcatTable",
@@ -184,7 +238,7 @@
     "setters": {},
     "types": {},
     "defaults": {},
-    "type": "Misc"
+    "type": "Container"
   },
   {
     "name": "Contiguous",
@@ -296,13 +350,16 @@
     "params": [
       "p",
       "v1",
-      "inplace"
+      "inplace",
+      "stochasticInference"
     ],
     "setters": {},
     "types": {
-      "p": "number"
+      "p": "number",
+      "stochasticInference": "boolean"
     },
     "defaults": {
+      "stochasticInference": false,
       "p": 0.5
     },
     "type": "Simple"
@@ -357,8 +414,11 @@
   {
     "name": "GradientReversal",
     "baseType": "Module",
-    "params": [],
+    "params": [
+      "lambda"
+    ],
     "setters": {},
+    "types": {},
     "defaults": {},
     "type": "Misc"
   },
@@ -382,7 +442,8 @@
     "baseType": "Module",
     "params": [
       "min_value",
-      "max_value"
+      "max_value",
+      "inplace"
     ],
     "setters": {},
     "types": {
@@ -508,9 +569,7 @@
   {
     "name": "Log",
     "baseType": "Module",
-    "params": [
-      "inputSize"
-    ],
+    "params": [],
     "setters": {},
     "types": {},
     "defaults": {},
@@ -865,7 +924,7 @@
     "setters": {},
     "types": {},
     "defaults": {},
-    "type": "Misc"
+    "type": "Container"
   },
   {
     "name": "ParallelCriterion",
@@ -885,7 +944,7 @@
     "setters": {},
     "types": {},
     "defaults": {},
-    "type": "Misc"
+    "type": "Container"
   },
   {
     "name": "PartialLinear",
@@ -900,6 +959,17 @@
     "defaults": {},
     "type": "Misc"
   },
+  {
+    "name": "PixelShuffle",
+    "baseType": "Module",
+    "params": [
+      "upscaleFactor"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Misc"
+  },
   {
     "name": "Power",
     "baseType": "Module",
@@ -939,6 +1009,17 @@
     "defaults": {},
     "type": "Transfer"
   },
+  {
+    "name": "ReLU6",
+    "baseType": "Module",
+    "params": [
+      "inplace"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Transfer"
+  },
   {
     "name": "Replicate",
     "baseType": "Module",
@@ -1156,6 +1237,18 @@
     "defaults": {},
     "type": "Convolution"
   },
+  {
+    "name": "SpatialClassNLLCriterion",
+    "baseType": "Criterion",
+    "params": [
+      "weights",
+      "sizeAverage"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Criterion"
+  },
   {
     "name": "SpatialContrastiveNormalization",
     "baseType": "Module",
@@ -1279,6 +1372,56 @@
     },
     "type": "Convolution"
   },
+  {
+    "name": "SpatialDilatedConvolution",
+    "baseType": "SpatialConvolution",
+    "params": [
+      "nInputPlane",
+      "nOutputPlane",
+      "kW",
+      "kH",
+      "dW",
+      "dH",
+      "padW",
+      "padH",
+      "dilationW",
+      "dilationH"
+    ],
+    "setters": {},
+    "types": {
+      "dilationW": "number",
+      "dilationH": "number"
+    },
+    "defaults": {
+      "dilationH": 1,
+      "dilationW": 1
+    },
+    "type": "Convolution"
+  },
+  {
+    "name": "SpatialDilatedMaxPooling",
+    "baseType": "SpatialMaxPooling",
+    "params": [
+      "kW",
+      "kH",
+      "dW",
+      "dH",
+      "padW",
+      "padH",
+      "dilationW",
+      "dilationH"
+    ],
+    "setters": {},
+    "types": {
+      "dilationW": "number",
+      "dilationH": "number"
+    },
+    "defaults": {
+      "dilationH": 1,
+      "dilationW": 1
+    },
+    "type": "Misc"
+  },
   {
     "name": "SpatialDivisiveNormalization",
     "baseType": "Module",
@@ -1301,13 +1444,16 @@
     "name": "SpatialDropout",
     "baseType": "Module",
     "params": [
-      "p"
+      "p",
+      "stochasticInference"
     ],
     "setters": {},
     "types": {
-      "p": "number"
+      "p": "number",
+      "stochasticInference": "boolean"
     },
     "defaults": {
+      "stochasticInference": false,
       "p": 0.5
     },
     "type": "Convolution"
@@ -1387,6 +1533,14 @@
     "defaults": {},
     "type": "Convolution"
   },
+  {
+    "name": "SpatialLogSoftMax",
+    "baseType": "Module",
+    "params": [],
+    "setters": {},
+    "defaults": {},
+    "type": "Misc"
+  },
   {
     "name": "SpatialMaxPooling",
     "baseType": "Module",
@@ -1497,6 +1651,17 @@
     },
     "type": "Convolution"
   },
+  {
+    "name": "SpatialUpSamplingBilinear",
+    "baseType": "Module",
+    "params": [
+      "params"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Misc"
+  },
   {
     "name": "SpatialUpSamplingNearest",
     "baseType": "Module",
@@ -1627,6 +1792,22 @@
     "defaults": {},
     "type": "Convolution"
   },
+  {
+    "name": "TemporalDynamicKMaxPooling",
+    "baseType": "Module",
+    "params": [
+      "minK",
+      "factor"
+    ],
+    "setters": {},
+    "types": {
+      "factor": "number"
+    },
+    "defaults": {
+      "factor": 0
+    },
+    "type": "Misc"
+  },
   {
     "name": "TemporalMaxPooling",
     "baseType": "Module",
@@ -1764,16 +1945,81 @@
     "type": "Convolution"
   },
   {
-    "name": "VolumetricDropout",
-    "baseType": "Module",
+    "name": "VolumetricDilatedConvolution",
+    "baseType": "VolumetricConvolution",
     "params": [
-      "p"
+      "nInputPlane",
+      "nOutputPlane",
+      "kT",
+      "kW",
+      "kH",
+      "dT",
+      "dW",
+      "dH",
+      "padT",
+      "padW",
+      "padH",
+      "dilationT",
+      "dilationW",
+      "dilationH"
     ],
     "setters": {},
     "types": {
-      "p": "number"
+      "dilationT": "number",
+      "dilationW": "number",
+      "dilationH": "number"
     },
     "defaults": {
+      "dilationH": 1,
+      "dilationW": 1,
+      "dilationT": 1
+    },
+    "type": "Misc"
+  },
+  {
+    "name": "VolumetricDilatedMaxPooling",
+    "baseType": "VolumetricMaxPooling",
+    "params": [
+      "kT",
+      "kW",
+      "kH",
+      "dT",
+      "dW",
+      "dH",
+      "padT",
+      "padW",
+      "padH",
+      "dilationT",
+      "dilationW",
+      "dilationH"
+    ],
+    "setters": {},
+    "types": {
+      "dilationT": "number",
+      "dilationW": "number",
+      "dilationH": "number"
+    },
+    "defaults": {
+      "dilationH": 1,
+      "dilationW": 1,
+      "dilationT": 1
+    },
+    "type": "Misc"
+  },
+  {
+    "name": "VolumetricDropout",
+    "baseType": "Module",
+    "params": [
+      "p",
+      "stochasticInference"
+    ],
+    "setters": {},
+    "types": {
+      "p": "number",
+      "stochasticInference": "boolean"
+    },
+    "defaults": {
+      "stochasticInference": false,
       "p": 0.5
     },
     "type": "Convolution"
@@ -1864,6 +2110,22 @@
     "defaults": {},
     "type": "Convolution"
   },
+  {
+    "name": "VolumetricReplicationPadding",
+    "baseType": "Module",
+    "params": [
+      "pleft",
+      "pright",
+      "ptop",
+      "pbottom",
+      "pfront",
+      "pback"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Misc"
+  },
   {
     "name": "WeightedEuclidean",
     "baseType": "Module",
@@ -1887,4 +2149,4 @@
     "defaults": {},
     "type": "Criterion"
   }
-]
+]

src/plugins/ExecuteJob/ExecuteJob.Files.js

@@ -0,0 +1,346 @@
+/*globals define*/
+define([
+    './templates/index',
+    'q',
+    'underscore',
+    'deepforge/Constants'
+], function(
+    Templates,
+    Q,
+    _,
+    CONSTANTS
+) {
+
+    var SKIP_ATTRIBUTES = [
+        'code',
+        'stdout',
+        'execFiles',
+        'jobId',
+        'secret',
+        CONSTANTS.LINE_OFFSET,
+        CONSTANTS.DISPLAY_COLOR
+    ];
+    var ExecuteJob = function() {
+    };
+
+    ExecuteJob.prototype.createOperationFiles = function (node) {
+        var files = {};
+        // For each operation, generate the output files:
+        //   inputs/<arg-name>/init.lua  (respective data deserializer)
+        //   pointers/<name>/init.lua  (result of running the main plugin on pointer target - may need a rename)
+        //   outputs/<name>/  (make dirs for each of the outputs)
+        //   outputs/init.lua  (serializers for data outputs)
+        //
+        //   attributes.lua (returns lua table of operation attributes)
+        //   init.lua (main file -> calls main and serializes outputs)
+        //   <name>.lua (entry point -> calls main operation code)
+
+        // add the given files
+        this.logger.info('About to create dist execution files');
+        files['start.js'] = _.template(Templates.START)(CONSTANTS);
+        return this.createEntryFile(node, files)
+            .then(() => this.createClasses(node, files))
+            .then(() => this.createCustomLayers(node, files))
+            .then(() => this.createInputs(node, files))
+            .then(() => this.createOutputs(node, files))
+            .then(() => this.createMainFile(node, files))
+            .then(() => {
+                this.createAttributeFile(node, files);
+                return Q.ninvoke(this, 'createPointers', node, files);
+            });
+    };
+
+    ExecuteJob.prototype.createEntryFile = function (node, files) {
+        this.logger.info('Creating entry files...');
+        return this.getOutputs(node)
+            .then(outputs => {
+                var name = this.getAttribute(node, 'name'),
+                    content = {};
+
+                // inputs and outputs
+                content.name = name;
+                content.outputs = outputs;
+
+                files['init.lua'] = _.template(Templates.ENTRY)(content);
+
+                // Create the deepforge file
+                files['deepforge.lua'] = _.template(Templates.DEEPFORGE)(CONSTANTS);
+            });
+    };
+
+    ExecuteJob.prototype.createClasses = function (node, files) {
+        var metaDict = this.core.getAllMetaNodes(this.rootNode),
+            isClass,
+            metanodes,
+            classNodes,
+            inheritanceLvl = {},
+            code;
+
+        this.logger.info('Creating custom layer file...');
+        metanodes = Object.keys(metaDict).map(id => metaDict[id]);
+        isClass = this.getTypeDictFor('Complex', metanodes);
+
+        classNodes = metanodes.filter(node => {
+            var base = this.core.getBase(node),
+                baseId,
+                count = 1;
+
+            // Count the sets back to a class node
+            while (base) {
+                baseId = this.core.getPath(base);
+                if (isClass[baseId]) {
+                    inheritanceLvl[this.core.getPath(node)] = count;
+                    return true;
+                }
+                base = this.core.getBase(base);
+                count++;
+            }
+
+            return false;
+        });
+
+        // Get the code definitions for each
+        // Sort by levels of inheritance...
+        code = classNodes.sort((a, b) => {
+            var aId = this.core.getPath(a),
+                bId = this.core.getPath(b);
+
+            return inheritanceLvl[aId] > inheritanceLvl[bId];
+        }).map(node =>
+            `require './${this.getAttribute(node, 'name')}.lua'`
+        ).join('\n');
+
+        // Create the class files
+        classNodes.forEach(node => {
+            var name = this.getAttribute(node, 'name');
+            files[`classes/${name}.lua`] = this.getAttribute(node, 'code');
+        });
+
+        // Create the custom layers file
+        files['classes/init.lua'] = code;
+    };
+
+    ExecuteJob.prototype.getTypeDictFor = function (name, metanodes) {
+        var isType = {};
+        // Get all the custom layers
+        for (var i = metanodes.length; i--;) {
+            if (this.getAttribute(metanodes[i], 'name') === name) {
+                isType[this.core.getPath(metanodes[i])] = true;
+            }
+        }
+        return isType;
+    };
+
+    ExecuteJob.prototype.createCustomLayers = function (node, files) {
+        var metaDict = this.core.getAllMetaNodes(this.rootNode),
+            isCustomLayer,
+            metanodes,
+            customLayers,
+            code;
+
+        this.logger.info('Creating custom layer file...');
+        metanodes = Object.keys(metaDict).map(id => metaDict[id]);
+        isCustomLayer = this.getTypeDictFor('CustomLayer', metanodes);
+
+        customLayers = metanodes.filter(node =>
+            this.core.getMixinPaths(node).some(id => isCustomLayer[id]));
+
+        // Get the code definitions for each
+        code = 'require \'nn\'\n\n' + customLayers
+            .map(node => this.getAttribute(node, 'code')).join('\n');
+
+        // Create the custom layers file
+        files['custom-layers.lua'] = code;
+    };
+
+    ExecuteJob.prototype.createInputs = function (node, files) {
+        var tplContents,
+            inputs;
+
+        this.logger.info('Retrieving inputs and deserialize fns...');
+        return this.getInputs(node)
+            .then(allInputs => {
+                // For each input, match the connection with the input name
+                //   [ name, type ] => [ name, type, node ]
+                //
+                // For each input,
+                //  - create the deserializer
+                //  - put it in inputs/<name>/init.lua
+                //  - copy the data asset to /inputs/<name>/init.lua
+                inputs = allInputs
+                    .filter(pair => !!this.getAttribute(pair[2], 'data'));  // remove empty inputs
+
+                files.inputAssets = {};  // data assets
+                return Q.all(inputs.map(pair => {
+                    var name = pair[0],
+                        node = pair[2],
+                        nodeId = this.core.getPath(node),
+                        fromNodeId;
+
+                    // Get the deserialize function. First, try to get it from
+                    // the source method (this guarantees that the correct
+                    // deserialize method is used despite any auto-upcasting
+                    fromNodeId = this.inputPortsFor[nodeId][0] || nodeId;
+
+                    return this.core.loadByPath(this.rootNode, fromNodeId)
+                        .then(fromNode => {
+                            var deserFn,
+                                base,
+                                className;
+
+                            deserFn = this.getAttribute(fromNode, 'deserialize');
+
+                            if (this.isMetaTypeOf(node, this.META.Complex)) {
+                                // Complex objects are expected to define their own
+                                // (static) deserialize factory method
+                                base = this.core.getMetaType(node);
+                                className = this.getAttribute(base, 'name');
+                                deserFn = `return ${className}.deserialize(path)`;
+                            }
+
+                            return {
+                                name: name,
+                                code: deserFn
+                            };
+                        });
+                }));
+            })
+            .then(_tplContents => {
+                tplContents = _tplContents;
+                var hashes = inputs.map(pair => {
+                    var hash = this.getAttribute(pair[2], 'data');
+                    files.inputAssets[pair[0]] = hash;
+                    return {
+                        hash: hash,
+                        name: pair[0]
+                    };
+                });
+
+                return Q.all(hashes.map(pair => 
+                    this.blobClient.getMetadata(pair.hash)
+                        .fail(err => this.onBlobRetrievalFail(node, pair.name, err))));
+            })
+            .then(metadatas => {
+                // Create the deserializer
+                tplContents.forEach((ctnt, i) => {
+                    // Get the name of the given asset
+                    ctnt.filename = metadatas[i].name;
+                    files['inputs/' + ctnt.name + '/init.lua'] = _.template(Templates.DESERIALIZE)(ctnt);
+                });
+                return files;
+            });
+    };
+
+    ExecuteJob.prototype.createOutputs = function (node, files) {
+        // For each of the output types, grab their serialization functions and
+        // create the `outputs/init.lua` file
+        this.logger.info('Creating outputs/init.lua...');
+        return this.getOutputs(node)
+            .then(outputs => {
+                var outputTypes = outputs
+                // Get the serialize functions for each
+                    .map(tuple => {
+                        var node = tuple[2],
+                            serFn = this.getAttribute(node, 'serialize');
+
+                        if (this.isMetaTypeOf(node, this.META.Complex)) {
+                            // Complex objects are expected to define their own
+                            // serialize methods
+                            serFn = 'if data ~= nil then data:serialize(path) end';
+                        }
+
+                        return [tuple[1], serFn];
+                    });
+
+                files['outputs/init.lua'] = _.template(Templates.SERIALIZE)({types: outputTypes});
+            });
+    };
+
+    ExecuteJob.prototype.createMainFile = function (node, files) {
+        this.logger.info('Creating main file...');
+        return this.getInputs(node)
+            .then(inputs => {
+                var name = this.getAttribute(node, 'name'),
+                    code = this.getAttribute(node, 'code'),
+                    pointers = this.core.getPointerNames(node).filter(ptr => ptr !== 'base'),
+                    content = {
+                        name: name
+                    };
+
+                // Get input data arguments
+                content.inputs = inputs
+                    .map(pair => [pair[0], !this.getAttribute(pair[2], 'data')]);  // remove empty inputs
+
+                // Defined variables for each pointers
+                content.pointers = pointers
+                    .map(id => [id, this.core.getPointerPath(node, id) === null]);
+
+                // Add remaining code
+                content.code = code;
+
+                files['main.lua'] = _.template(Templates.MAIN)(content);
+
+                // Set the line offset
+                var lineOffset = this.getLineOffset(files['main.lua'], code);
+                this.setAttribute(node, CONSTANTS.LINE_OFFSET, lineOffset);
+            });
+    };
+
+    ExecuteJob.prototype.getLineOffset = function (main, snippet) {
+        var i = main.indexOf(snippet),
+            lines = main.substring(0, i).match(/\n/g);
+
+        return lines ? lines.length : 0;
+    };
+
+    ExecuteJob.prototype.createAttributeFile = function (node, files) {
+        var numOrBool = /^(-?\d+\.?\d*((e|e-)\d+)?|(true|false))$/,
+            table;
+
+        this.logger.info('Creating attributes file...');
+        table = '{\n\t' + this.core.getAttributeNames(node)
+            .filter(attr => SKIP_ATTRIBUTES.indexOf(attr) === -1)
+            .map(name => {
+                var value = this.getAttribute(node, name);
+                if (!numOrBool.test(value)) {
+                    value = `"${value}"`;
+                }
+                return [name, value];
+            })
+            .map(pair => pair.join(' = '))
+            .join(',\n\t') + '\n}';
+
+        files['attributes.lua'] = `-- attributes of ${this.getAttribute(node, 'name')}\nreturn ${table}`;
+    };
+
+    ExecuteJob.prototype.createPointers = function (node, files, cb) {
+        var pointers,
+            nIds;
+
+        this.logger.info('Creating pointers file...');
+        pointers = this.core.getPointerNames(node)
+            .filter(name => name !== 'base')
+            .filter(id => this.core.getPointerPath(node, id) !== null);
+
+        nIds = pointers.map(p => this.core.getPointerPath(node, p));
+        files.ptrAssets = {};
+        Q.all(
+            nIds.map(nId => this.getPtrCodeHash(nId))
+        )
+        .then(resultHashes => {
+            var name = this.getAttribute(node, 'name');
+            this.logger.info(`Pointer generation for ${name} FINISHED!`);
+            resultHashes.forEach((hash, index) => {
+                files.ptrAssets[`pointers/${pointers[index]}/init.lua`] = hash;
+            });
+            return cb(null, files);
+        })
+        .fail(e => {
+            this.logger.error(`Could not generate pointer files for ${this.getAttribute(node, 'name')}: ${e.toString()}`);
+            return cb(e);
+        });
+    };
+
+    return ExecuteJob;
+});
+

src/plugins/ExecuteJob/ExecuteJob.Metadata.js

@@ -0,0 +1,146 @@
+/*globals define*/
+define([
+    'deepforge/Constants'
+], function(
+    CONSTANTS
+) {
+
+    var ExecuteJob = function() {
+    };
+
+    ExecuteJob.prototype[CONSTANTS.GRAPH_CREATE] = function (job, id) {
+        var graph,
+            name = Array.prototype.slice.call(arguments, 2).join(' '),
+            jobId = this.core.getPath(job);
+
+        id = jobId + '/' + id;
+        this.logger.info(`Creating graph ${id} named ${name}`);
+
+        // Check if the graph already exists
+        graph = this._getExistingMetadata(jobId, 'Graph', name);
+        if (!graph) {
+            graph = this.createNode('Graph', job);
+
+            if (name) {
+                this.setAttribute(graph, 'name', name);
+            }
+            this.createIdToMetadataId[graph] = id;
+        }
+
+        this._metadata[id] = graph;
+    };
+
+    ExecuteJob.prototype[CONSTANTS.GRAPH_LABEL_AXIS.X] = function (job, id) {
+        var name = Array.prototype.slice.call(arguments, 2).join(' '),
+            jobId = this.core.getPath(job),
+            graph;
+
+        id = jobId + '/' + id;
+        this.logger.info(`Labeling the x-axis of ${id}: ${name}`);
+
+        graph = this._metadata[id];
+        this.setAttribute(graph, 'xlabel', name);
+    };
+
+    ExecuteJob.prototype[CONSTANTS.GRAPH_LABEL_AXIS.Y] = function (job, id) {
+        var name = Array.prototype.slice.call(arguments, 2).join(' '),
+            jobId = this.core.getPath(job),
+            graph;
+
+        id = jobId + '/' + id;
+        this.logger.info(`Labeling the y-axis of ${id}: ${name}`);
+
+        graph = this._metadata[id];
+        this.setAttribute(graph, 'ylabel', name);
+    };
+
+    ExecuteJob.prototype[CONSTANTS.GRAPH_PLOT] = function (job, id, x, y) {
+        var jobId = this.core.getPath(job),
+            nonNum = /[^\d-\.]*/g,
+            line,
+            points;
+            
+
+        id = jobId + '/' + id;
+        this.logger.info(`Adding point ${x}, ${y} to ${id}`);
+        line = this._metadata[id];
+        if (!line) {
+            this.logger.warn(`Can't add point to non-existent line: ${id}`);
+            return;
+        }
+
+        // Clean the points by removing and special characters
+        x = x.replace(nonNum, '');
+        y = y.replace(nonNum, '');
+        points = this.getAttribute(line, 'points');
+        points += `${x},${y};`;
+        this.setAttribute(line, 'points', points);
+    };
+
+    ExecuteJob.prototype[CONSTANTS.GRAPH_CREATE_LINE] = function (job, graphId, id) {
+        var jobId = this.core.getPath(job),
+            graph = this._metadata[jobId + '/' + graphId],
+            name = Array.prototype.slice.call(arguments, 3).join(' '),
+            line;
+
+        // Create a 'line' node in the given Graph metadata node
+        name = name.replace(/\s+$/, '');
+        line = this.createNode('Line', graph);
+        this.setAttribute(line, 'name', name);
+        this._metadata[jobId + '/' + id] = line;
+        this.createIdToMetadataId[line] = jobId + '/' + id;
+    };
+
+    ExecuteJob.prototype[CONSTANTS.IMAGE.BASIC] =
+    ExecuteJob.prototype[CONSTANTS.IMAGE.UPDATE] =
+    ExecuteJob.prototype[CONSTANTS.IMAGE.CREATE] = function (job, hash, imgId) {
+        var name = Array.prototype.slice.call(arguments, 3).join(' '),
+            imageNode = this._getImageNode(job, imgId, name);
+
+        this.setAttribute(imageNode, 'data', hash);
+    };
+
+    ExecuteJob.prototype[CONSTANTS.IMAGE.NAME] = function (job, imgId) {
+        var name = Array.prototype.slice.call(arguments, 2).join(' '),
+            imageNode = this._getImageNode(job, imgId, name);
+
+        this.setAttribute(imageNode, 'name', name);
+    };
+
+    ExecuteJob.prototype._getImageNode = function (job, imgId, name) {
+        var jobId = this.core.getPath(job),
+            id = jobId + '/IMAGE/' + imgId,
+            imageNode = this._metadata[id];  // Look for the metadata imageNode
+
+        if (!imageNode) {
+
+            // Check if the imageNode already exists
+            imageNode = this._getExistingMetadata(jobId, 'Image', name);
+            if (!imageNode) {
+                this.logger.info(`Creating image ${id} named ${name}`);
+                imageNode = this.createNode('Image', job);
+                this.setAttribute(imageNode, 'name', name);
+                this.createIdToMetadataId[imageNode] = id;
+            }
+            this._metadata[id] = imageNode;
+        }
+        return imageNode;
+    };
+
+    ExecuteJob.prototype._getExistingMetadata = function (jobId, type, name) {
+        var oldMetadata = this._oldMetadataByName[jobId] && this._oldMetadataByName[jobId][type],
+            node,
+            id;
+
+        if (oldMetadata && oldMetadata[name]) {
+            id = oldMetadata[name];
+            node = this._markForDeletion[jobId][id];
+            delete this._markForDeletion[jobId][id];
+            this.createdMetadataIds[jobId].push(id);  // used for resuming jobs
+        }
+
+        return node || null;
+    };
+
+    return ExecuteJob;
+});

src/plugins/ExecuteJob/ExecuteJob.SafeSave.js

@@ -0,0 +1,377 @@
+/*globals define*/
+define([
+    'plugin/PluginBase',
+    'common/storage/constants',
+    'q',
+    'common/util/assert'
+], function(
+    PluginBase,
+    STORAGE_CONSTANTS,
+    Q,
+    assert
+) {
+
+    var CREATE_PREFIX = 'created_node_',
+        INDEX = 1;
+
+    var ExecuteJob = function() {
+        this.forkNameBase = null;
+        this.runningJobHashes = [];
+        this._currentSave = Q();
+    };
+
+    ExecuteJob.prototype.getCreateId = function () {
+        return CREATE_PREFIX + (++INDEX);
+    };
+
+    ExecuteJob.prototype.isCreateId = function (id) {
+        return (typeof id === 'string') && (id.indexOf(CREATE_PREFIX) === 0);
+    };
+
+    ExecuteJob.prototype.createNode = function (baseType, parent) {
+        var id = this.getCreateId(),
+            parentId = this.isCreateId(parent) ? parent : this.core.getPath(parent);
+
+        this.logger.info(`Creating ${id} of type ${baseType} in ${parentId}`);
+        assert(this.META[baseType], `Cannot create node w/ unrecognized type: ${baseType}`);
+        this.creations[id] = {
+            base: baseType,
+            parent: parentId
+        };
+        return id;
+    };
+
+    ExecuteJob.prototype.deleteNode = function (nodeId) {
+        this.deletions.push(nodeId);
+    };
+
+    ExecuteJob.prototype.delAttribute = function (node, attr) {
+        return this.setAttribute(node, attr, null);
+    };
+
+    ExecuteJob.prototype.setAttribute = function (node, attr, value) {
+        var nodeId;
+
+        if (this.isCreateId(node)) {
+            nodeId = node;
+        } else {
+            nodeId = this.core.getPath(node);
+            assert(typeof nodeId === 'string', `Cannot set attribute of ${nodeId}`);
+        }
+
+        if (value !== null) {
+            this.logger.info(`Setting ${attr} of ${nodeId} to ${value}`);
+        } else {
+            this.logger.info(`Deleting ${attr} of ${nodeId}`);
+        }
+
+        if (!this.changes[nodeId]) {
+            this.changes[nodeId] = {};
+        }
+        this.changes[nodeId][attr] = value;
+    };
+
+    ExecuteJob.prototype.getAttribute = function (node, attr) {
+        var nodeId;
+
+        assert(this.deletions.indexOf(nodeId) === -1,
+            `Cannot get ${attr} from deleted node ${nodeId}`);
+
+        // Check if it was newly created
+        if (this.isCreateId(node)) {
+            nodeId = node;
+            assert(this.creations[nodeId], `Creation node not updated: ${nodeId}`);
+            node = this.META[this.creations[nodeId].base];
+        } else {
+            nodeId = this.core.getPath(node);
+        }
+
+        // Check the most recent changes, then the currentChanges, then the model
+        var value = this._getValueFrom(nodeId, attr, node, this.changes) ||
+            this._getValueFrom(nodeId, attr, node, this.currentChanges);
+
+        if (value) {
+            return value;
+        }
+
+        return this.core.getAttribute(node, attr);
+    };
+
+    ExecuteJob.prototype._getValueFrom = function (nodeId, attr, node, changes) {
+        var base;
+        if (changes[nodeId] && changes[nodeId][attr] !== undefined) {
+            // If deleted the attribute, get the default (inherited) value
+            if (changes[nodeId][attr] === null) {
+                base = this.isCreateId(nodeId) ? node : this.core.getBase(node);
+                return this.getAttribute(base, attr);
+            }
+            return changes[nodeId][attr];
+        }
+    };
+
+    ExecuteJob.prototype._applyNodeChanges = function (node, changes) {
+        var attr,
+            value;
+
+        this.logger.info(`About to apply changes for ${this.core.getPath(node)}`);
+        for (var i = changes.length; i--;) {
+            attr = changes[i][0];
+            value = changes[i][1];
+            if (value !== null) {
+                this.logger.info(`Setting ${attr} to ${value} (${this.core.getPath(node)})`);
+                this.core.setAttribute(node, attr, value);
+            } else {
+                this.core.delAttribute(node, attr);
+            }
+        }
+        return node;
+    };
+
+    ExecuteJob.prototype.applyModelChanges = function () {
+        return this.applyCreations()
+            .then(() => this.applyChanges())
+            .then(() => this.applyDeletions());
+    };
+
+    ExecuteJob.prototype.applyChanges = function () {
+        var nodeIds = Object.keys(this.changes),
+            attrs,
+            value,
+            changes,
+            promises = [],
+            changesFor = {},
+            id,
+            promise;
+
+        this.logger.info('Collecting changes to apply in commit');
+
+        for (var i = nodeIds.length; i--;) {
+            changes = [];
+            attrs = Object.keys(this.changes[nodeIds[i]]);
+            for (var a = attrs.length; a--;) {
+                value = this.changes[nodeIds[i]][attrs[a]];
+                changes.push([attrs[a], value]);
+            }
+            changesFor[nodeIds[i]] = changes;
+
+            assert(changes, `changes are invalid for ${nodeIds[i]}: ${changes}`);
+            assert(!this.isCreateId(nodeIds[i]),
+                `Creation id not resolved to actual id: ${nodeIds[i]}`);
+            promise = this.core.loadByPath(this.rootNode, nodeIds[i]);
+            promises.push(promise);
+        }
+
+        this.currentChanges = this.changes;
+        this.changes = {};
+        // Need to differentiate between read/write changes.
+        this.logger.info(`About to apply changes for ${promises.length} nodes`);
+        return Q.all(promises)
+            .then(nodes => {
+                for (var i = nodes.length; i--;) {
+                    id = this.core.getPath(nodes[i]);
+                    assert(nodes[i], `node is ${nodes[i]} (${nodeIds[i]})`);
+                    this._applyNodeChanges(nodes[i], changesFor[id]);
+                }
+
+                // Local model is now up-to-date. No longer need currentChanges
+                this.currentChanges = {};
+            });
+    };
+
+    ExecuteJob.prototype.applyCreations = function () {
+        var nodeIds = Object.keys(this.creations),
+            tiers = this.createCreationTiers(nodeIds),
+            creations = this.creations,
+            newIds = {},
+            promise = Q(),
+            tier;
+
+        this.logger.info('Applying node creations');
+        for (var i = 0; i < tiers.length; i++) {
+            tier = tiers[i];
+            // Chain the promises, loading each tier sequentially
+            promise = promise.then(this.applyCreationTier.bind(this, creations, newIds, tier));
+        }
+
+        this.creations = {};
+        return promise;
+    };
+
+    ExecuteJob.prototype.applyCreationTier = function (creations, newIds, tier) {
+        var promises = [],
+            parentId,
+            node;
+
+        for (var j = tier.length; j--;) {
+            node = creations[tier[j]];
+            assert(node, `Could not find create info for ${tier[j]}`);
+            parentId = newIds[node.parent] || node.parent;
+            promises.push(this.applyCreation(tier[j], node.base, parentId));
+        }
+        return Q.all(promises).then(nodes => {
+            // Record the newIds so they can be used to resolve creation ids
+            // in subsequent tiers
+            for (var i = tier.length; i--;) {
+                newIds[tier[i]] = this.core.getPath(nodes[i]);
+            }
+        });
+    };
+
+    // Figure out the dependencies between nodes to create.
+    // eg, if newId1 is to be created in newId2, then newId2 will
+    // be in an earlier tier than newId1. Essentially a topo-sort
+    // on a tree structure
+    ExecuteJob.prototype.createCreationTiers = function (nodeIds) {
+        var tiers = [],
+            prevTier = {},
+            tier = {},
+            id,
+            prevLen,
+            i;
+
+        // Create first tier (created inside existing nodes)
+        for (i = nodeIds.length; i--;) {
+            id = nodeIds[i];
+            if (!this.isCreateId(this.creations[id].parent)) {
+                tier[id] = true;
+                nodeIds.splice(i, 1);
+            }
+        }
+        prevTier = tier;
+        tiers.push(Object.keys(tier));
+
+        // Now, each tier consists of the nodes to be created inside a
+        // node from the previous tier
+        while (nodeIds.length) {
+            prevLen = nodeIds.length;
+            tier = {};
+            for (i = nodeIds.length; i--;) {
+                id = nodeIds[i];
+                if (prevTier[this.creations[id].parent]) {
+                    tier[id] = true;
+                    nodeIds.splice(i, 1);
+                }
+            }
+            prevTier = tier;
+            tiers.push(Object.keys(tier));
+            // Every iteration should find at least one node
+            assert(prevLen > nodeIds.length,
+                `Created empty create tier! Remaining: ${nodeIds.join(', ')}`);
+        }
+
+        return tiers;
+    };
+
+    ExecuteJob.prototype.applyCreation = function (tmpId, baseType, parentId) {
+        var base = this.META[baseType],
+            nodeId,
+            id;
+
+        this.logger.info(`Applying creation of ${tmpId} (${baseType}) in ${parentId}`);
+
+        assert(!this.isCreateId(parentId),
+            `Did not resolve parent id: ${parentId} for ${tmpId}`);
+        assert(base, `Invalid base type: ${baseType}`);
+        return this.core.loadByPath(this.rootNode, parentId)
+            .then(parent => this.core.createNode({base, parent}))
+            .then(node => {  // Update the _metadata records
+                id = this.createIdToMetadataId[tmpId];
+                delete this.createIdToMetadataId[tmpId];
+                this._metadata[id] = node;
+
+                // Update creations
+                nodeId = this.core.getPath(node);
+                if (this.changes[tmpId]) {
+                    assert(!this.changes[nodeId],
+                        `Newly created node cannot already have changes! (${nodeId})`);
+                    this.changes[nodeId] = this.changes[tmpId];
+                    delete this.changes[tmpId];
+                }
+                return node;
+            });
+    };
+
+    ExecuteJob.prototype.applyDeletions = function () {
+        var deletions = this.deletions;
+
+        this.deletions = [];
+        return Q.all(deletions.map(id => this.core.loadByPath(this.rootNode, id)))
+            .then(nodes => {
+                for (var i = nodes.length; i--;) {
+                    this.core.deleteNode(nodes[i]);
+                }
+            });
+    };
+
+    // Override 'save' to notify the user on fork
+    ExecuteJob.prototype.save = function (msg) {
+        this._currentSave = this._currentSave
+            .then(() => this.updateForkName(this.forkNameBase))
+            .then(() => this.applyModelChanges())
+            .then(() => PluginBase.prototype.save.call(this, msg))
+            .then(result => {
+                this.logger.info(`Save finished w/ status: ${result.status}`);
+                if (result.status === STORAGE_CONSTANTS.FORKED) {
+                    return this.onSaveForked(result.forkName);
+                } else if (result.status === STORAGE_CONSTANTS.MERGED) {
+                    this.logger.debug('Merged changes. About to update plugin nodes');
+                    return this.updateNodes();
+                }
+            });
+
+        return this._currentSave;
+    };
+
+    ExecuteJob.prototype.onSaveForked = function (forkName) {
+        var name = this.getAttribute(this.activeNode, 'name'),
+            msg = `"${name}" execution has forked to "${forkName}"`;
+        this.currentForkName = forkName;
+
+        this.logManager.fork(forkName);
+        this.runningJobHashes.forEach(jobId => this.originManager.fork(jobId, forkName));
+        this.sendNotification(msg);
+    };
+
+    ExecuteJob.prototype.updateNodes = function (hash) {
+        var activeId = this.core.getPath(this.activeNode);
+
+        hash = hash || this.currentHash;
+        return Q.ninvoke(this.project, 'loadObject', hash)
+            .then(commitObject => {
+                return this.core.loadRoot(commitObject.root);
+            })
+            .then(rootObject => {
+                this.rootNode = rootObject;
+                return this.core.loadByPath(rootObject,activeId);
+            })
+            .then(activeObject => this.activeNode = activeObject)
+            .then(() => {
+                var metaNames = Object.keys(this.META);
+
+                return Q.all(metaNames.map(name => this.updateMetaNode(name)));
+            })
+            .then(() => {
+                var mdNodes,
+                    mdIds;
+
+                mdIds = Object.keys(this._metadata)
+                    .filter(id => !this.isCreateId(this._metadata[id]));
+
+                mdNodes = mdIds.map(id => this.core.getPath(this._metadata[id]))
+                    .map(nodeId => this.core.loadByPath(this.rootNode, nodeId));
+
+                return Q.all(mdNodes).then(nodes => {
+                    for (var i = nodes.length; i--;) {
+                        this._metadata[mdIds[i]] = nodes[i];
+                    }
+                });
+            });
+    };
+
+    ExecuteJob.prototype.updateMetaNode = function (name) {
+        var id = this.core.getPath(this.META[name]);
+        return this.core.loadByPath(this.rootNode, id).then(node => this.META[name] = node);
+    };
+
+    return ExecuteJob;
+});

src/plugins/ExecuteJob/templates/deepforge.ejs

@@ -52,6 +52,14 @@ function Graph:line(name, opts)
     return deepforge._Line(self.id, name, opts)
 end
 
+function Graph:xlabel(name)
+    deepforge._cmd('<%= GRAPH_LABEL_AXIS.X %>', self.id, name)
+end
+
+function Graph:ylabel(name)
+    deepforge._cmd('<%= GRAPH_LABEL_AXIS.Y %>', self.id, name)
+end
+
 -- Image support
 local function saveImage(name, tensor)
     require 'image'

src/plugins/ExecuteJob/templates/start.ejs

@@ -74,7 +74,7 @@ requirejs([
     var checkFinished = () => {
         if (exitCode !== null && remainingImageCount === 0) {
             log('finished!');
-            process.exit(exitCode);
+            cleanup();
         }
     };
 
@@ -201,14 +201,45 @@ requirejs([
 
     // Download the large files
     var inputData = JSON.parse(fs.readFileSync('./input-data.json')),
-        inputPaths = Object.keys(inputData);
+        inputPaths = Object.keys(inputData),
+        pid,
+        job,
+        cleanup;
+
+    // Make sure to kill the spawned process group on exit
+    cleanup = function() {
+        if (job) {
+            pid = job.pid;
+            job = null;
+            log(`killing process group: ${pid}`);
+            process.kill(-pid, 'SIGTERM');
+            if (exitCode !== null) {
+                log(`exiting w/ code ${exitCode}`);
+                process.exit(exitCode);
+            }
+        }
+    };
+
+    process.on('exit', () => {
+        log('received "exit" event')
+        cleanup();
+    });
+    process.on('SIGINT', function() {
+        log('received "SIGINT" event')
+        cleanup();
+        process.exit(130);
+    });
+    process.on('uncaughtException', () => {
+        log('received "uncaughtException" event')
+        cleanup();
+    });
 
     // Request the data from the blob
     prepareCache()
         .then(() => Q.all(inputPaths.map(ipath => getData(ipath, inputData[ipath]))))
         .then(() => {
             // Run 'th init.lua' and merge the stdout, stderr
-            var job = spawn('th', ['init.lua']);
+            job = spawn('th', ['init.lua'], {detached: true});
             job.stdout.on('data', onStdout);
             job.stderr.on('data', onStderr);
             job.on('close', code => {

src/plugins/ExecutePipeline/ExecutePipeline.js

@@ -4,17 +4,17 @@
 define([
     'plugin/CreateExecution/CreateExecution/CreateExecution',
     'plugin/ExecuteJob/ExecuteJob/ExecuteJob',
-    'deepforge/JobLogsClient',
     'common/storage/constants',
     'common/core/constants',
+    'deepforge/Constants',
     'q',
     'text!./metadata.json',
     'underscore'
 ], function (
     CreateExecution,
     ExecuteJob,
-    JobLogsClient,
     STORAGE_CONSTANTS,
+    GME_CONSTANTS,
     CONSTANTS,
     Q,
     pluginMetadata,
@@ -33,10 +33,14 @@ define([
     var ExecutePipeline = function () {
         // Call base class' constructor.
         CreateExecution.call(this);
+        ExecuteJob.call(this);
         this.pluginMetadata = pluginMetadata;
 
-        this._currentSave = Q();
         this.changes = {};
+        this.currentChanges = {};  // read-only changes being applied
+        this.creations = {};
+        this.deletions = [];
+        this.createIdToMetadataId = {};
         this.initRun();
     };
 
@@ -97,7 +101,8 @@ define([
      * @param {function(string, plugin.PluginResult)} callback - the result callback
      */
     ExecutePipeline.prototype.main = function (callback) {
-        var startPromise;
+        var startPromise,
+            runId;
 
         this.initRun();
         if (this.core.isTypeOf(this.activeNode, this.META.Pipeline)) {
@@ -114,58 +119,133 @@ define([
             return callback('Current node is not a Pipeline or Execution!', this.result);
         }
 
-        // Get the gmeConfig...
-        this.logManager = new JobLogsClient({
-            logger: this.logger,
-            port: this.gmeConfig.server.port,
-            branchName: this.branchName,
-            projectId: this.projectId
-        });
         this._callback = callback;
         this.currentForkName = null;
 
         startPromise
-        .then(() => this.core.loadSubTree(this.activeNode))
-        .then(subtree => {
-            var children = subtree
-                .filter(n => this.core.getParent(n) === this.activeNode);
+            .then(() => this.core.loadSubTree(this.activeNode))
+            .then(subtree => {
+                var children;
 
-            this.pipelineName = this.getAttribute(this.activeNode, 'name');
-            this.logger.debug(`Loaded subtree of ${this.pipelineName}. About to build cache`);
-            this.buildCache(subtree);
-            this.logger.debug('Parsing execution for job inter-dependencies');
-            this.parsePipeline(children);  // record deps, etc
+                children = subtree
+                    .filter(n => this.core.getParent(n) === this.activeNode);
 
-            this.logger.debug('Clearing old results');
-            return this.clearResults();
-        })
-        .then(() => this.executePipeline())
-        .fail(e => this.logger.error(e));
+                this.pipelineName = this.getAttribute(this.activeNode, 'name');
+                this.forkNameBase = this.pipelineName;
+                this.logger.debug(`Loaded subtree of ${this.pipelineName}. About to build cache`);
+                this.buildCache(subtree);
+                this.logger.debug('Parsing execution for job inter-dependencies');
+                this.parsePipeline(children);  // record deps, etc
+
+                // Detect if resuming execution
+                runId = this.getAttribute(this.activeNode, 'runId');
+                return this.isResuming().then(resuming => {
+                    if (resuming) {
+                        this.currentRunId = runId;
+                        this.startExecHeartBeat();
+                        return this.resumePipeline();
+                    }
+                    return this.startPipeline();
+                });
+
+            })
+            .fail(e => this.logger.error(e));
     };
 
-    // Override 'save' to prevent race conditions while saving
-    ExecutePipeline.prototype.save = function (msg) {
-        // When 'save'  is called, it should still finish any current save op
-        // before continuing
-        this._currentSave = this._currentSave
-            .then(() => this.updateForkName(this.pipelineName))
-            .then(() => this.applyChanges())
-            .then(() => CreateExecution.prototype.save.call(this, msg))
-            .then(result => {
-                var msg;
-                if (result.status === STORAGE_CONSTANTS.FORKED) {
-                    this.currentForkName = result.forkName;
-                    this.logManager.fork(result.forkName);
-                    msg = `"${this.pipelineName}" execution has forked to "${result.forkName}"`;
-                    this.sendNotification(msg);
-                } else if (result.status === STORAGE_CONSTANTS.MERGED) {
-                    this.logger.debug('Merged changes. About to update plugin nodes');
-                    return this.updateNodes();
+    ExecutePipeline.prototype.isResuming = function () {
+        var currentlyRunning = this.getAttribute(this.activeNode, 'status') === 'running',
+            runId = this.getAttribute(this.activeNode, 'runId');
+
+        if (runId && currentlyRunning) {
+            // Verify that it is on the correct branch
+            return this.originManager.getOrigin(runId)
+                .then(origin => {
+                    if (origin && origin.branch === this.branchName) {
+                        return this.pulseClient.check(runId)
+                            // If it is dead (not unknown!), then resume
+                            .then(status => status === CONSTANTS.PULSE.DEAD);
+                    } else {
+                        return false;
+                    }
+                });
+        }
+        return Q().then(() => false);
+    };
+
+    ExecutePipeline.prototype.resumePipeline = function () {
+        var nodes = Object.keys(this.nodes).map(id => this.nodes[id]),
+            allJobs = nodes.filter(node => this.core.isTypeOf(node, this.META.Job)),
+            status,
+            jobs = {
+                success: [],
+                failed: [],
+                running: [],
+                pending: []
+            };
+
+        this.logger.info(`Resuming pipeline execution: ${this.currentRunId}`);
+
+        // Get all completed jobs' operations and update records for these
+        for (var i = allJobs.length; i--;) {
+            status = this.getAttribute(allJobs[i], 'status');
+            if (!jobs[status]) {
+                jobs[status] = [];
+            }
+
+            // If any running jobs are missing jobIds, set them to pending
+            if (status === 'running' && !this.canResumeJob(allJobs[i])) {
+                jobs.pending.push(allJobs[i]);
+            } else {
+                jobs[status].push(allJobs[i]);
+            }
+        }
+
+        // Remove finished jobs from incomingCounts
+        jobs.success.concat(jobs.failed, jobs.running)
+            .map(job => this.core.getPath(job))
+            .forEach(id => delete this.incomingCounts[id]);
+
+        return Q.all(allJobs.map(job => this.recordOldMetadata(job, true)))
+            .then(() => Q.all(jobs.success.map(job => this.getOperation(job))))
+            .then(ops => ops.forEach(op => this.updateJobCompletionRecords(op)))
+            .then(() => {
+                
+                if (jobs.running.length) {  // Resume all running jobs
+                    return Q.all(jobs.running.map(job => this.resumeJob(job)));
+                } else if (this.completedCount === this.totalCount) {
+                    return this.onPipelineComplete();
+                } else {
+                    // If none are running, try to start the next ones
+                    return this.executeReadyOperations();
                 }
+            })
+            .fail(err => this._callback(err));
+    };
 
-            });
+    ExecutePipeline.prototype.startPipeline = function () {
+        var rand = Math.floor(Math.random()*10000),
+            commit = this.commitHash.replace('#', '');
 
-        return this._currentSave;
+        this.logger.debug('Clearing old results');
+        this.currentRunId = `Pipeline_${commit}_${Date.now()}_${rand}`;
+
+        // Record the execution origin
+        this.originManager.record(this.currentRunId, {
+            nodeId: this.core.getPath(this.activeNode),
+            job: 'N/A',
+            execution: this.getAttribute(this.activeNode, 'name')
+        });
+
+        this.startExecHeartBeat();
+        return this.clearResults()
+            .then(() => this.executePipeline())
+            .fail(e => this.logger.error(e));
+    };
+
+    ExecutePipeline.prototype.onSaveForked = function (forkName) {
+        // Update the origin on fork
+        this.originManager.fork(this.currentRunId, forkName);
+        return ExecuteJob.prototype.onSaveForked.call(this, forkName);
     };
 
     ExecutePipeline.prototype.updateNodes = function (hash) {
@@ -185,6 +265,10 @@ define([
         });
     };
 
+    ExecutePipeline.prototype.isExecutionCanceled = function () {
+        return this.getAttribute(this.activeNode, 'status') === 'canceled';
+    };
+
     ExecutePipeline.prototype.isInputData = function (node) {
         var prnt = this.core.getParent(node);
         return this.core.isTypeOf(prnt, this.META.Inputs);
@@ -212,6 +296,7 @@ define([
         this.logger.info('Setting all jobs status to "pending"');
         this.logger.debug(`Making a commit from ${this.currentHash}`);
         this.setAttribute(this.activeNode, 'startTime', Date.now());
+        this.setAttribute(this.activeNode, 'runId', this.currentRunId);
         this.core.delAttribute(this.activeNode, 'endTime');
         return this.save(`Initializing ${this.pipelineName} for execution`);
     };
@@ -289,10 +374,10 @@ define([
     };
 
     ExecutePipeline.prototype.getSiblingIdContaining = function (nodeId) {
-        var parentId = this.core.getPath(this.activeNode) + CONSTANTS.PATH_SEP,
+        var parentId = this.core.getPath(this.activeNode) + GME_CONSTANTS.PATH_SEP,
             relid = nodeId.replace(parentId, '');
 
-        return parentId + relid.split(CONSTANTS.PATH_SEP).shift();
+        return parentId + relid.split(GME_CONSTANTS.PATH_SEP).shift();
     };
 
     ExecutePipeline.prototype.executePipeline = function() {
@@ -350,7 +435,8 @@ define([
             msg += 'finished!';
         }
 
-        this.isDeleted().then(isDeleted => {
+        return this.isDeleted().then(isDeleted => {
+            this.stopExecHeartBeat();
             if (!isDeleted) {
 
                 this.logger.debug(`Pipeline "${name}" complete!`);
@@ -433,10 +519,9 @@ define([
 
     ExecutePipeline.prototype.onOperationComplete = function (opNode) {
         var name = this.getAttribute(opNode, 'name'),
-            nextPortIds = this.getOperationOutputIds(opNode),
             jNode = this.core.getParent(opNode),
-            resultPorts,
             jobId = this.core.getPath(jNode),
+            counts,
             hasReadyOps;
 
         // Set the operation to 'success'!
@@ -449,35 +534,9 @@ define([
         this.save(`Operation "${name}" in ${this.pipelineName} completed successfully`)
             .then(() => {
 
-                // Transport the data from the outputs to any connected inputs
-                //   - Get all the connections from each outputId
-                //   - Get the corresponding dst outputs
-                //   - Use these new ids for checking 'hasReadyOps'
-                resultPorts = nextPortIds.map(id => this.inputPortsFor[id])
-                    .reduce((l1, l2) => l1.concat(l2), []);
+                counts = this.updateJobCompletionRecords(opNode);
+                hasReadyOps = counts.indexOf(0) > -1;
 
-                resultPorts
-                    .map((id, i) => [this.nodes[id], this.nodes[nextPortIds[i]]])
-                    .forEach(pair => {  // [ resultPort, nextPort ]
-                        var result = pair[0],
-                            next = pair[1],
-                            hash = this.getAttribute(result, 'data');
-                        
-                        this.logger.info(`forwarding data (${hash}) from ${this.core.getPath(result)} ` +
-                            `to ${this.core.getPath(next)}`);
-                        this.setAttribute(next, 'data', hash);
-                        this.logger.info(`Setting ${jobId} data to ${hash}`);
-                    });
-
-                // For all the nextPortIds, decrement the corresponding operation's incoming counts
-                hasReadyOps = nextPortIds.map(id => this.getSiblingIdContaining(id))
-                    .reduce((l1, l2) => l1.concat(l2), [])
-
-                    // decrement the incoming counts for each operation id
-                    .map(opId => --this.incomingCounts[opId])
-                    .indexOf(0) > -1;
-
-                this.completedCount++;
                 this.logger.debug(`Operation "${name}" completed. ` + 
                     `${this.totalCount - this.completedCount} remaining.`);
                 if (hasReadyOps) {
@@ -488,9 +547,47 @@ define([
             });
     };
 
+    ExecutePipeline.prototype.updateJobCompletionRecords = function (opNode) {
+        var nextPortIds = this.getOperationOutputIds(opNode),
+            resultPorts,
+            counts;
+
+        // Transport the data from the outputs to any connected inputs
+        //   - Get all the connections from each outputId
+        //   - Get the corresponding dst outputs
+        //   - Use these new ids for checking 'hasReadyOps'
+
+        resultPorts = nextPortIds.map(id => this.inputPortsFor[id])  // dst -> src port
+            .reduce((l1, l2) => l1.concat(l2), []);
+
+        resultPorts
+            .map((id, i) => [this.nodes[id], this.nodes[nextPortIds[i]]])
+            .forEach(pair => {  // [ resultPort, nextPort ]
+                var result = pair[0],
+                    next = pair[1],
+                    hash = this.getAttribute(result, 'data');
+                
+                this.logger.info(`forwarding data (${hash}) from ${this.core.getPath(result)} ` +
+                    `to ${this.core.getPath(next)}`);
+                this.setAttribute(next, 'data', hash);
+                //this.logger.info(`Setting ${jobId} data to ${hash}`);
+            });
+
+        // For all the nextPortIds, decrement the corresponding operation's incoming counts
+        counts = nextPortIds.map(id => this.getSiblingIdContaining(id))
+            .reduce((l1, l2) => l1.concat(l2), [])
+
+            // decrement the incoming counts for each operation id
+            .map(opId => --this.incomingCounts[opId]);
+
+        this.completedCount++;
+        return counts;
+    };
+
     ExecutePipeline.prototype.getOperationOutputIds = function(node) {
         var jobId = this.getSiblingIdContaining(this.core.getPath(node));
 
+        // Map the job to it's output ports
         return this.outputsOf[jobId] || [];
     };
 

src/plugins/Export/Export.js

@@ -0,0 +1,925 @@
+/*globals define */
+/*jshint node:true, browser:true*/
+
+define([
+    'text!./metadata.json',
+    'text!./deepforge.ejs',
+    './format',
+    'plugin/PluginBase',
+    'deepforge/plugin/PtrCodeGen',
+    'deepforge/Constants',
+    'blob/BlobConfig',
+    'underscore',
+    'q'
+], function (
+    pluginMetadata,
+    DeepForgeBaseCode,
+    FORMATS,
+    PluginBase,
+    PtrCodeGen,
+    CONSTANTS,
+    BlobConfig,
+    _,
+    Q
+) {
+    'use strict';
+
+    pluginMetadata = JSON.parse(pluginMetadata);
+    var HEADER_LENGTH = 60,
+        SKIP_ATTRS = {
+            lineOffset: true,
+            code: true
+        },
+        RESERVED = /^(and|break|do|else|elseifend|false|for|function|if|in|local|nil|not|orrepeat|return|then|true|until|while|print)$/,
+        DeepForgeTpl = _.template(DeepForgeBaseCode);
+
+    /**
+     * Initializes a new instance of Export.
+     * @class
+     * @augments {PluginBase}
+     * @classdesc This class represents the plugin Export.
+     * @constructor
+     */
+    var Export = function () {
+        // Call base class' constructor.
+        PluginBase.call(this);
+        this.initRecords();
+    };
+
+    /**
+     * Metadata associated with the plugin. Contains id, name, version, description, icon, configStructue etc.
+     * This is also available at the instance at this.pluginMetadata.
+     * @type {object}
+     */
+    Export.metadata = pluginMetadata;
+
+    // Prototypical inheritance from PluginBase.
+    Export.prototype = Object.create(PluginBase.prototype);
+    Export.prototype.constructor = Export;
+
+    Export.prototype.initRecords = function() {
+        this.pluginMetadata = pluginMetadata;
+
+        this._srcIdFor = {};  // input path -> output data node path
+
+        this._nameFor = {};  // input path -> opname
+        this._outputNames = {};
+        this._baseNameFor = {};
+        this._dataNameFor = {};  
+        this._instanceNames = {};
+        this._opBaseNames = {};
+        this._fnNameFor = {};
+        this._functions = {};  // function definitions for the operations
+
+        // topo sort stuff
+        this._nextOps = {};
+        this._incomingCnts = {};
+
+        this._operations = {};
+        this.activeNodeId = null;
+        this.activeNodeDepth = null;
+
+        this.isInputOp = {};
+        this._portCache = {};
+        this.inputNode = {};
+        this.outputDataToOpId = {};
+        this.isOutputOp = {};
+    };
+
+    /**
+     * Main function for the plugin to execute. This will perform the execution.
+     * Notes:
+     * - Always log with the provided logger.[error,warning,info,debug].
+     * - Do NOT put any user interaction logic UI, etc. inside this method.
+     * - callback always has to be called even if error happened.
+     *
+     * @param {function(string, plugin.PluginResult)} callback - the result callback
+     */
+    Export.prototype.main = function (callback) {
+        this.initRecords();
+
+        // Get all the children and call generate exec file
+        this.activeNodeId = this.core.getPath(this.activeNode);
+        this.activeNodeDepth = this.activeNodeId.split('/').length + 1;
+
+        if (this.isMetaTypeOf(this.activeNode, this.META.Execution)) {
+            this.activeNodeDepth++;
+        }
+
+        return this.core.loadChildren(this.activeNode)
+            .then(nodes => this.generateOutputFiles(nodes))
+            .catch(err => callback(err))
+            .then(hash => {
+                this.result.addArtifact(hash);
+                this.result.setSuccess(true);
+                callback(null, this.result);
+            })
+            .fail(err => callback(err));
+    };
+
+    Export.prototype.getCurrentConfig = function () {
+        var config = PluginBase.prototype.getCurrentConfig.call(this);
+        config.staticInputs = config.staticInputs || [];
+        return config;
+    };
+
+    Export.prototype.getExporterFor = function (name) {
+        var Exporter = function() {},
+            format = FORMATS[name],
+            exporter;
+
+        Exporter.prototype = this;
+        exporter = new Exporter();
+
+        if (typeof format === 'function') {
+            exporter.main = format;
+        } else {
+            _.extend(exporter, format);
+        }
+        return exporter;
+    };
+
+    Export.prototype.generateOutputFiles = function (children) {
+        var name = this.core.getAttribute(this.activeNode, 'name');
+
+        return this.createCodeSections(children)
+            .then(sections => {
+                // Get the selected format
+                var config = this.getCurrentConfig(),
+                    format = config.format || 'Basic CLI',
+                    exporter,
+                    staticInputs,
+                    files;
+
+                this.logger.info(`About to retrieve ${config.format} exporter`);
+                exporter = this.getExporterFor(format);
+
+                staticInputs = config.staticInputs.map(id => {
+                    var opId = id.split('/').splice(0, this.activeNodeDepth).join('/'),
+                        port = this._portCache[id];
+
+                    return {
+                        portId: id,
+                        id: opId,
+                        hash: this.core.getAttribute(port, 'data'),
+                        name: this._nameFor[opId]
+                    };
+                });
+
+                this.logger.info('Invoking exporter "main" function...');
+                try {
+                    files = exporter.main(sections, staticInputs, config.extensionConfig);
+                } catch (e) {
+                    this.logger.error(`Exporter failed: ${e.toString()}`);
+                    throw e;
+                }
+                // If it returns a string, just put a single file
+                if (typeof files === 'string') {
+                    return this.blobClient.putFile(`${name}.lua`, files);
+                } else {  // filename -> content
+                    var artifact = this.blobClient.createArtifact(name),
+                        objects = {};
+
+                    Object.keys(files).forEach(key => {
+                        if (BlobConfig.hashRegex.test(files[key])) {
+                            objects[key] = files[key];
+                            delete files[key];
+                        }
+                    });
+
+                    return artifact.addFiles(files)
+                        .then(() => artifact.addObjectHashes(objects))
+                        .then(() => artifact.save());
+                }
+            });
+    };
+
+    Export.prototype.createCodeSections = function (children) {
+        // Convert opNodes' jobs to the nested operations
+        var opNodes,
+            nodes;
+
+        return this.unpackJobs(children)
+            .then(_nodes => {
+                nodes = _nodes;
+                opNodes = nodes
+                    .filter(node => this.isMetaTypeOf(node, this.META.Operation));
+
+                // Sort the connections to come first
+                nodes
+                    .map(node => [
+                        node,
+                        this.isMetaTypeOf(node, this.META.Transporter) ? -1 : 1
+                    ])
+                    .sort((a, b) => a[1] < b[1] ? -1 : 1);
+
+                return Q.all(nodes.map(node => this.registerNode(node)));
+            })
+            .then(() => Q.all(opNodes
+                .filter(n => {
+                    var id = this.core.getPath(n);
+                    return !this.isInputOp[id];
+                })
+                .map(node => this.createOperation(node)))
+            )
+            .then(operations => {
+                var opDict = {},
+                    firstOpIds;
+
+                firstOpIds = opNodes.map(n => this.core.getPath(n))
+                    .filter(id => !this._incomingCnts[id]);
+
+                operations.forEach(op => opDict[op.id] = op);
+
+                // Toposort!
+                return this.sortOperations(opDict, firstOpIds);
+            })
+            .then(operations => this.generateCodeSections(operations))
+            .fail(err => this.logger.error(err));
+    };
+
+    Export.prototype.unpackJobs = function (nodes) {
+        return Q.all(
+            nodes.map(node => {
+                if (!this.isMetaTypeOf(node, this.META.Job)) {
+                    return node;
+                }
+                return this.core.loadChildren(node)
+                    .then(children =>
+                        children.find(c => this.isMetaTypeOf(c, this.META.Operation))
+                    );
+            })
+        );
+    };
+
+    Export.prototype.sortOperations = function (operationDict, opIds) {
+        var nextIds = [],
+            sorted = opIds,
+            dstIds,
+            id;
+
+        if (!opIds.length) {
+            return [];
+        }
+
+        // Decrement all next ops
+        dstIds = opIds.map(id => this._nextOps[id])
+            .reduce((l1, l2) => l1.concat(l2), []);
+
+        for (var i = dstIds.length; i--;) {
+            id = dstIds[i];
+            if (--this._incomingCnts[id] === 0) {
+                nextIds.push(id);
+            }
+        }
+
+        // append
+        return sorted
+            .map(id => operationDict[id])
+            .filter(op => !!op)
+            .concat(this.sortOperations(operationDict, nextIds));
+    };
+
+    Export.prototype.generateCodeSections = function(sortedOps) {
+        // Create the code sections:
+        //  - operation definitions
+        //  - pipeline definition
+        //  - main
+        var code = {},
+            baseIds = [],
+            outputOps = [],
+            mainOps = [];
+
+        // Define the operation functions...
+        code.operations = {};
+        for (var i = 0; i < sortedOps.length; i++) {
+            if (this.isInputOp[sortedOps[i].id]) {
+                continue;
+            }
+            if (!this.isOutputOp[sortedOps[i].id]) {
+                if (!baseIds.includes(sortedOps[i].baseId)) {  // new definition
+                    code.operations[sortedOps[i].basename] = this.defineOperationFn(sortedOps[i]);
+                    baseIds.push(sortedOps[i].baseId);
+                }
+                mainOps.push(sortedOps[i]);
+            } else {
+                outputOps.push(sortedOps[i]);
+            }
+        }
+
+        // Define the pipeline function
+        code.pipelines = this.definePipelineFn(mainOps, outputOps);
+
+        // Define the serializers/deserializers
+        this.addCodeSerializers(code);
+
+        // Define the main input names
+        code.pipelineName = Object.keys(code.pipelines)[0];
+        code.pipelineInputNames = Object.keys(this.isInputOp).map(id => this._nameFor[id]);
+
+        // Add custom class definitions
+        this.addCustomClasses(code);
+
+        // Add custom layer definitions
+        this.addCustomLayers(code);
+
+        return code;
+    };
+
+    // expose this utility function to format extensions
+    var indent = Export.prototype.indent = function(text, spaces) {
+        spaces = spaces || 3;
+        return text.replace(/^/mg, new Array(spaces+1).join(' '));
+    };
+
+    Export.prototype.defineOperationFn = function(operation) {
+        var lines = [],
+            args = operation.inputNames || [];
+
+        // Create the function definition
+        args.unshift('attributes');
+        // Add the refs to the end
+        args = args.concat(operation.refNames);
+
+        args = args.join(', ');
+
+        lines.push(`local function ${operation.basename}(${args})`);
+        lines.push(indent(operation.code));
+        lines.push('end');
+
+        return lines.join('\n');
+    };
+
+    Export.prototype.definePipelineFn = function(sortedOps, outputOps) {
+        var inputArgs = Object.keys(this.isInputOp).map(id => this._nameFor[id]),
+            name = this.core.getAttribute(this.activeNode, 'name'),
+            safename = getUniqueName(name, this._opBaseNames),
+            results = [],
+            result = {},
+            returnStat,
+            fnbody;
+
+        // Call each function in order, with the respective attributes, etc
+        fnbody = sortedOps.map(op => this.getOpInvocation(op)).join('\n');
+
+        // Create the return statement
+        results.push('\n\nresults = {}');
+        outputOps.map(op => this.getOutputPair(op))
+            .forEach(pair => results.push(`results['${pair[0]}'] = ${pair[1]}`));
+        results.push('return results');
+        returnStat = results.join('\n');
+
+        // Merge the fnbody, return statement and the function def
+        result[safename] = `local function ${safename} (${inputArgs.join(', ')})\n` +
+            `${indent(fnbody + returnStat)}\nend`;
+
+        return result;
+    };
+
+    Export.prototype.getOutputPair = function(operation) {
+        var input = operation.inputValues[0].slice(),
+            value;
+
+        // Get the src operation name and data value name
+        input[0] += '_results';
+        value = input.join('.');
+        return [this._nameFor[operation.id], value];
+    };
+
+    Export.prototype.addCodeSerializers = function(sections) {
+        var loadNodes = {},
+            saveNodes = {};
+
+        // Add the serializer fn names for each input
+        sections.serializerFor = {};
+        sections.deserializerFor = {};
+
+        Object.keys(this.isOutputOp).map(id => {
+            var name = this._nameFor[id];
+            sections.serializerFor[name] = `__save['${name}']`;
+        });
+
+        // Add the serializer definitions
+        Object.keys(this.isInputOp).forEach(id => {
+            var node = this.inputNode[id],
+                name = this._nameFor[id];
+
+            loadNodes[id] = node;
+            sections.deserializerFor[name] = `__load['${this._nameFor[id]}']`;
+        });
+
+        sections.deserializers = this.createTorchFnDict(
+            '__load',
+            loadNodes,
+            'deserialize',
+            'path'
+        );
+
+        // Add the deserializer definitions
+        Object.keys(this.outputDataToOpId).forEach(dataId => {
+            var opId = this.outputDataToOpId[dataId];
+            // The key is used for the output name resolution. The
+            // value is used for the serialization fn look-up. So,
+            // the key is the output operation id and the value is
+            // the data port connected to the output operation
+            saveNodes[opId] = this._portCache[this._srcIdFor[dataId]];
+        });
+
+        sections.serializers = this.createTorchFnDict(
+            '__save',
+            saveNodes,
+            'serialize',
+            'path, data'
+        );
+
+        // Add a saveOutputs method for convenience
+        sections.serializeOutputsDef = [
+            'local function __saveOutputs(data)',
+            indent(Object.keys(this.isOutputOp).map(id => {
+                var name = this._nameFor[id];
+                return [
+                    `print('saving ${name}...')`,
+                    `${sections.serializerFor[name]}('${name}', data['${name}'])`
+                ].join('\n');
+
+            }).join('\n')),
+            'end'
+        ].join('\n');
+
+        sections.serializeOutputs = '__saveOutputs(outputs)';
+    };
+
+    Export.prototype.createTorchFnDict = function(name, nodeDict, attr, args) {
+        return [
+            `local ${name} = {}`,
+            Object.keys(nodeDict).map(id => {
+                var node = nodeDict[id];
+                return [
+                    `${name}['${this._nameFor[id]}'] = function(${args})`,
+                    indent(this.core.getAttribute(node, attr)),
+                    'end'
+                ].join('\n');
+            }).join('\n')
+        ].join('\n');
+    };
+
+    Export.prototype.addCustomClasses = function(sections) {
+        var metaDict = this.core.getAllMetaNodes(this.rootNode),
+            isClass,
+            metanodes,
+            classNodes,
+            inheritanceLvl = {};
+
+        this.logger.info('Creating custom layer file...');
+        metanodes = Object.keys(metaDict).map(id => metaDict[id]);
+        isClass = this.getTypeDictFor('Complex', metanodes);
+
+        // Store the dependencies for each class
+        sections.classDependencies = {};
+
+        classNodes = metanodes.filter(node => {
+            var base = this.core.getBase(node),
+                baseId,
+                deps = [],
+                name,
+                count = 1;
+
+            // Count the sets back to a class node
+            while (base) {
+                deps.push(this.core.getAttribute(base, 'name'));
+                baseId = this.core.getPath(base);
+                if (isClass[baseId]) {
+                    inheritanceLvl[this.core.getPath(node)] = count;
+                    name = this.core.getAttribute(node, 'name');
+                    sections.classDependencies[name] = deps;
+                    return true;
+                }
+                base = this.core.getBase(base);
+                count++;
+            }
+
+            return false;
+        });
+
+        // Get the code definitions for each
+        sections.classes = {};
+        classNodes
+            .sort((a, b) => {
+                var aId = this.core.getPath(a),
+                    bId = this.core.getPath(b);
+
+                return inheritanceLvl[aId] > inheritanceLvl[bId];
+            })
+            .forEach(node => {
+                var name = this.core.getAttribute(node, 'name'),
+                    code = this.core.getAttribute(node, 'code');
+
+                sections.classes[name] = code;
+            });
+
+        // order classes by dependency
+        sections.orderedClasses = Object.keys(sections.classes)
+            .sort((a, b) => {
+                // if a depends on b, switch them (return 1)
+                if (sections.classDependencies[a].includes(b)) {
+                    return 1;
+                }
+                return -1;
+            });
+    };
+
+    Export.prototype.addCustomLayers = function(sections) {
+        var metaDict = this.core.getAllMetaNodes(this.rootNode),
+            isCustomLayer,
+            metanodes,
+            customLayers;
+
+        this.logger.info('Creating custom layer file...');
+        metanodes = Object.keys(metaDict).map(id => metaDict[id]);
+        isCustomLayer = this.getTypeDictFor('CustomLayer', metanodes);
+
+        customLayers = metanodes.filter(node =>
+            this.core.getMixinPaths(node).some(id => isCustomLayer[id]));
+
+        // Get the code definitions for each
+        sections.layers = {};
+        customLayers
+            .map(layer => [
+                this.core.getAttribute(layer, 'name'),
+                this.core.getAttribute(layer, 'code')
+            ])
+            .forEach(pair => sections.layers[pair[0]] = pair[1]);
+    };
+
+
+    Export.prototype.getTypeDictFor = function (name, metanodes) {
+        var isType = {};
+        // Get all the custom layers
+        for (var i = metanodes.length; i--;) {
+            if (this.core.getAttribute(metanodes[i], 'name') === name) {
+                isType[this.core.getPath(metanodes[i])] = true;
+            }
+        }
+        return isType;
+    };
+
+    var toAttrString = function(attr) {
+        if (/^\d+\.?\d*$/.test(attr) || /^(true|false|nil)$/.test(attr)) {
+            return attr;
+        }
+        return `"${attr}"`;
+    };
+
+    Export.prototype.getOpInvocation = function(op) {
+        var lines = [],
+            attrs,
+            refInits = [],
+            args;
+
+        attrs = '{' +
+            Object.keys(op.attributes).map(key => `${key}=${toAttrString(op.attributes[key])}`)
+            .join(',') +
+        '}';
+
+        lines.push(`local ${op.name}_attrs = ${attrs}`);
+        args = (op.inputValues || [])
+            .map(val => val instanceof Array ? `${val[0]}_results.${val[1]}` : val);
+
+        args.unshift(op.name + '_attrs');
+            
+        // Create the ref init functions
+        refInits = op.refs.map((code, index) => {
+            return [
+                `local function create_${op.refNames[index]}()`,
+                indent(code),
+                'end'
+            ].join('\n');
+        });
+        lines = lines.concat(refInits);
+        args = args.concat(op.refNames.map(name => `create_${name}()`));
+        args = args.join(', ');
+        lines.push(`local ${op.name}_results = ${op.basename}(${args})`);
+
+        return lines.join('\n');
+    };
+
+    Export.prototype.getOutputName = function(node) {
+        var basename = this.core.getAttribute(node, 'saveName');
+
+        return getUniqueName(basename, this._outputNames, true);
+    };
+
+    Export.prototype.getVariableName = function (/*node*/) {
+        var c = Object.keys(this.isInputOp).length;
+
+        if (c !== 1) {
+            return `input${c}`;
+        }
+
+        return 'input';
+    };
+
+    Export.prototype.registerNode = function (node) {
+        if (this.isMetaTypeOf(node, this.META.Operation)) {
+            return this.registerOperation(node);
+        } else if (this.isMetaTypeOf(node, this.META.Transporter)) {
+            return this.registerTransporter(node);
+        }
+    };
+
+    var getUniqueName = function(namebase, takenDict, unsafeAllowed) {
+        var name,
+            i = 2,
+            isUnsafe = function(name) {
+                return !unsafeAllowed && RESERVED.test(name);
+            };
+
+        if (!unsafeAllowed) {
+            namebase = namebase.replace(/[^A-Za-z\d]/g, '_');
+        }
+        name = namebase;
+        // Get a unique operation name
+        while (takenDict[name] || isUnsafe(name)) {
+            name = namebase + '_' + i;
+            i++;
+        }
+        takenDict[name] = true;
+
+        return name;
+    };
+
+    Export.prototype.registerOperation = function (node) {
+        var name = this.core.getAttribute(node, 'name'),
+            id = this.core.getPath(node),
+            base = this.core.getBase(node),
+            baseId = this.core.getPath(base),
+            baseName = this.core.getAttribute(base, 'name');
+
+        // If it is an Input/Output operation, assign it a variable name
+        if (baseName === CONSTANTS.OP.INPUT) {
+            this.isInputOp[id] = node;
+            name = this.getVariableName(node);
+        } else if (baseName === CONSTANTS.OP.OUTPUT) {
+            this.isOutputOp[id] = node;
+            name = this.getOutputName(node);
+        } else {
+            // get a unique operation instance name
+            name = getUniqueName(name, this._instanceNames);
+        }
+
+        this._nameFor[id] = name;
+
+        // get a unique operation base name
+        if (!this._fnNameFor[baseId]) {
+            name = this.core.getAttribute(base, 'name');
+            name = getUniqueName(name, this._opBaseNames);
+            this._fnNameFor[baseId] = name;
+        }
+
+        // For operations, register all output data node names by path
+        return this.core.loadChildren(node)
+            .then(cntrs => {
+                var outputs = cntrs.find(n => this.isMetaTypeOf(n, this.META.Outputs)),
+                    inputs = cntrs.find(n => this.isMetaTypeOf(n, this.META.Inputs));
+
+                return Q.all([inputs, outputs].map(cntr => this.core.loadChildren(cntr)));
+            })
+            .then(data => {
+                var inputs = data[0],
+                    outputs = data[1];
+
+                // Get the input type
+                outputs.forEach(output => {
+                    var dataId = this.core.getPath(output);
+
+                    name = this.core.getAttribute(output, 'name');
+                    this._dataNameFor[dataId] = name;
+
+                    this._portCache[dataId] = output;
+                });
+                inputs.forEach(input => 
+                    this._portCache[this.core.getPath(input)] = input
+                );
+
+                // Extra recording for input/output nodes in the pipeline
+                if (this.isInputOp[id]) {
+                    this.inputNode[id] = outputs[0];
+                } else if (this.isOutputOp[id]) {
+                    this.outputDataToOpId[this.core.getPath(inputs[0])] = id;
+                }
+            });
+    };
+
+    Export.prototype.registerTransporter = function (node) {
+        var outputData = this.core.getPointerPath(node, 'src'),
+            inputData = this.core.getPointerPath(node, 'dst'),
+            srcOpId = this.getOpIdFor(outputData),
+            dstOpId = this.getOpIdFor(inputData);
+
+        this._srcIdFor[inputData] = outputData;
+
+        // Store the next operation ids for the op id
+        if (!this._nextOps[srcOpId]) {
+            this._nextOps[srcOpId] = [];
+        }
+        this._nextOps[srcOpId].push(dstOpId);
+
+        // Increment the incoming counts for each dst op
+        this._incomingCnts[dstOpId] = this._incomingCnts[dstOpId] || 0;
+        this._incomingCnts[dstOpId]++;
+    };
+
+    Export.prototype.getOpIdFor = function (dataId) {
+        var ids = dataId.split('/'),
+            depth = ids.length;
+
+        ids.splice(this.activeNodeDepth - depth);
+        return ids.join('/');
+    };
+
+    // For each operation...
+    //   - unpack the inputs from prev ops
+    //   - add the attributes table (if used)
+    //     - check for '\<attributes\>' in code
+    //   - add the references
+    //     - generate the code
+    //     - replace the `return <thing>` w/ `<ref-name> = <thing>`
+    Export.prototype.createOperation = function (node) {
+        var id = this.core.getPath(node),
+            baseId = this.core.getPath(this.core.getBase(node)),
+            attrNames = this.core.getValidAttributeNames(node),
+            operation = {};
+
+        operation.name = this._nameFor[id];
+        operation.basename = this._fnNameFor[baseId];
+        operation.baseId = baseId;
+        operation.id = id;
+        operation.code = this.core.getAttribute(node, 'code');
+        operation.attributes = {};
+        for (var i = attrNames.length; i--;) {
+            if (!SKIP_ATTRS[attrNames[i]]) {
+                operation.attributes[attrNames[i]] = this.core.getAttribute(node, attrNames[i]);
+            }
+        }
+
+        // Get all the input names (and sources)
+        return this.core.loadChildren(node)
+            .then(containers => {
+                var inputs;
+
+                inputs = containers
+                    .find(cntr => this.isMetaTypeOf(cntr, this.META.Inputs));
+
+                this.logger.info(`${operation.name} has ${containers.length} cntrs`);
+                return this.core.loadChildren(inputs);
+            })
+            .then(data => {
+                // Get the input names and sources
+                var inputNames = data.map(d => this.core.getAttribute(d, 'name')),
+                    ids = data.map(d => this.core.getPath(d)),
+                    srcIds = ids.map(id => this._srcIdFor[id]);
+
+                operation.inputNames = inputNames || [];
+                operation.inputValues = inputNames.map((name, i) => {
+                    var id = srcIds[i],
+                        srcDataName = this._dataNameFor[id],
+                        srcOpId = this.getOpIdFor(id),
+                        srcOpName = this._nameFor[srcOpId];
+
+                    if (this.isInputOp[srcOpId]) {
+                        return this._nameFor[srcOpId];
+                    } else {
+                        return [srcOpName, srcDataName];
+                    }
+                });
+
+                return operation;
+
+            })
+            .then(operation => {
+
+                // For each reference, run the plugin and retrieve the generated code
+                operation.refNames = [];
+
+                if (!this.isInputOp[operation.id]) {
+                    operation.refNames = this.core.getPointerNames(node)
+                        .filter(name => name !== 'base');
+                }
+
+                var refs = operation.refNames
+                    .map(ref => [ref, this.core.getPointerPath(node, ref)]);
+
+                return Q.all(
+                    refs.map(pair => this.genPtrSnippet.apply(this, pair))
+                );
+            })
+            .then(codeFiles => {
+                operation.refs = codeFiles;
+                return operation;
+            });
+    };
+
+    Export.prototype.genPtrSnippet = function (ptrName, pId) {
+        return this.getPtrCodeHash(pId)
+            .then(hash => this.blobClient.getObjectAsString(hash));
+    };
+
+    Export.prototype.createHeader = function (title, length) {
+        var len;
+        title = ` ${title} `;
+        length = length || HEADER_LENGTH;
+
+        len = Math.max(
+            Math.floor((length - title.length)/2),
+            2
+        );
+
+        return [
+            '',
+            title,
+            ''
+        ].join(new Array(len+1).join('-')) + '\n';
+
+    };
+
+    Export.prototype.genOperationCode = function (operation) {
+        var header = this.createHeader(`"${operation.name}" Operation`),
+            codeParts = [],
+            body = [];
+
+        codeParts.push(header);
+        codeParts.push(`local ${operation.name}_results`);
+        codeParts.push('do');
+
+        if (operation.inputs.length) {
+            body.push(operation.inputs.join('\n'));
+        }
+
+        if (operation.refs.length) {
+            body.push(operation.refs.join('\n'));
+        }
+
+        body.push(operation.code);
+
+        codeParts.push(indent(body.join('\n')));
+        codeParts.push('end');
+        codeParts.push('');
+
+        operation.code = codeParts.join('\n');
+        return operation;
+    };
+
+    _.extend(Export.prototype, PtrCodeGen.prototype);
+
+    // Extra utilities for export types
+    Export.prototype.INIT_CLASSES_FN = '__init_classes';
+    Export.prototype.INIT_LAYERS_FN = '__init_layers';
+    Export.prototype.getAllDefinitions = function (sections) {
+        var code = [],
+            classes,
+            initClassFn,
+            initLayerFn;
+
+        classes = sections.orderedClasses
+            // Create fns from the classes
+            .map(name => this.indent(sections.classes[name])).join('\n');
+
+        initClassFn = [
+            `local function ${this.INIT_CLASSES_FN}()`,
+            this.indent(classes),
+            'end'
+        ].join('\n');
+
+        code = code.concat(initClassFn);
+
+        // wrap the layers in a function
+        initLayerFn = [
+            `local function ${this.INIT_LAYERS_FN}()`,
+            this.indent(_.values(sections.layers).join('\n\n')),
+            'end'
+        ].join('\n');
+        code = code.concat(initLayerFn);
+
+        // Add operation fn definitions
+        code = code.concat(_.values(sections.operations));
+        code = code.concat(_.values(sections.pipelines));
+
+        // define deserializers, serializers
+        code.push(sections.deserializers);
+        code.push(sections.serializers);
+
+        code.push(this.getDeepforgeObject());
+        code.push('deepforge.initialize()');
+
+        code.push(sections.serializeOutputsDef);
+        return code.join('\n\n');
+    };
+
+    Export.prototype.getDeepforgeObject = function (content) {
+        content = content || {};
+        content.initCode = content.initCode || `${this.INIT_CLASSES_FN}()\n${'   '}${this.INIT_LAYERS_FN}()`;
+        return DeepForgeTpl(content);
+    };
+
+    return Export;
+});

src/plugins/Export/deepforge.ejs

@@ -0,0 +1,47 @@
+-- Instantiate the deepforge object
+deepforge = {}
+
+function deepforge.initialize()
+    require 'nn'
+    require 'rnn'
+    <%= initCode %>
+end
+
+-- Graph support
+torch.class('deepforge.Graph')
+
+function deepforge.Graph:__init(name)
+    -- nop
+end
+
+torch.class('deepforge._Line')
+
+function deepforge._Line:__init(graphId, name, opts)
+   -- nop
+end
+
+function deepforge._Line:add(x, y)
+   -- nop
+end
+
+function deepforge.Graph:line(name, opts)
+    return deepforge._Line(self.id, name, opts)
+end
+
+-- Image support
+function deepforge.image(name, tensor)
+   -- nop
+end
+
+torch.class('deepforge.Image')
+function deepforge.Image:__init(name, tensor)
+   -- nop
+end
+
+function deepforge.Image:update(tensor)
+   -- nop
+end
+
+function deepforge.Image:title(name)
+   -- nop
+end

src/plugins/Export/format.js

@@ -0,0 +1,13 @@
+
+/* globals define*/
+// The supported export formats and metadata
+define([
+    './formats/cli/cli'
+], function(
+    Format0
+) {
+
+    return {
+        'Basic CLI': Format0
+    };
+});

src/plugins/Export/format.js.ejs

@@ -0,0 +1,21 @@
+<% // Add default format
+formats.unshift({ name: 'cli', main: 'cli.js', displayName: 'Basic CLI' })
+%>
+/* globals define*/
+// The supported export formats and metadata
+define([
+<%= formats.map(function(format) {
+    return '    \'./formats/' + format.name + '/' +
+        path.basename(format.main.replace(/\.js$/, '')) + '\''
+    })
+    .join(',\n') %>
+], function(
+<%= formats.map(function(f, index) { return '    Format' + index; }).join(',\n') %>
+) {
+
+    return {
+<%= formats.map(function(f, index) {
+        return '        \'' + f.displayName + '\': Format' + index;
+    }).join(',\n') %>
+    };
+});

src/plugins/Export/formats/cli/cli.js

@@ -0,0 +1,103 @@
+/*globals define*/
+// Simple torch cli for the given pipeline
+define([
+], function(
+) {
+
+    var TOBOOLEAN = 
+`local function toboolean(str)
+    if str == 'true' then
+        return true
+    elseif str == 'false' then
+        return false
+    end
+end`;
+
+    var CliExporter = {};
+
+    CliExporter.deserializersFromString = function(sections) {
+        var hasBool = false;
+
+        // Add serializers given cli string input
+        Object.keys(this.isInputOp).forEach(id => {
+            var node = this.inputNode[id],
+                base = this.core.getBase(node),
+                type = this.core.getAttribute(base, 'name'),
+                name = this._nameFor[id];
+
+            if (type === 'boolean') {
+                hasBool = true;
+                sections.deserializerFor[name] = 'toboolean';
+            } else if (type === 'number') {
+                sections.deserializerFor[name] = 'tonumber';
+            } else if (type === 'string') {
+                sections.deserializerFor[name] = 'tostring';
+            }
+        });
+
+        if (hasBool) {
+            sections.deserializers += '\n' + TOBOOLEAN;
+        }
+
+        return sections;
+    };
+
+    CliExporter.main = function (sections, staticInputs) {
+        var code = [];
+
+        // Update deserializers for cli input
+        this.deserializersFromString(sections);
+
+        // Define all the operations, pipelines, etc
+        // 'getAllDefinitions' is provided as part of the public api
+        code.push(this.getAllDefinitions(sections));
+
+        // Command line specific stuff
+        var files = {},
+            main,
+            args,
+            staticNames = staticInputs.map(input => input.name),
+            varDefs,
+            index = 1;
+
+        // Create some names for the inputs
+        args = sections.pipelineInputNames.map(name => `${sections.deserializerFor[name]}(${name})`);
+
+        main = `local outputs = ${sections.pipelineName}(${args.join(', ')})`;
+
+        // Grab the args from the cli
+        code.push(sections.pipelineInputNames.map((name, index) => {
+            return `local ${name} = arg[${index + 1}]`;
+        }).join('\n'));
+
+        // Add the hash for each of the static inputs and reference them
+        staticInputs.forEach(input => {
+            files[`res/${input.name}`] = input.hash;
+        });
+
+        varDefs = staticNames.map(name => {
+            return `local ${name} = './res/${name}'`;
+        });
+
+        // Grab the remaining args from the cli
+        varDefs = varDefs.concat(sections.pipelineInputNames.map(name => {
+            if (!staticNames.includes(name)) {
+                return `local ${name} = arg[${index++}]`;
+            }
+        }));
+
+        // Add the main fn
+        code.push(varDefs.join('\n'));
+        code.push(main);
+
+        // Save outputs to disk
+        code.push(sections.serializeOutputs);
+
+        files['init.lua'] = code.join('\n\n');
+
+        // if no extra assets, just return the main file
+        return staticInputs.length ? files : files['init.lua'];
+    };
+
+    return CliExporter;
+});

src/plugins/Export/metadata.json

@@ -1,7 +1,7 @@
 {
-  "id": "GenerateExecFile",
-  "name": "Generate Execution File",
-  "version": "0.1.0",
+  "id": "Export",
+  "name": "Export",
+  "version": "1.0.0",
   "description": "",
   "icon": {
     "class": "glyphicon glyphicon-cog",

src/plugins/Export/toboolean.lua

@@ -0,0 +1,7 @@
+local function toboolean(str)
+    if str == 'true' then
+        return true
+    elseif str == 'false' then
+        return false
+    end
+end

src/plugins/GenerateArchitecture/GenerateArchitecture.js

@@ -6,13 +6,15 @@ define([
     'SimpleNodes/Constants',
     'deepforge/layer-args',
     'deepforge/utils',
+    'deepforge/Constants',
     'underscore',
     'text!./metadata.json'
 ], function (
     PluginBase,
-    Constants,
+    SimpleNodeConstants,
     createLayerDict,
     utils,
+    Constants,
     _,
     metadata
 ) {
@@ -47,6 +49,11 @@ define([
         this.LayerDict = createLayerDict(this.core, this.META);
         this.uniqueId = 2;
         this.varnames = {net: true};
+        this.definitions = [
+            'require \'nn\'',
+            'require \'rnn\''
+        ];
+
         return PluginBase.prototype.main.apply(this, arguments);
     };
 
@@ -65,15 +72,10 @@ define([
     };
 
     GenerateArchitecture.prototype.createOutputFiles = function (tree) {
-        var layers = tree[Constants.CHILDREN],
+        var layers = tree[SimpleNodeConstants.CHILDREN],
             result = {},
             code = '';
 
-        this.definitions = [
-            'require \'nn\'',
-            'require \'rnn\''
-        ];
-
         // Add an index to each layer
         layers.forEach((l, index) => l[INDEX] = index);
 
@@ -125,12 +127,61 @@ define([
     };
 
     GenerateArchitecture.prototype.createLayer = function (layer) {
-        var args = this.createArgString(layer);
-        return `nn.${layer.name}${args}`;
+        var args = this.createArgString(layer),
+            def = `nn.${layer.name}${args}`,
+            type = layer.base.base.name,
+            memberIds,
+            node,
+            name,
+            children,
+            id;
+
+        // Check if it is a container and has the 'addLayers' set
+        // If so, it should sort them by their registry 'index' and add
+        // each nested architecture's code to the given container
+        if (type === 'Container') {
+            // Get the members of the 'addLayers' set
+            memberIds = {};
+            id = layer[SimpleNodeConstants.NODE_PATH];
+            node = this._nodeCache[id];
+            this.core.getMemberPaths(node, Constants.CONTAINED_LAYER_SET)
+                .forEach(id => memberIds[id] = true);
+
+            // Get the (sorted) children
+            children = layer[SimpleNodeConstants.CHILDREN]
+                .map(child => {  // get (child, index) tuples
+                    var index = null;
+
+                    id = child[SimpleNodeConstants.NODE_PATH];
+                    if (memberIds[id]) {
+                        index = this.core.getMemberRegistry(node,
+                            Constants.CONTAINED_LAYER_SET, id, Constants.CONTAINED_LAYER_INDEX);
+                    }
+                    return [child, index];
+                })
+                .filter(pair => pair[1] !== null)  // remove non-members
+                .sort((a, b) => a[1] < b[1] ? -1 : 1)  // sort by 'index'
+                .map(pair => pair[0]);
+
+
+            var addedLayerDefs = '',
+                firstLayer;
+
+            for (var i = 0; i < children.length; i++) {
+                id = children[i][SimpleNodeConstants.NODE_PATH];
+                // Get the children!
+                firstLayer = children[i][SimpleNodeConstants.CHILDREN][0];
+                name = this.getVarName(utils.abbr(layer.name + '_' + i));
+                addedLayerDefs += this.createSequential(firstLayer, name).code;
+                def += `:add(${name})`;
+            }
+            this.hoist(addedLayerDefs);
+        }
+        return def;
     };
 
     GenerateArchitecture.prototype.createSequential = function (layer, name) {
-        var next = layer[Constants.NEXT][0],
+        var next = layer[SimpleNodeConstants.NEXT][0],
             args,
             snippet,
             snippets,
@@ -142,7 +193,7 @@ define([
 
         while (layer) {
             // if there is only one successor, just add the given layer
-            if (layer[Constants.PREV].length > 1) {  // sequential layers are over
+            if (layer[SimpleNodeConstants.PREV].length > 1) {  // sequential layers are over
                 next = layer;  // the given layer will be added by the caller
                 break;
             } else {  // add the given layer
@@ -151,11 +202,11 @@ define([
 
             }
 
-            while (layer && layer[Constants.NEXT].length > 1) {  // concat/parallel
+            while (layer && layer[SimpleNodeConstants.NEXT].length > 1) {  // concat/parallel
                 // if there is a fork, recurse and add a concat layer
 
-                this.logger.debug(`detected fork of size ${layer[Constants.NEXT].length}`);
-                snippets = layer[Constants.NEXT].map(nlayer =>
+                this.logger.debug(`detected fork of size ${layer[SimpleNodeConstants.NEXT].length}`);
+                snippets = layer[SimpleNodeConstants.NEXT].map(nlayer =>
                     this.createSequential(nlayer, this.getVarName('net')));
                 code += '\n' + snippets.map(snippet => snippet.code).join('\n');
 
@@ -183,7 +234,7 @@ define([
                                 `concat_${layer[INDEX]}:add(${snippet.name})`)
                             .join('\n') + `\n\n${name}:add(concat_${layer[INDEX]})`;
                         
-                        next = layer[Constants.NEXT][0];
+                        next = layer[SimpleNodeConstants.NEXT][0];
                     } else {
                         next = null;  // no next layers
                     }
@@ -203,7 +254,7 @@ define([
             }
 
             layer = next;
-            next = layer && layer[Constants.NEXT][0];
+            next = layer && layer[SimpleNodeConstants.NEXT][0];
         }
 
         return {
@@ -218,14 +269,14 @@ define([
         var content = layer[arg];
 
         if (typeof content === 'object') {  // layer as arg
-            if (content[Constants.CHILDREN].length) {
+            if (content[SimpleNodeConstants.CHILDREN].length) {
                 // Generate the code for the children of layer[arg]
                 var name = this.getVarName(utils.abbr(arg)),
                     layers;
 
                 this.logger.debug(`Adding layer arg for ${arg} (${layer.name})`);
                 try {
-                    layers = this.genRawArchCode(layer[arg][Constants.CHILDREN], name);
+                    layers = this.genRawArchCode(layer[arg][SimpleNodeConstants.CHILDREN], name);
                 } catch (e) {
                     this.logger.error(`Layer arg creation failed: ${e}`);
                     return null;
@@ -244,7 +295,7 @@ define([
     GenerateArchitecture.prototype.createArgString = function (layer) {
         var setters = this.LayerDict[layer.name].setters,
             setterNames = Object.keys(this.LayerDict[layer.name].setters),
-            base = layer[Constants.BASE],
+            base = layer[SimpleNodeConstants.BASE],
             desc,
             fn,
             layerCode,

src/plugins/GenerateExecFile/GenerateExecFile.js

@@ -1,344 +0,0 @@
-/*globals define, _*/
-/*jshint node:true, browser:true*/
-
-/**
- * Generated by PluginGenerator 1.7.0 from webgme on Sat Jun 04 2016 18:01:54 GMT-0500 (CDT).
- * A plugin that inherits from the PluginBase. To see source code documentation about available
- * properties and methods visit %host%/docs/source/PluginBase.html.
- */
-
-define([
-    'text!./metadata.json',
-    'plugin/PluginBase',
-    'deepforge/plugin/PtrCodeGen',
-    'q'
-], function (
-    pluginMetadata,
-    PluginBase,
-    PtrCodeGen,
-    Q
-) {
-    'use strict';
-
-    pluginMetadata = JSON.parse(pluginMetadata);
-    var HEADER_LENGTH = 60;
-
-    /**
-     * Initializes a new instance of GenerateExecFile.
-     * @class
-     * @augments {PluginBase}
-     * @classdesc This class represents the plugin GenerateExecFile.
-     * @constructor
-     */
-    var GenerateExecFile = function () {
-        // Call base class' constructor.
-        PluginBase.call(this);
-        this.pluginMetadata = pluginMetadata;
-
-        this._srcIdFor = {};  // input path -> output data node path
-
-        this._nameFor = {};  // input path -> opname
-        this._dataNameFor = {};  
-        this._opNames = {};
-
-        // topo sort stuff
-        this._nextOps = {};
-        this._incomingCnts = {};
-
-        this._operations = {};
-        this.activeNodeId = null;
-        this.activeNodeDepth = null;
-    };
-
-    /**
-     * Metadata associated with the plugin. Contains id, name, version, description, icon, configStructue etc.
-     * This is also available at the instance at this.pluginMetadata.
-     * @type {object}
-     */
-    GenerateExecFile.metadata = pluginMetadata;
-
-    // Prototypical inheritance from PluginBase.
-    GenerateExecFile.prototype = Object.create(PluginBase.prototype);
-    GenerateExecFile.prototype.constructor = GenerateExecFile;
-
-    /**
-     * Main function for the plugin to execute. This will perform the execution.
-     * Notes:
-     * - Always log with the provided logger.[error,warning,info,debug].
-     * - Do NOT put any user interaction logic UI, etc. inside this method.
-     * - callback always has to be called even if error happened.
-     *
-     * @param {function(string, plugin.PluginResult)} callback - the result callback
-     */
-    GenerateExecFile.prototype.main = function (callback) {
-        // Get all the children and call generate exec file
-        this.activeNodeId = this.core.getPath(this.activeNode);
-        this.activeNodeDepth = this.activeNodeId.split('/').length + 1;
-
-        if (this.isMetaTypeOf(this.activeNode, this.META.Execution)) {
-            this.activeNodeDepth++;
-        }
-
-        return this.core.loadChildren(this.activeNode)
-            .then(nodes => this.createExecFile(nodes))
-            .then(code => this.blobClient.putFile('init.lua', code))
-            .then(hash => {
-                this.result.addArtifact(hash);
-                this.result.setSuccess(true);
-                callback(null, this.result);
-            })
-            .fail(err => callback(err));
-    };
-
-    GenerateExecFile.prototype.createExecFile = function (children) {
-        // Convert opNodes' jobs to the nested operations
-        var opNodes,
-            nodes;
-
-        return this.unpackJobs(children)
-            .then(_nodes => {
-                nodes = _nodes;
-                opNodes = nodes
-                    .filter(node => this.isMetaTypeOf(node, this.META.Operation));
-                return Q.all(nodes.map(node => this.registerNameAndData(node)));
-            })
-            .then(() => Q.all(opNodes.map(node => this.createOperation(node))))
-            .then(operations => {
-                var nextIds = opNodes.map(n => this.core.getPath(n))
-                        .filter(id => !this._incomingCnts[id]);
-
-                operations.forEach(op => this._operations[op.id] = op);
-
-                // Toposort and concat!
-                return this.combineOpNodes(nextIds);
-            })
-            .fail(err => this.logger.error(err));
-    };
-
-    GenerateExecFile.prototype.unpackJobs = function (nodes) {
-        return Q.all(
-            nodes.map(node => {
-                if (!this.isMetaTypeOf(node, this.META.Job)) {
-                    return node;
-                }
-                return this.core.loadChildren(node)
-                    .then(children =>
-                        children.find(c => this.isMetaTypeOf(c, this.META.Operation))
-                    );
-            })
-        );
-    };
-
-    GenerateExecFile.prototype.combineOpNodes = function (opIds) {
-        var nextIds = [],
-            dstIds,
-            code,
-            id;
-
-        // Combine all nodes with incoming cnts of 0
-        code = opIds.map(id => this._operations[id].code).join('\n');
-
-        // Decrement all next ops
-        dstIds = opIds.map(id => this._nextOps[id])
-            .reduce((l1, l2) => l1.concat(l2), []);
-        for (var i = dstIds.length; i--;) {
-            id = dstIds[i];
-            if (--this._incomingCnts[id] === 0) {
-                nextIds.push(id);
-            }
-        }
-
-        // append
-        return [
-            code,
-            nextIds.length ? this.combineOpNodes(nextIds) : ''
-        ].join('\n');
-    };
-
-    GenerateExecFile.prototype.registerNameAndData = function (node) {
-        var name = this.core.getAttribute(node, 'name'),
-            id = this.core.getPath(node),
-            basename = name,
-            i = 2;
-
-        if (this.isMetaTypeOf(node, this.META.Operation)) {
-
-            // Get a unique operation name
-            while (this._opNames[name]) {
-                name = basename + '_' + i;
-                i++;
-            }
-
-            // register the unique name
-            this._opNames[name] = true;
-            this._nameFor[id] = name;
-
-            // For operations, register all output data node names by path
-            return this.core.loadChildren(node)
-                .then(cntrs => {
-                    var cntr = cntrs.find(n => this.isMetaTypeOf(n, this.META.Outputs));
-                    return this.core.loadChildren(cntr);
-                })
-                .then(outputs => {
-                    outputs.forEach(output => {
-                        var dataId = this.core.getPath(output);
-
-                        name = this.core.getAttribute(output, 'name');
-                        this._dataNameFor[dataId] = name;
-                    });
-                });
-
-        // For each input data node, register the associated output id
-        } else if (this.isMetaTypeOf(node, this.META.Transporter)) {
-            var outputData = this.core.getPointerPath(node, 'src'),
-                inputData = this.core.getPointerPath(node, 'dst'),
-                srcOpId = this.getOpIdFor(outputData),
-                dstOpId = this.getOpIdFor(inputData);
-
-            this._srcIdFor[inputData] = outputData;
-
-            // Store the next operation ids for the op id
-            if (!this._nextOps[srcOpId]) {
-                this._nextOps[srcOpId] = [];
-            }
-            this._nextOps[srcOpId].push(dstOpId);
-
-            // Increment the incoming counts for each dst op
-            this._incomingCnts[dstOpId] = this._incomingCnts[dstOpId] || 0;
-            this._incomingCnts[dstOpId]++;
-        }
-    };
-
-    GenerateExecFile.prototype.getOpIdFor = function (dataId) {
-        var ids = dataId.split('/'),
-            depth = ids.length;
-
-        ids.splice(this.activeNodeDepth - depth);
-        return ids.join('/');
-    };
-
-    // For each operation...
-    //   - unpack the inputs from prev ops
-    //   - add the attributes table (if used)
-    //     - check for '\<attributes\>' in code
-    //   - add the references
-    //     - generate the code
-    //     - replace the `return <thing>` w/ `<ref-name> = <thing>`
-    GenerateExecFile.prototype.createOperation = function (node) {
-        var id = this.core.getPath(node),
-            operation = {};
-
-        operation.name = this._nameFor[id];
-        operation.id = id;
-        operation.code = this.core.getAttribute(node, 'code');
-
-        // Update the 'code' attribute
-        // Change the last return statement to assign the results to a table
-        operation.code = this.assignResultToVar(operation.code,
-            `${operation.name}_results`);
-
-        // Get all the input names (and sources)
-        return this.core.loadChildren(node)
-            .then(containers => {
-                var inputs;
-
-                inputs = containers
-                    .find(cntr => this.isMetaTypeOf(cntr, this.META.Inputs));
-
-                this.logger.info(`${name} has ${containers.length} cntrs`);
-                return this.core.loadChildren(inputs);
-            })
-            .then(data => {
-                // Get the input names and sources
-                var inputNames = data.map(d => this.core.getAttribute(d, 'name')),
-                    ids = data.map(d => this.core.getPath(d)),
-                    srcIds = ids.map(id => this._srcIdFor[id]);
-
-                operation.inputs = inputNames.map((name, i) => {
-                    var id = srcIds[i],
-                        srcDataName = this._dataNameFor[id],
-                        srcOpId = this.getOpIdFor(id),
-                        srcOpName = this._nameFor[srcOpId];
-
-                    return `local ${name} = ${srcOpName}_results.${srcDataName}`;
-                });
-
-                return operation;
-
-            })
-            .then(operation => {
-
-                // For each reference, run the plugin and retrieve the generated code
-                operation.refNames = this.core.getPointerNames(node)
-                    .filter(name => name !== 'base');
-
-                var refs = operation.refNames
-                    .map(ref => [ref, this.core.getPointerPath(node, ref)]);
-
-                return Q.all(
-                    refs.map(pair => this.genPtrSnippet.apply(this, pair))
-                );
-            })
-            .then(codeFiles => {
-                operation.refs = codeFiles;
-                this.genOperationCode(operation);
-                return operation;
-            });
-    };
-
-    GenerateExecFile.prototype.genPtrSnippet = function (ptrName, pId) {
-        return this.getPtrCodeHash(pId)
-            .then(hash => this.blobClient.getObjectAsString(hash))
-            .then(code => this.createHeader(`creating ${ptrName}`, 40) + '\n' +
-                this.assignResultToVar(code, ptrName));
-    };
-
-    GenerateExecFile.prototype.createHeader = function (title, length) {
-        var len;
-        title = ` ${title} `;
-        length = length || HEADER_LENGTH;
-
-        len = Math.max(
-            Math.floor((length - title.length)/2),
-            2
-        );
-
-        return [
-            '',
-            title,
-            ''
-        ].join(new Array(len+1).join('-')) + '\n';
-
-    };
-
-    GenerateExecFile.prototype.genOperationCode = function (operation) {
-        var header = this.createHeader(`"${operation.name}" Operation`),
-            codeParts = [];
-
-        codeParts.push(header);
-
-        if (operation.inputs.length) {
-            codeParts.push(operation.inputs.join('\n'));
-        }
-
-        if (operation.refs.length) {
-            codeParts.push(operation.refs.join('\n'));
-        }
-
-        codeParts.push(operation.code);
-        codeParts.push('');
-        operation.code = codeParts.join('\n');
-        return operation;
-    };
-
-    GenerateExecFile.prototype.assignResultToVar = function (code, name) {
-        var i = code.lastIndexOf('return');
-        return code.substring(0, i) +
-            code.substring(i)
-                .replace('return', `local ${name} = `);
-    };
-
-    _.extend(GenerateExecFile.prototype, PtrCodeGen.prototype);
-
-    return GenerateExecFile;
-});

src/plugins/ImportArtifact/ImportArtifact.js

@@ -86,6 +86,7 @@ define([
                 });
 
                 this.core.setAttribute(dataNode, 'data', hash);
+                this.core.setAttribute(dataNode, 'createdAt', Date.now());
                 baseName = this.core.getAttribute(baseType, 'name');
 
                 var getName;

src/plugins/ImportTorch/nn.js

@@ -3,10 +3,12 @@
 define([
     'deepforge/layer-args',
     'common/util/assert',
+    'deepforge/Constants',
     'deepforge/lua'
 ], function(
     createLayerDict,
     assert,
+    Constants,
     lua
 ) {
     'use strict';
@@ -41,6 +43,20 @@ define([
             return '{' + strings.join(', ') + '}';
         };
 
+        var getAttributeString = function(value, layerType) {
+            if (value instanceof lua.types.LuaTable) {
+                if (value.get('_node')) {
+                    throw Error(`Detected unsupported varargs (composed of layers) for ${layerType}`);
+                }
+                return stringify(value);
+            } else if ((typeof value) === 'object') {
+                // special lua.js object
+                value = value.valueOf();
+            }
+
+            return value;
+        };
+
         var allConnectedTo = function(current) {
             var connectedIds = {},
                 node,
@@ -79,6 +95,7 @@ define([
                 connsFrom[id] = [];
             }
             connsFrom[id].push(conn, dst);
+            return conn;
         };
 
         // nn drawing library
@@ -104,7 +121,8 @@ define([
                 cntr,
                 layer,
                 cntrName,
-                value;
+                value,
+                i;
 
             if (this._cachedNode) {
                 // only generate a single node for each layer
@@ -117,7 +135,17 @@ define([
                 parent: parent
             });
 
-            for (var i = this._attrs.length; i--;) {
+            // merge all the last arguments into a single one (ie, assume the last
+            // attribute is varargs
+            if (this._attrs.length < this._values.length) {
+                i = this._attrs.length;
+                value = this._values.splice(i-1)
+                    .map(val => getAttributeString(val, this._base)).join(', ');
+                this._values.push(value);
+            }
+
+            // Add the attributes to the layer
+            for (i = this._attrs.length; i--;) {
                 name = this._attrs[i].name;
                 value = this._values[i];
 
@@ -169,56 +197,131 @@ define([
             return self;
         };
 
-        // Each container will have `inputs` and `outputs`
+        Layer.prototype._getAllNodes = function() {
+            return [this._node()];
+        };
+
         var Container = function() {
+            Layer.apply(this, arguments);
+            this._nestedIndex = 0;
+        };
+
+        Container.prototype = Object.create(Layer.prototype);
+
+        Container.prototype.add = function(self, tlayer) {
+            var layer = tlayer.get('_node'),
+                container = this._node(),
+                children,
+                arch;
+
+            // Add a nested 'Architecture' node
+            arch = core.createNode({
+                parent: container,
+                base: META.Architecture
+            });
+
+            // Add this node to the 'addLayers' set
+            core.addMember(container, Constants.CONTAINED_LAYER_SET, arch);
+            // Assign it an appropriate 'index' value
+            core.setMemberRegistry(
+                container,
+                Constants.CONTAINED_LAYER_SET,
+                core.getPath(arch),
+                Constants.CONTAINED_LAYER_INDEX,
+                this._nestedIndex++
+            );
+
+            // Move the added node(s)/conns to this architecture node
+            children = layer._getAllNodes();
+            for (var i = children.length; i--;) {
+                core.moveNode(children[i], arch);
+            }
+            layer._parent = arch;
+            return self;
+        };
+
+        // Implicit Containers are sequential and concat containers;
+        // these containers are visually implied in deepforge (although
+        // they are explicitly defined in torch)
+        var ImplicitContainer = function() {
             // inputs and outputs are webgme nodes
             this._inputs = [];
             this._outputs = [];
+            this._children = [];
+            this._connections = [];
         };
 
-        Container.prototype.add = function() {
+        // Implicit containers will have to record their 'children'.
+        // When an implicit container is added to an actual container,
+        // the container will set it's '_parent' value. If any additional
+        // layers are added to the implicit container after, they will
+        // need to be moved to the parent of the implicit container
+        ImplicitContainer.prototype.add = function() {
             logger.error('Add is not overridden!');
         };
 
-        var Sequential = function(/*attrs, args*/) {
-            Container.call(this);
+        ImplicitContainer.prototype._getAllNodes = function() {
+            var nodes = this._children.map(layer => layer._getAllNodes())
+                .reduce((l1, l2) => l1.concat(l2), []);
+
+            return this._connections.concat(nodes);
         };
 
-        Sequential.prototype = new Container();
+        var Sequential = function(/*attrs, args*/) {
+            ImplicitContainer.call(this);
+        };
+
+        Sequential.prototype = new ImplicitContainer();
 
         Sequential.prototype.add = function(self, tlayer) {
             var layer = tlayer.get('_node'),
-                nodes = layer._inputs;
+                nodes = layer._inputs,
+                connections = [];
 
             // If this._inputs is empty, add the layer to the inputs list
             if (this._inputs.length === 0) {  // first node
                 this._inputs = this._inputs.concat(nodes);
             } else {
                 // connect all inputs of the added node to the current outputs
+                // add the connection to the list of allNodes
                 this._outputs.forEach(src =>
-                    nodes.forEach(dst => connect(src, dst))
+                    nodes.forEach(dst => connections.push(connect(src, dst)))
                 );
             }
             this._outputs = layer._outputs;
+            this._children.push(layer);
+            this._connections = this._connections.concat(connections);
+
+            // If _parent is set, move the nodes and connection to the _parent node
+            if (this._parent) {
+                nodes = layer._getAllNodes().concat(connections);
+                for (var i = nodes.length; i--;) {
+                    core.moveNode(nodes[i], this._parent);
+                }
+            }
             return self;
         };
 
         var Concat = function(attrs, args) {
-            Container.call(this);
+            ImplicitContainer.call(this);
 
             // Create a concat node and add it to this._outputs
             var concat = new Layer('Concat', attrs, args);
             this._outputs.push(concat._node());
+            this._children.push(concat);
         };
 
-        Concat.prototype = new Container();
+        Concat.prototype = new ImplicitContainer();
 
         Concat.prototype.add = function(self, tlayer) {
             // Connect the tlayer outputs to this._outputs
             var layer = tlayer.get('_node'),
-                concatLayer = this._outputs[0];
+                concatLayer = this._outputs[0],
+                connections = [],
+                nodes;
 
-            layer._outputs.forEach(output => connect(output, concatLayer));
+            layer._outputs.forEach(output =>
+                connections.push(connect(output, concatLayer)));
 
             // Connect the incomingly connected node to tlayer
             // TODO: This might not work if adding layers after this container is
@@ -226,14 +329,23 @@ define([
 
             // Add the layer's inputs to the inputs
             this._inputs = this._inputs.concat(layer._inputs);
+            this._children.push(layer);
+            this._connections = this._connections.concat(connections);
+            if (this._parent) {
+                nodes = layer._getAllNodes().concat(connections);
+                for (var i = nodes.length; i--;) {
+                    core.moveNode(nodes[i], this._parent);
+                }
+            }
             return self;
         };
 
         // Special layers (with special functions - like 'add')
-        var LAYERS = {
-            Concat: Concat,
-            Sequential: Sequential
-        };
+        var CONTAINERS,
+            LAYERS = {
+                Concat: Concat,
+                Sequential: Sequential
+            };
 
         var getValue = function(txt) {
             if (txt === 'true') {
@@ -286,6 +398,9 @@ define([
 
             if (LAYERS[type]) {
                 node = new LAYERS[type](args, attrs);
+            } else if (CONTAINERS[type]) {
+                node = new Container(type, args, attrs);
+                res.set('add', node.add.bind(node)); // add the 'add' method
             } else {  // Call generic Layer with type name
                 node = new Layer(type, args, attrs);
             }
@@ -327,11 +442,23 @@ define([
                 return;
             }
 
-            // TODO: Create the nn object
+            // Mocking the nn layers (as defined in the metamodel)
             var nn = lua.newContext()._G,
-                names = Object.keys(LayerDict);
+                names = Object.keys(LayerDict),
+                base,
+                baseName;
+
+            // For each layer, check the name of the base type. If it is 'Container',
+            // then it should be added to the CONTAINERS dictionary. This will change how
+            // it is handled in 'CreateLayer'
+            CONTAINERS = {};
 
             for (var i = names.length; i--;) {
+                base = core.getBase(META[names[i]]);
+                baseName = core.getAttribute(base, 'name');
+                if (baseName === 'Container') {
+                    CONTAINERS[names[i]] = true;
+                }
                 nn.set(names[i], CreateLayer.bind(null, names[i]));
             }
 

src/plugins/UpdateLibrarySeed/UpdateLibrarySeed.js

@@ -84,19 +84,27 @@ define([
             .fail(err => callback(err, this.result));
     };
 
+    UpdateLibrarySeed.prototype.bumpVersion = function (rawVersion, releaseType) {
+        var vnames = ['major', 'minor', 'patch'],
+            bumpIndex = vnames.indexOf(releaseType),
+            version = rawVersion.split('.').map(num => parseInt(num));
+
+        version[bumpIndex]++;
+        // zero out the smaller numbers
+        while (++bumpIndex < version.length) {
+            version[bumpIndex] = 0;
+        }
+        return version.join('.');
+    };
+
     UpdateLibrarySeed.prototype.getLibraryVersion = function () {
         var version,
             config = this.getCurrentConfig(),
-            vnames = ['major', 'minor', 'patch'],
-            bumpIndex,
             newVersion;
 
-        version = (this.core.getAttribute(this.rootNode, 'version') || '0.0.0')
-            .split('.').map(num => parseInt(num));
-        bumpIndex = vnames.indexOf(config.releaseType);
-        version[bumpIndex]++;
+        version = (this.core.getAttribute(this.rootNode, 'version') || '0.0.0');
+        newVersion = this.bumpVersion(version, config.releaseType);
 
-        newVersion = version.join('.');
         this.core.setAttribute(this.rootNode, 'version', newVersion);
         return this.save(`Bumped version to ${newVersion}`).then(() => newVersion);
     };
@@ -116,6 +124,10 @@ define([
         this.logger.info(`Updating ${seedName} seed`);
         job.on('error', _err => {
             err = _err;
+            if (err.code === 'ENOENT') {
+                return deferred.reject('"webgme" command not found. Is webgme-cli installed on the server?');
+            }
+            return deferred.reject(err);
         });
 
         job.on('exit', code => {

src/plugins/ValidateArchitecture/ValidateArchitecture.js

@@ -0,0 +1,194 @@
+/*globals define*/
+/*jshint node:true, browser:true*/
+
+define([
+    'plugin/GenerateArchitecture/GenerateArchitecture/GenerateArchitecture',
+    'SimpleNodes/Constants',
+    'text!./metadata.json',
+    'q',
+    'fs',
+    'path',
+    'child_process',
+    'rimraf'
+], function (
+    PluginBase,
+    SimpleNodeConstants,
+    pluginMetadata,
+    Q,
+    fs,
+    path,
+    childProcess,
+    rm_rf
+) {
+    'use strict';
+
+    pluginMetadata = JSON.parse(pluginMetadata);
+
+    /**
+     * Initializes a new instance of ValidateArchitecture.
+     * @class
+     * @augments {PluginBase}
+     * @classdesc This class represents the plugin ValidateArchitecture.
+     * @constructor
+     */
+    var TMP_DIR = '/tmp',
+        spawn = childProcess.spawn,
+        GET_ARG_INDEX = /argument #([0-9]+) to/,
+        TORCH_INSTALLED = true;
+
+    var ValidateArchitecture = function () {
+        // Call base class' constructor.
+        PluginBase.call(this);
+        this.pluginMetadata = pluginMetadata;
+    };
+
+    /**
+     * Metadata associated with the plugin. Contains id, name, version, description, icon, configStructue etc.
+     * This is also available at the instance at this.pluginMetadata.
+     * @type {object}
+     */
+    ValidateArchitecture.metadata = pluginMetadata;
+
+    // Prototypical inheritance from PluginBase.
+    ValidateArchitecture.prototype = Object.create(PluginBase.prototype);
+    ValidateArchitecture.prototype.constructor = ValidateArchitecture;
+
+    ValidateArchitecture.prototype.main = function (callback) {
+        var name = this.core.getAttribute(this.activeNode, 'name');
+
+        this._callback = callback;
+        // make the tmp dir
+        this._tmpFileId = path.join(TMP_DIR, `${name}_${Date.now()}`);
+        fs.mkdir(this._tmpFileId, err => {
+            if (err) throw err;
+            return PluginBase.prototype.main.call(this, callback);
+        });
+    };
+
+    ValidateArchitecture.prototype.createOutputFiles = function (tree) {
+        var layers = tree[SimpleNodeConstants.CHILDREN],
+            tests = [],
+            id;
+
+        if (!TORCH_INSTALLED) {
+            return this.validationFinished();
+        }
+
+        // Generate code for each layer
+        this.layerName = {};
+        for (var i = layers.length; i--;) {
+            id = layers[i][SimpleNodeConstants.NODE_PATH];
+            this.layerName[id] = layers[i].name;
+            tests.push([id, this.createLayerTestCode(layers[i])]);
+        }
+
+        // Run each code snippet
+        this.validateLayers(tests)
+            .then(errors => this.validationFinished(errors))
+            .fail(err => this.logger.error(`validation failed: ${err}`));
+    };
+
+    ValidateArchitecture.prototype.validationFinished = function (errors) {
+        if (!TORCH_INSTALLED) {
+            this.logger.warn('Torch is not installed. Architecture validation is not supported.');
+        } else {
+            this.logger.info(`found ${errors.length} validation errors`);
+        }
+
+        this.createMessage(null, {
+            errors: TORCH_INSTALLED ? errors : null
+        });
+        this.result.setSuccess(true);
+        this._callback(null, this.result);
+    };
+
+    ValidateArchitecture.prototype.createLayerTestCode = function (layer) {
+        var customLayerDefs = this.genLayerDefinitions([layer]);
+
+        return this.definitions.concat([
+            customLayerDefs,
+            this.createLayer(layer)
+        ]).join('\n');
+    };
+
+    ValidateArchitecture.prototype.validateLayers = function (layerTests) {
+        return Q.all(layerTests.map(layer => this.validateLayer(layer[0], layer[1])))
+            .then(results => Q.nfcall(rm_rf, this._tmpFileId)
+                .then(() => results.filter(result => !!result))
+            );
+    };
+
+    ValidateArchitecture.prototype.validateLayer = function (id, code) {
+        var deferred = Q.defer(),
+            tmpPath = path.join(this._tmpFileId, id.replace(/[^a-zA-Z\d]+/g, '_'));
+
+        if (!TORCH_INSTALLED) {
+            deferred.resolve(null);
+        } else {
+            // Write to a temp file
+            fs.writeFile(tmpPath, code, err => {
+                var job,
+                    stderr = '',
+                    stdout = '';
+
+                if (err) {
+                    return deferred.reject(`Could not create tmp file at ${tmpPath}: ${err}`);
+                }
+                // Run the file
+                job = spawn('th', [tmpPath]);
+                job.stderr.on('data', data => stderr += data.toString());
+                job.stdout.on('data', data => stdout += data.toString());
+                job.on('error', err => {
+                    if (err.code === 'ENOENT') {
+                        TORCH_INSTALLED = false;
+                    }
+                });
+                job.on('close', code => {
+                    if (code === 0) {
+                        deferred.resolve(null);
+                    } else {
+                        // If it errored, clean the error and return it
+                        deferred.resolve(this.parseError(id, stderr));
+                    }
+                });
+            });
+        }
+
+        return deferred.promise;
+    };
+
+    ValidateArchitecture.prototype.parseError = function (id, stderr) {
+        var msg = stderr
+            .split('\n').shift()  // first line
+            .replace(/^[^:]*: /, '')  // remove the file path
+            .replace(/ at [^ ]*\)/, ')')  // remove last line number
+            .replace(/ to '\?'/, '');  // remove unknown symbol
+
+        // convert 'bad argument #[num]' to the argument name
+        if (msg.indexOf('bad argument') === 0) {
+            var layerName = this.layerName[id],
+                args = this.LayerDict[layerName].args,
+                argIndex = +(stderr.match(GET_ARG_INDEX)[1]),
+                argName = args[argIndex-1].name;
+
+            // FIXME: This is not the correct index...
+            // This is the index for the incorrect argument passed to the
+            // tensor...
+            msg = msg.replace(`#${argIndex}`, `"${argName}"`);
+        }
+
+        return {
+            id: id,
+            msg: msg
+        };
+    };
+
+    ValidateArchitecture.prototype._saveOutput = function () {};
+
+    // for testing
+    ValidateArchitecture.prototype.setTorchInstalled = function (value) {
+        TORCH_INSTALLED = !!value;
+    };
+
+    return ValidateArchitecture;
+});

src/plugins/ValidateArchitecture/metadata.json

@@ -0,0 +1,14 @@
+{
+  "id": "ValidateArchitecture",
+  "name": "ValidateArchitecture",
+  "version": "0.1.0",
+  "description": "",
+  "icon": {
+    "class": "glyphicon glyphicon-cog",
+    "src": ""
+  },
+  "disableServerSideExecution": false,
+  "disableBrowserSideExecution": true,
+  "writeAccessRequired": false,
+  "configStructure": []
+}

src/routers/ExecPulse/ExecPulse.js

@@ -0,0 +1,131 @@
+/*jshint node:true*/
+
+// This is a REST endpoint keeping track of the heartbeats of each execution. This
+// allows detection of "disconnected" executions (enabling the reconnection of the
+// executions - issue #821)
+'use strict';
+
+var express = require('express'),
+    MONGO_COLLECTION = 'ExecPulse',
+    CONSTANTS = require('../../common/Constants').PULSE,
+    mongo,
+    storage,
+    router = express.Router();
+
+/**
+ * Called when the server is created but before it starts to listening to incoming requests.
+ * N.B. gmeAuth, safeStorage and workerManager are not ready to use until the start function is called.
+ * (However inside an incoming request they are all ensured to have been initialized.)
+ *
+ * @param {object} middlewareOpts - Passed by the webgme server.
+ * @param {GmeConfig} middlewareOpts.gmeConfig - GME config parameters.
+ * @param {GmeLogger} middlewareOpts.logger - logger
+ * @param {function} middlewareOpts.ensureAuthenticated - Ensures the user is authenticated.
+ * @param {function} middlewareOpts.getUserId - If authenticated retrieves the userId from the request.
+ * @param {object} middlewareOpts.gmeAuth - Authorization module.
+ * @param {object} middlewareOpts.safeStorage - Accesses the storage and emits events (PROJECT_CREATED, COMMIT..).
+ * @param {object} middlewareOpts.workerManager - Spawns and keeps track of "worker" sub-processes.
+ */
+function initialize(middlewareOpts) {
+    var logger = middlewareOpts.logger.fork('ExecPulse'),
+        ensureAuthenticated = middlewareOpts.ensureAuthenticated,
+        STALE_THRESHOLD = 7500;
+
+    storage = require('../storage')(logger, middlewareOpts.gmeConfig);
+    logger.debug('initializing ...');
+
+    // Ensure authenticated can be used only after this rule.
+    router.use('*', function (req, res, next) {
+        res.setHeader('X-WebGME-Media-Type', 'webgme.v1');
+        next();
+    });
+
+    // Use ensureAuthenticated if the routes require authentication. (Can be set explicitly for each route.)
+    router.use('*', ensureAuthenticated);
+
+    router.get('/', function (req, res) {
+        mongo.find().toArray((err, all) => {
+            if (err) {
+                return res.status(500).send(err);
+            }
+            res.json(all.map(entry => {
+                delete entry._id;
+                return entry;
+            }));
+        });
+    });
+
+    router.get('/:hash', function (req, res) {
+        // Check if the given job is alive (has a valid heartbeat).
+        // If the data doesn't exist, then it is considered alive
+        if (!req.params.hash) {
+            return res.status(400).send('Missing hash');
+        }
+
+        logger.debug('getting pulse of ', req.params.hash);
+        mongo.findOne({hash: req.params.hash})
+            .then(job => {
+                var current = Date.now(),
+                    result = CONSTANTS.DOESNT_EXIST;
+
+                if (job) {
+                    result = (current - job.timestamp) < STALE_THRESHOLD ?
+                        CONSTANTS.ALIVE : CONSTANTS.DEAD;
+                }
+                return res.status(200).send(result.toString());
+            });
+    });
+
+    router.post('/:hash', function (req, res) {
+        var timestamp = Date.now(),
+            job = {
+                hash: req.params.hash,
+                timestamp: timestamp
+            };
+
+        // Validate the input
+        logger.debug('Received heartbeat for ', job.hash);
+        if (!job.hash) {
+            return res.status(400).send('Missing hash');
+        }
+
+        // Delete the given job from the database
+        mongo.update({hash: job.hash}, job, {upsert: true})
+            .then(() => res.sendStatus(201));
+    });
+
+    router.delete('/:hash', function (req, res) {
+        // Delete the given job from the database
+        return mongo.findOneAndDelete({hash: req.params.hash})
+            .then(() => res.sendStatus(204));
+    });
+
+    logger.debug('ready');
+}
+
+/**
+ * Called before the server starts listening.
+ * @param {function} callback
+ */
+function start(callback) {
+    storage.then(db => {
+        mongo = db.collection(MONGO_COLLECTION);
+        callback();
+    });
+}
+
+/**
+ * Called after the server stopped listening.
+ * @param {function} callback
+ */
+function stop(callback) {
+    callback();
+}
+
+
+module.exports = {
+    initialize: initialize,
+    router: router,
+    start: start,
+    stop: stop
+};

src/routers/JobLogsAPI/JobLogManager.js

@@ -12,7 +12,7 @@ var JobLogManager = function(logger, config) {
 };
 
 JobLogManager.prototype._getFilePath = function(jInfo) {
-    this.logger.info(`getting file path for ${jInfo.job} in ${jInfo.project} on ${jInfo.branch}`);
+    this.logger.debug(`getting file path for ${jInfo.job} in ${jInfo.project} on ${jInfo.branch}`);
     var jobId = jInfo.job.replace(/\//g, '_'),
         filename = `${jobId}.txt`;
 
@@ -71,7 +71,7 @@ JobLogManager.prototype.migrate = function(migrationInfo, jobIds) {
     }
 
     // Copy the job files and evaluate each of the finish functions
-    this.logger.info('migrating from ' + migrationInfo.srcBranch + ' to '+ migrationInfo.dstBranch);
+    this.logger.debug('migrating from ' + migrationInfo.srcBranch + ' to '+ migrationInfo.dstBranch);
     return Q.all(jobIds.map(jobId => {
         src = this._getFilePath({
             project: migrationInfo.project,
@@ -112,7 +112,7 @@ JobLogManager.prototype.appendTo = function(jobInfo, logs) {
         branchDirname = path.dirname(filename),
         projDirname = path.dirname(branchDirname);
 
-    this.logger.info(`Appending content to ${filename}`);
+    this.logger.debug(`Appending content to ${filename}`);
     // Make directory if needed
     return this.mkdirIfNeeded(this.rootDir)
         .then(() => this.mkdirIfNeeded(projDirname))
@@ -123,7 +123,7 @@ JobLogManager.prototype.appendTo = function(jobInfo, logs) {
 JobLogManager.prototype.getLog = function(jobInfo) {
     var filename = this._getFilePath(jobInfo);
 
-    this.logger.info(`Getting log content to ${filename}`);
+    this.logger.info(`Getting log content from ${filename}`);
     return this.exists(jobInfo)
         .then(exists => {
             if (exists) {
@@ -142,7 +142,7 @@ JobLogManager.prototype.delete = function(jobInfo) {
                 this.logger.debug(`Removing file ${filename}`);
                 return Q.nfcall(fs.unlink, filename);
             }
-            this.logger.info(`${filename} doesn't exist. No need to delete...`);
+            this.logger.debug(`${filename} doesn't exist. No need to delete...`);
         });
 };
 

src/routers/JobLogsAPI/JobLogsAPI.js

@@ -4,7 +4,10 @@
 
 var express = require('express'),
     JobLogManager = require('./JobLogManager'),
-    router = express.Router();
+    MONGO_COLLECTION = 'JobLogsMetadata',
+    mongo,
+    router = express.Router(),
+    storage;
 
 /**
  * Called when the server is created but before it starts to listening to incoming requests.
@@ -27,6 +30,7 @@ function initialize(middlewareOpts) {
         logManager = new JobLogManager(logger, gmeConfig);
 
     logger.debug('initializing ...');
+    storage = require('../storage')(logger, gmeConfig);
 
     // Ensure authenticated can be used only after this rule.
     router.use('*', function (req, res, next) {
@@ -40,29 +44,72 @@ function initialize(middlewareOpts) {
 
     router.get('/:project/:branch/:job', function (req, res/*, next*/) {
         // Retrieve the job logs for the given job
-        logManager.getLog(req.params).then(log => {
-            res.set('Content-Type', 'text/plain');
-            res.send(log);
-        });
+        logger.info(`Requested logs for ${req.params.job} in ${req.params.project}`);
+        logManager.getLog(req.params)
+            .then(log => {
+                res.set('Content-Type', 'text/plain');
+                res.send(log);
+            })
+            .catch(err => logger.error(`Log retrieval failed: ${err}`));
+    });
+
+    router.get('/metadata/:project/:branch/:job', function (req, res/*, next*/) {
+        logger.info(`Requested metadata for ${req.params.job} in ${req.params.project}`);
+        return mongo.findOne(req.params)
+            .then(info => {
+                var lineCount = info ? info.lineCount : -1;
+
+                return res.json({
+                    lineCount: lineCount
+                });
+            })
+            .catch(err => logger.error(`Metadata retrieval failed: ${err}`));
     });
 
     router.patch('/:project/:branch/:job', function (req, res/*, next*/) {
         var logs = req.body.patch;
         logger.info(`Received append request for ${req.params.job} in ${req.params.project}`);
-        logManager.appendTo(req.params, logs)
-            .then(() => res.send('Append successful'))
+        return logManager.appendTo(req.params, logs)
+            .then(() => {
+                if (req.body.lineCount || req.body.cmdCount || req.body.createdIds) {
+                    var info = {
+                        project: req.params.project,
+                        branch: req.params.branch,
+                        job: req.params.job,
+                        lineCount: req.body.lineCount || -1,
+                        createdIds: req.body.createdIds || [],
+                        cmdCount: req.body.cmdCount || 0
+                    };
+                    logger.debug('metadata is', info);
+                    return mongo.update(req.params, info, {upsert: true})
+                        .then(() => res.send('Append successful'));
+                } else {
+                    res.send('Append successful');
+                }
+            })
             .catch(err => logger.error(`Append failed: ${err}`));
     });
 
     router.delete('/:project/:branch/:job', function (req, res/*, next*/) {
-        logManager.delete(req.params).then(() => res.send('delete successful'));
+        logger.info(`Request to delete logs for ${req.params.job} in ${req.params.project}`);
+        logManager.delete(req.params)
+            .then(() => mongo.findOneAndDelete(req.params))
+            .then(() => {
+                logger.info('Job log deletion successful!');
+                res.status(204).send('delete successful');
+            })
+            .catch(err => logger.error(`Job log deletion failed: ${err}`));
     });
 
     router.post('/migrate/:project/:srcBranch/:dstBranch', function (req, res/*, next*/) {
         var jobs = req.body.jobs;
+        logger.info(`Migrating logs from ${req.params.srcBranch} to ${req.params.dstBranch} in ${req.params.project}`);
         logManager.migrate(req.params, jobs)
-            .then(() => res.send('migration successful'))
-            .fail(err => logger.error(err));
+            .then(() => {
+                logger.info('Log migration successful!');
+                res.send('migration successful');
+            })
+            .fail(err => logger.error(`migration failed: ${err}`));
     });
 
     logger.debug('ready');
@@ -73,7 +120,11 @@ function initialize(middlewareOpts) {
  * @param {function} callback
  */
 function start(callback) {
-    callback();
+    storage.then(db => {
+        mongo = db.collection(MONGO_COLLECTION);
+        callback();
+    });
+
 }
 
 /**

src/routers/JobOriginAPI/JobOriginAPI.js

@@ -0,0 +1,162 @@
+/*jshint node:true*/
+
+'use strict';
+
+var express = require('express'),
+    MONGO_COLLECTION = 'JobOrigins',
+    utils = require('../utils'),
+    mongo,
+    router = express.Router(),
+    storage;
+
+/**
+ * Called when the server is created but before it starts to listening to incoming requests.
+ * N.B. gmeAuth, safeStorage and workerManager are not ready to use until the start function is called.
+ * (However inside an incoming request they are all ensured to have been initialized.)
+ *
+ * @param {object} middlewareOpts - Passed by the webgme server.
+ * @param {GmeConfig} middlewareOpts.gmeConfig - GME config parameters.
+ * @param {GmeLogger} middlewareOpts.logger - logger
+ * @param {function} middlewareOpts.ensureAuthenticated - Ensures the user is authenticated.
+ * @param {function} middlewareOpts.getUserId - If authenticated retrieves the userId from the request.
+ * @param {object} middlewareOpts.gmeAuth - Authorization module.
+ * @param {object} middlewareOpts.safeStorage - Accesses the storage and emits events (PROJECT_CREATED, COMMIT..).
+ * @param {object} middlewareOpts.workerManager - Spawns and keeps track of "worker" sub-processes.
+ */
+// When testing, use in memory storage...
+function initialize(middlewareOpts) {
+    var logger = middlewareOpts.logger.fork('JobOriginAPI'),
+        gmeConfig = middlewareOpts.gmeConfig,
+        ensureAuthenticated = middlewareOpts.ensureAuthenticated,
+        REQUIRED_FIELDS = ['hash', 'project', 'execution', 'job', 'nodeId', 'branch'];
+
+    storage = require('../storage')(logger, gmeConfig);
+
+    logger.debug('initializing ...');
+    // Ensure authenticated can be used only after this rule.
+    router.use('*', function (req, res, next) {
+        // This header ensures that any failures with authentication won't redirect.
+        res.setHeader('X-WebGME-Media-Type', 'webgme.v1');
+        next();
+    });
+
+    // Use ensureAuthenticated if the routes require authentication. (Can be set explicitly for each route.)
+    router.use('*', ensureAuthenticated);
+
+    // Connect to mongo...
+
+    router.get('/', function (req, res) {
+        mongo.find().toArray((err, all) => {
+            if (err) {
+                return res.status(500).send(err);
+            }
+            res.json(all.map(entry => {
+                delete entry._id;
+                return entry;
+            }));
+        });
+    });
+
+    router.get('/:jobHash', function (req, res/*, next*/) {
+        var hash = req.params.jobHash,
+            jobInfo = {};
+
+        mongo.findOne({hash: hash})
+            .then(result => {
+                if (result) {
+                    // Filter the result object
+                    for (var i = REQUIRED_FIELDS.length; i--;) {
+                        jobInfo[REQUIRED_FIELDS[i]] = result[REQUIRED_FIELDS[i]];
+                    }
+                    return res.json(jobInfo);
+                }
+                res.sendStatus(404);
+            })
+            .catch(err => {
+                logger.error(`Storing job info failed: ${err}`);
+                res.status(500).send(err);
+            });
+    });
+
+    router.post('/:jobHash', function (req, res/*, next*/) {
+        var hash = req.params.jobHash,
+            jobInfo = {
+                hash: hash,
+                project: req.body.project,
+                execution: req.body.execution,
+                branch: req.body.branch,
+                job: req.body.job,  // job name
+                nodeId: req.body.nodeId
+            };
+
+        // Check that none of the fields are undefined
+        var missing = utils.getMissingField(jobInfo, REQUIRED_FIELDS);
+        if (missing) {
+            return res.status(400).send(`Missing required field: ${missing}`);
+        }
+
+        logger.debug(`Storing job info for ${hash}`);
+        return mongo.insertOne(jobInfo)
+            .then(() => res.sendStatus(201))
+            .catch(err => {
+                logger.error(`Storing job info failed: ${err}`);
+                res.status(500).send(err.toString());
+            });
+    });
+
+    router.delete('/:jobHash', function (req, res/*, next*/) {
+        var hash = req.params.jobHash;
+
+        mongo.findOneAndDelete({hash: hash})
+            .then(() => res.sendStatus(204));
+    });
+
+    // on fork
+    router.patch('/:jobHash', function (req, res) {
+        var hash = req.params.jobHash;
+
+        if (!req.body.branch) {
+            return res.status(400).send('Missing "branch" field');
+        }
+
+        return mongo.findOneAndUpdate({hash: hash}, {$set: {branch: req.body.branch}})
+            .then(() => {
+                logger.debug('Finished updateOne!');
+                res.sendStatus(200);
+            })
+            .catch(err => {
+                logger.error(`Job update failed: ${err}`);
+                res.status(500).send(err);
+            });
+    });
+
+    logger.debug('ready');
+}
+
+/**
+ * Called before the server starts listening.
+ * @param {function} callback
+ */
+function start(callback) {
+    storage.then(db => {
+        mongo = db.collection(MONGO_COLLECTION);
+        callback();
+    });
+
+}
+
+/**
+ * Called after the server stopped listening.
+ * @param {function} callback
+ */
+function stop(callback) {
+    callback();
+}
+
+
+module.exports = {
+    initialize: initialize,
+    router: router,
+    start: start,
+    stop: stop
+};

src/routers/storage.js

@@ -0,0 +1,19 @@
+// Get a mongodb connection
+
+var mongodb = require('mongodb'),
+    connection;
+
+module.exports = function(logger, gmeConfig) {
+    if (!connection) {
+        connection = mongodb.MongoClient.connect(gmeConfig.mongo.uri, gmeConfig.mongo.options)
+            .then(db => {
+                logger.debug('Connected to mongo!');
+                return db;
+            })
+            .catch(err => {
+                logger.error(`Could not connect to mongo: ${err}`);
+                throw err;
+            });
+    }
+    return connection;
+};

src/routers/utils.js

@@ -0,0 +1,10 @@
+module.exports = {
+    getMissingField: function(array, fields) {
+        for (var i = fields.length; i--;) {
+            if (!array[fields[i]]) {
+                return fields[i];
+            }
+        }
+        return null;
+    }
+};