v0.19.0

* WIP #880 Fixed initial load of 'pipelines'

This was breaking the detection of the retrieval of the node history
from localStorage

* WIP #880 Fixed the root child check to not require node load

* WIP #880 Removed unused var

.codeclimate.yml

@@ -0,0 +1,38 @@
+---
+engines:
+  csslint:
+    enabled: true
+  duplication:
+    enabled: true
+    exclude_fingerprints:
+      - 1e004cf4e49528a58a0ac3858112601c
+    config:
+      languages:
+      - ruby
+      - javascript
+      - python
+      - php
+  eslint:
+    enabled: true
+  fixme:
+    enabled: true
+ratings:
+  paths:
+  - "**.css"
+  - "**.inc"
+  - "**.js"
+  - "**.jsx"
+  - "**.module"
+  - "**.php"
+  - "**.py"
+  - "**.rb"
+exclude_paths:
+- config/
+- test/
+- src/common/lua.js
+- src/common/js-yaml.min.js
+- src/visualizers/widgets/Sidebar/lib/
+- src/visualizers/widgets/TextEditor/lib/
+- src/visualizers/widgets/PipelineIndex/styles/PipelineIndex.css
+- src/visualizers/widgets/LineGraph/lib/
+- src/visualizers/widgets/PipelineEditor/klay.js

.csslintrc

@@ -0,0 +1,3 @@
+--exclude-exts=.min.css
+--exclude-list=src/visualizers/widgets/PipelineIndex/styles/PipelineIndex.css
+--ignore=adjoining-classes,box-model,ids,order-alphabetical,unqualified-attributes

.eslintignore

@@ -0,0 +1,5 @@
+src/common/lua.js
+src/visualizers/widgets/TextEditor/lib/*
+src/common/js-yaml.min.js
+src/visualizers/Visualizers.json
+config/config.webgme.js

.eslintrc.yml

@@ -0,0 +1,19 @@
+env:
+  browser: true
+  node: true
+  mocha: true
+  es6: true
+extends: 'eslint:recommended'
+rules:
+  indent:
+    - 2
+    - 4
+  linebreak-style:
+    - 2
+    - unix
+  quotes:
+    - 2
+    - single
+  semi:
+    - 2
+    - always

.gitattributes

@@ -0,0 +1 @@
+

.gitignore

@@ -1,6 +1,8 @@
 src/worker/tmp
+.env
 *.swp
 *.swo
+**.sass-cache
 # Logs
 logs
 *.log
@@ -29,4 +31,7 @@ build/Release
 # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git
 node_modules
 tmp/
+test-tmp/
 blob-local-storage/
+src/seeds/nn/hash.txt
+src/seeds/pipeline/hash.txt

.jshintrc

@@ -0,0 +1,3 @@
+{
+    "esversion": 6
+}

.npmignore

@@ -0,0 +1,40 @@
+src/worker/tmp
+.env
+*.swp
+*.swo
+**.sass-cache
+# Logs
+logs
+*.log
+
+# Runtime data
+pids
+*.pid
+*.seed
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (http://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directory
+# https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git
+node_modules
+tmp/
+test-tmp/
+blob-local-storage/
+src/seeds/nn/hash.txt
+src/seeds/pipeline/hash.txt
+
+# docs
+images/

.travis.yml

@@ -2,5 +2,4 @@ language: node_js
 services: mongodb
 sudo: false
 node_js:
-  - "4.1.1"
-  - "4.2.5"
+  - "6.2.1"

README.md

@@ -1,22 +1,46 @@
-[![Release State](https://img.shields.io/badge/state-pre--alpha-red.svg)](https://img.shields.io/badge/state-pre--alpha-red.svg)
+[![Release State](https://img.shields.io/badge/state-beta-yellow.svg)](https://img.shields.io/badge/state-beta-yellow.svg)
 [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](./LICENSE)
 [![Build Status](https://travis-ci.org/dfst/deepforge.svg?branch=master)](https://travis-ci.org/dfst/deepforge)
 [![Join the chat at https://gitter.im/dfst/deepforge](https://badges.gitter.im/dfst/deepforge.svg)](https://gitter.im/dfst/deepforge?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 [![Stories in Ready](https://badge.waffle.io/dfst/deepforge.png?label=ready&title=Ready)](https://waffle.io/dfst/deepforge)
+
+**Notice**: DeepForge is still a work in progress and in beta! That being said, any contributions and/or feedback is greatly appreciated. If you have any questions, check out the [wiki](https://github.com/dfst/deepforge/wiki/) or drop me a line on the gitter!
+
+
 # DeepForge
-DeepForge is an open-source visual development environment for deep learning. Currently, it supports Convolutional Neural Networks but we are planning on supporting additional deep learning classifiers such as RNNs and LSTMs. Additional features include real-time collaborative editing and version control.
+DeepForge is an open-source visual development environment for deep learning providing end-to-end support for creating deep learning models. This is achieved through providing the ability to design **architectures**, create training **pipelines**, and then execute these pipelines over a cluster. Using a notebook-esque api, users can get real-time feedback about the status of any of their **executions** including compare them side-by-side in real-time.
 
-## Quick Setup
-After cloning the repo, run
+![overview](images/overview.png "")
+
+Additional features include:
+- Graphical architecture editor
+- Training/testing pipeline creation
+- Distributed pipeline execution
+- Real-time pipeline feedback
+- Collaborative editing
+- Automatic version control.
+- Facilitates defining custom layers 
+
+## Quick Start
+Simply run the following command to install deepforge with its dependencies:
 
 ```
-npm install && npm start
+curl -o- https://raw.githubusercontent.com/dfst/deepforge/master/install.sh | bash
 ```
 
-Now, navigate to `localhost:8888` in a browser and create a new project. Select `nn` as the seed and start creating your neural nets!
+Or, if you already have NodeJS (v6) installed, simply run
 
-## Caffe Support?
-DeepForge uses Torch to perform the actual training and testing of the models. If you are interested in DeepForge using Caffe for actual training and testing, check out [DeepForge-Caffe](https://github.com/dfst/deepforge-caffe).
+```
+npm install -g deepforge
+```
+
+Finally, start deepforge with `deepforge start`and navigate to [http://localhost:8888](http://localhost:8888) to start using DeepForge! For more, detailed instructions, check out the [wiki](https://github.com/dfst/deepforge/wiki/Installation-Guide).
+
+**Note**: running deepforge w/ `deepforge start` will also require [MongoDB](https://www.mongodb.com/download-center?jmp=nav#community) to be installed locally.
+
+Also, be sure to check out the other available features of the `deepforge` cli; it can be used to update, manage your torch installation, uninstall deepforge and run individual components!
 
 ## Interested in contributing?
 Contributions are welcome! Either fork the project and submit some PR's or shoot me an email about getting more involved!
+
+Sponsored by [Digital Reasoning](http://www.digitalreasoning.com/)

app.js

@@ -3,11 +3,21 @@
 
 var gmeConfig = require('./config'),
     webgme = require('webgme'),
+    path = require('path'),
+    rm_rf = require('rimraf'),
     myServer;
 
 webgme.addToRequireJsPaths(gmeConfig);
 
+// Clear seed hash info
+['nn', 'pipeline'].map(lib => path.join(__dirname, 'src', 'seeds', lib, 'hash.txt'))
+    .forEach(file => rm_rf.sync(file));
+
 myServer = new webgme.standaloneServer(gmeConfig);
-myServer.start(function () {
-    //console.log('server up');
-});
+myServer.start(function (err) {
+    if (err) {
+        process.exit(1);
+    }
+
+    console.log('DeepForge now listening on port', gmeConfig.server.port);
+});

bin/.eslintrc.yml

@@ -0,0 +1,21 @@
+env:
+  browser: true
+  node: true
+  mocha: true
+  es6: true
+extends: 'eslint:recommended'
+rules:
+  no-console:
+    - 0
+  indent:
+    - 2
+    - 4
+  linebreak-style:
+    - 2
+    - unix
+  quotes:
+    - 2
+    - single
+  semi:
+    - 2
+    - always

bin/config.json

@@ -0,0 +1,18 @@
+{
+  "torch": {
+    "dir": "~/.deepforge/torch"
+  },
+  "blob": {
+    "dir": "~/.deepforge/blob"
+  },
+  "worker": {
+    "cache": {
+        "useBlob": true,
+        "dir": "~/.deepforge/worker/cache"
+    },
+    "dir": "~/.deepforge/worker"
+  },
+  "mongo": {
+    "dir": "~/.deepforge/data"
+  }
+}

bin/deepforge

@@ -0,0 +1,459 @@
+#!/usr/bin/env node
+
+var Command = require('commander').Command,
+    program = new Command(),
+    childProcess = require('child_process'),
+    rawSpawn = childProcess.spawn,
+    Q = require('q'),
+    execSync = childProcess.execSync,
+    path = require('path'),
+    fs = require('fs'),
+    version = require('../package.json').version,
+    exists = require('exists-file'),
+    DEFAULT_CONFIG = require('./config.json'),
+    merge = require('lodash.merge'),
+    config,
+
+    configDir = path.join(process.env.HOME, '.deepforge'),
+    configPath = path.join(configDir, 'config.json'),
+    dataPath = path.join(configDir, 'data'),
+
+    localConfig,
+    rm_rf = require('rimraf'),
+    p = dir => {
+        if (typeof dir === 'string') {
+            return dir.replace(/^~/, process.env.HOME);  // resolve '~' to '$HOME'
+        }
+        return dir;
+    };
+
+// Check for any commands
+if (process.argv.length === 2) {
+    process.argv.push('--help');
+}
+
+// Create the config if it doesn't exist
+if (!exists.sync(configDir)) {
+    fs.mkdirSync(configDir);
+}
+if (!exists.sync(dataPath)) {
+    fs.mkdirSync(dataPath);
+}
+if (!exists.sync(configPath)) {
+    fs.writeFileSync(configPath, '{\n}');
+}
+
+localConfig = require(configPath);
+config = merge({}, DEFAULT_CONFIG, localConfig);
+
+var getConfigValue = function(id, srcConfig) {
+    var keys = id.split('.'),
+        value = srcConfig || config;
+
+    for (var i = 0; i < keys.length; i++) {
+        if (!value.hasOwnProperty(keys[i])) {
+            return null;
+        }
+        value = value[keys[i]];
+    }
+    return value;
+};
+
+var storeConfig = function(id, value) {
+    // load the config
+    var keys = id.split('.').filter(k => k),
+        lastKey = keys.pop(),
+        currentObj = localConfig,
+        current = getConfigValue(id),
+        expType = typeof getConfigValue(id, DEFAULT_CONFIG);
+
+    // Check if it is a valid key
+    if (current === null) {
+        return false;
+    }
+
+    for (var i = 0; i < keys.length; i++) {
+        if (!currentObj[keys[i]]) {
+            currentObj[keys[i]] = {};
+        }
+        currentObj = currentObj[keys[i]];
+    }
+
+    if (expType !== 'string') {
+        try {  // try to downcast
+            value = JSON.parse(value);
+        } catch (e) {
+            console.log(`Invalid value: "${value}" (expected ${expType})`);
+            return;
+        }
+    }
+
+    currentObj[lastKey] = value;
+    fs.writeFileSync(configPath, JSON.stringify(localConfig, null, 2));
+    return true;
+};
+
+(function() {  // Load config to env
+    var envToConf = require('./envConfig.json');
+    Object.keys(envToConf).forEach(env => {
+        var cKey = envToConf[env];
+        process.env[env] = process.env[env] || p(getConfigValue(cKey));
+    });
+
+    // Special cases
+    if (process.env.DEEPFORGE_WORKER_USE_BLOB === 'true' &&
+        exists.sync(process.env.DEEPFORGE_BLOB_DIR)) {
+
+        process.env.DEEPFORGE_WORKER_CACHE = process.env.DEEPFORGE_BLOB_DIR + '/wg-content';
+    }
+})();
+
+program
+    .version('v' + version)
+    .description('Command line interface for managing deepforge');
+
+var isLocalUri = function(protocol, uri) {
+    return uri.indexOf(protocol + '://localhost') === 0 ||
+        uri.indexOf(protocol + '://127.0.0.1') === 0;
+};
+
+var checkMongo = function(args, notSilent) {
+    // check the webgme config
+    var gmeConfig = require('../config'),
+        mongoUri = gmeConfig.mongo.uri;
+
+    if (isLocalUri('mongodb', mongoUri)) {
+        // Make sure mongo is running locally (using pgrep)
+        try {
+            execSync('pgrep mongod').toString();
+            console.log('MongoDB is already running!');
+        } catch (e) {  // no pIds
+            console.log('Starting MongoDB...');
+            var match = mongoUri.match(/:([0-9]+)/),
+                port = '80';
+
+            if (match) {
+                port = match[1];
+            }
+
+            startMongo(args, port, !notSilent);
+        }
+    } else if (notSilent) {
+        console.log(`Cannot start remote mongo locally: ${mongoUri}`);
+    } else {
+        console.log(`Using remote mongo: ${mongoUri}`);
+    }
+};
+
+var startMongo = function(args, port, silent) {
+    var opts = ['--dbpath', p(config.mongo.dir), '--port', port],
+        job = rawSpawn('mongod', opts, {cwd: process.env.HOME});
+
+    if (!silent) {
+        job.stdout.on('data',
+            data => process.stdout.write(data.toString()));
+    }
+    job.on('error', err => {
+        if (err.code === 'ENOENT') {
+            console.log('Could not find MongoDB. Is it installed?');
+            if (!args.mongo) {
+                console.log('Otherwise, set MONGO_URI to the desired mongo uri and try again:');
+                console.log('');
+                console.log('    MONGO_URI=mongodb://some.other.ip:27017' +
+                    `/deepforge deepforge ${process.argv.slice(2).join(' ')}`);
+                console.log('');
+            }
+        } else {
+            console.log('Error encountered while starting MongoDB');
+            throw err;
+        }
+    });
+    job.stderr.on('data', data => {
+        var msg = 'mongodb: ' + data;
+        process.stdout.write(msg);
+    });
+    job.on('exit', code => {
+        if (code) {
+            console.log('MongoDB closed w/ error code: ' + code);
+        }
+    });
+};
+
+var hasTorch = function() {
+    var result = childProcess.spawnSync('th', ['--help']);
+    return !result.error;
+};
+
+var installTorchExtras = function() {
+    // Check if rnn is installed
+    var result = childProcess.spawnSync('luarocks', ['list', '--porcelain']),
+        pkgs = result.stdout.toString().split('\n')
+            .map(line => line.match(/^[a-zA-Z0-9]+/g))
+            .map(m => m && m[0]);
+
+    if (pkgs.indexOf('rnn') === -1) {
+        return spawn('luarocks', ['install', 'rnn']);
+    } else {
+        return Q();
+    }
+};
+
+var installTorch = function() {
+    var tgtDir = p(config.torch.dir),
+        args;
+
+    if (!hasTorch()) {
+        // Try to install torch
+        console.log(`Torch7 not found. Installing to ${tgtDir}...`);
+        args = `clone https://github.com/torch/distro.git ${tgtDir} --recursive`.split(' ');
+
+        return spawn('git', args)
+            .then(code => {
+                if (code !== 0) {
+                    if (code === 128) {
+                        console.error(`${tgtDir} is not empty. ` +
+                            'Please empty it or change the torch directory:\n' +
+                            '\n    deepforge config torch.dir NEW/TORCH/PATH\n');
+
+                    }
+
+                    throw `Torch install Failed with exit code ${code}`;
+                } else {  // continue installation
+                    process.chdir(tgtDir);
+                    return spawn('bash', ['install-deps'])
+                        .then(() => spawn('bash', ['install.sh'], true))
+                        .then(() => {
+                            storeConfig('torch.dir', tgtDir);
+                            console.log('Installed torch. Please close and ' +
+                                're-open your terminal to use DeepForge w/ ' +
+                                'torch support!');
+                            process.exit(0);
+                        });
+                }
+            });
+    } else {
+        return Q();
+    }
+};
+
+var spawn = function(cmd, args, opts) {
+    var deferred = Q.defer(),
+        job,
+        spawnOpts = typeof opts === 'object' ? opts : null,
+        forwardStdin = opts === true,
+        isOpen = true,
+        err;
+
+    args = args || [];
+    job = spawnOpts ? rawSpawn(cmd, args, spawnOpts) : rawSpawn(cmd, args);
+    job.stdout.on('data', data => process.stdout.write(data));
+    job.stderr.on('data', data => process.stderr.write(data));
+    job.on('close', code => {
+        isOpen = false;
+        if (err) {
+            deferred.reject(err, code);
+        } else {
+            deferred.resolve(code);
+        }
+    });
+    job.on('error', e => err = e);
+
+    if (forwardStdin) {
+        process.stdin.on('data', data => {
+            if (isOpen) {
+                job.stdin.write(data);
+            }
+        });
+    }
+
+    return deferred.promise;
+};
+
+program.command('start')
+    .description('start deepforge locally (default) or specific components')
+    .option('-p, --port <port>', 'specify the port to use')
+    .option('-s, --server', 'start the server')
+    .option('-w, --worker [url]', 'start a worker and connect to given url. Defaults to local deepforge')
+    .option('-m, --mongo', 'start MongoDB')
+    .action(args => {
+        var main = path.join(__dirname, 'start-local.js');
+
+        if (args.port) {
+            process.env.PORT = args.port;
+        }
+
+        if (args.server) {
+            checkMongo(args);
+            main = path.join(__dirname, '..', 'app.js');
+            spawn('node', [main]);
+        }
+
+        if (args.worker) {
+            if (hasTorch()) {
+                installTorchExtras().then(() => {
+                    main = path.join(__dirname, 'start-worker.js');
+                    if (args.worker !== true) {
+                        spawn('node', [main, args.worker]);
+                    } else {
+                        spawn('node', [main]);
+                    }
+                });
+            } else {
+                installTorch();
+            }
+        }
+
+        if (args.mongo) {
+            checkMongo(args, true);
+        }
+
+        if (!args.server && !args.worker && !args.mongo) {
+            // Starting everything
+            checkMongo(args);
+            if (hasTorch()) {
+                installTorchExtras().then(() => spawn('node', [main]));
+            } else {
+                installTorch();
+            }
+        }
+
+    });
+
+// update
+program
+    .command('update')
+    .description('upgrade deepforge to latest version')
+    .option('-g, --git', 'update tracking the git repo')
+    .option('-t, --torch', 'update torch installation')
+    .option('-s, --server', 'update deepforge')
+    .action(args => {
+        var pkg = 'deepforge',
+            latestVersion;
+
+        // Install the project
+        if (!args.torch || args.server) {
+
+            if (args.git) {
+                pkg = 'dfst/deepforge';
+            } else {
+                // Check the version
+                try {
+                    latestVersion = execSync('npm show deepforge version')
+                        .toString().replace(/\s+$/, '');
+
+                    if (latestVersion === version) {
+                        console.log('Already up-to-date');
+                        return;
+                    }
+                } catch (e) {
+                    console.log('Could not retrieve the latest deepforge version');
+                }
+            }
+
+            spawn('npm', ['install', '-g', pkg])
+                .then(() => {
+                    console.log('Upgrade successful!');
+                })
+                .catch(code => console.log('Upgrade failed w/ error code: ' + code));
+        }
+
+        if (args.torch || !args.server) {
+            // Update torch
+            if (hasTorch()) {
+                // Upgrade torch
+                console.log('Upgrading torch...');
+                console.log(`Checking for torch in ${config.torch.dir}`);
+                // Verify that torch is installed in the config's location
+                if (!exists.sync(path.join(config.torch.dir, 'update.sh'))) {
+                    // config is incorrect!
+                    console.log('Could not find torch installation. Please update the deepforge config with:');
+                    console.log('');
+                    console.log('    deepforge config torch.dir ~/path/to/torch/install');
+                    console.log('');
+                    return;
+                }
+
+                spawn('bash', ['./update.sh'], {cwd: p(config.torch.dir)})
+                    .catch(err => console.log('Upgrade failed w/ error code: ' + err.code))
+                    .then(() => {
+                        console.log('About to update rnn package...');
+                        // Update rnn
+                        return spawn('luarocks', ['install', 'rnn']);
+                    })
+                    .then(() => {
+                        console.log('Upgrade successful!');
+                    })
+                    .catch(code => console.log('Upgrade failed w/ error code: ' + code));
+            } else {
+                installTorch();
+            }
+        }
+    });
+
+// uninstall command
+program
+    .command('uninstall')
+    .description('uninstall deepforge from the system')
+    .option('-t, --torch', 'uninstall torch')
+    .option('-c, --clean', 'uninstall deepforge, torch and all associated data/config')
+    .action(opts => {
+        if (opts.torch || opts.clean) {
+            if (opts.torch) {
+                console.log(`uninstalling torch at ${p(config.torch.dir)}`);
+            }
+            rm_rf.sync(p(config.torch.dir));
+        }
+
+        if (opts.clean) {  // remove the .deepforge directory
+            console.log('removing config and data files...');
+            rm_rf.sync(p(config.mongo.dir));
+            rm_rf.sync(p(configDir));
+        }
+
+        if (!opts.torch || opts.clean) {  // uninstall deepforge
+            spawn('npm', ['uninstall', '-g', 'deepforge'])
+                .then(() => console.log('deepforge has been uninstalled!'))
+                .catch(() => console.log('uninstall failed'));
+        }
+    });
+
+// config
+program
+    .command('config [key] [value]')
+    .description('read or edit config options (omit "value" to see current value)')
+    .action(key => {
+        var value = program.args[1],
+            success;
+
+        if (value) {  // write a value
+            success = storeConfig(key, value);
+            if (success) {
+                console.log('Config has been updated!');
+            }
+        } else if (key) {  // read a single value
+            value = getConfigValue(key);
+            if (value === null) {
+                console.log(`Invalid config value: "${key}"`);
+                return;
+            } 
+
+            if (typeof value === 'object') {
+                value = JSON.stringify(value, null, 2);
+            }
+
+            console.log(value);
+        } else {  // print entire config
+            console.log(`Current config:\n${JSON.stringify(config, null, 2)}`);
+        }
+    });
+
+module.exports = function(cmd) {
+    var cmds = cmd.split(/\s+/).filter(w => !!w);
+    cmds.unshift('./bin/deepforge');
+    cmds.unshift('node');
+    program.parse(cmds);
+};
+
+if (require.main === module) {
+    program.parse(process.argv);
+}

bin/envConfig.json

@@ -0,0 +1,6 @@
+{
+    "DEEPFORGE_BLOB_DIR": "blob.dir",
+    "DEEPFORGE_WORKER_CACHE": "worker.cache.dir",
+    "DEEPFORGE_WORKER_DIR": "worker.dir",
+    "DEEPFORGE_WORKER_USE_BLOB": "worker.cache.useBlob"
+}

bin/start-local.js

@@ -0,0 +1,34 @@
+// Run `npm start` and listen for 'DeepForge' then start worker
+var spawn = require('child_process').spawn,
+    stdout = '',
+    execJob,
+    path = require('path'),
+    env = {cwd: path.join(__dirname, '..')},
+    workerJob = null,
+    gmeConfig = require(__dirname + '/../config');
+
+// Set the cache to the blob
+if (gmeConfig.blob.type === 'FS') {
+    process.env.DEEPFORGE_WORKER_CACHE = path.resolve(gmeConfig.blob.fsDir + '/wg-content');
+}
+
+process.env.NODE_ENV = 'local';
+execJob = spawn('npm', [
+    'start'
+], env);
+execJob.stdout.pipe(process.stdout);
+execJob.stderr.pipe(process.stderr);
+
+execJob.stdout.on('data', function(chunk) {
+    if (!workerJob) {
+        stdout += chunk;
+        if (stdout.indexOf('DeepForge') > -1) {
+            workerJob = spawn('npm', ['run', 'worker'], env);
+            workerJob.stdout.pipe(process.stdout);
+            workerJob.stderr.pipe(process.stderr);
+            workerJob.on('close', code => code && process.exit(code));
+        }
+    }
+});
+
+execJob.on('close', code => code && process.exit(code));

bin/start-worker.js

@@ -1,24 +1,69 @@
-'use strict';
+/*globals process, __dirname, require*/
 
 var path = require('path'),
     fs = require('fs'),
-    spawn = require('child_process').spawn,
+    childProcess = require('child_process'),
+    spawn = childProcess.spawn,
+    rm_rf = require('rimraf'),
     projectConfig = require(__dirname + '/../config'),
     executorSrc = path.join(__dirname, '..', 'node_modules', 'webgme', 'src',
         'server', 'middleware', 'executor', 'worker'),
-    workerPath = path.join(__dirname, '..', 'src', 'worker'),
+    id = Date.now(),
+    workerRootPath = process.env.DEEPFORGE_WORKER_DIR || path.join(__dirname, '..', 'src', 'worker'),
+    workerPath = path.join(workerRootPath, `worker_${id}`),
     workerConfigPath =  path.join(workerPath, 'config.json'),
     workerTmp = path.join(workerPath, 'tmp'),
     address,
     config = {};
 
+var createDir = function(dir) {
+    try {
+        fs.statSync(dir);
+    } catch (e) {
+        // Create dir
+        fs.mkdirSync(dir);
+        return true;
+    }
+    return false;
+};
+createDir(workerRootPath);
+createDir(workerPath);
+createDir(workerTmp);
+
+// Create sym link to the node_modules
+var modules = path.join(workerRootPath, 'node_modules');
+try {
+    fs.statSync(modules);
+} catch (e) {
+    // Create dir
+    childProcess.spawnSync('ln', ['-s', `${__dirname}/../node_modules`, modules]);
+    return true;
+}
+
+// Check torch support
+var result = childProcess.spawnSync('th', ['--help']);
+if (result.error) {
+    console.error('Checking Torch7 dependency failed. Do you have Torch7 installed ' + 
+        'and in your PATH?\n\nFor Torch7 installation instructions, check out ' +
+        'http://torch.ch/docs/getting-started.html');
+    process.exit(1);
+}
+
+var cleanUp = function() {
+    console.log('removing worker directory ', workerPath);
+    rm_rf.sync(workerPath);
+};
+
 var startExecutor = function() {
+    process.on('SIGINT', cleanUp);
+    process.on('uncaughtException', cleanUp);
+
     // Start the executor
     var execJob = spawn('node', [
         'node_worker.js',
         workerConfigPath,
         workerTmp
-        ]);
+    ]);
     execJob.stdout.pipe(process.stdout);
     execJob.stderr.pipe(process.stderr);
 };
@@ -32,7 +77,6 @@ var createConfigJson = function() {
     }
 
     config[address] = {};
-    // TODO: Check if the config already exists
     fs.writeFile(workerConfigPath, JSON.stringify(config), startExecutor);
 };
 

config/components.json

@@ -1,19 +1,68 @@
 {
+    "AutoViz": {
+        "preloadIds": [
+            "ArchEditor",
+            "ArchIndex",
+            "PipelineIndex",
+            "PipelineEditor",
+            "OperationEditor",
+            "ExecutionView"
+        ],
+        "visualizerOverrides": {
+            "": "ForwardViz",
+            "MyArtifacts": "ArtifactIndex",
+            "MyArchitectures": "ArchIndex",
+            "MyExecutions": "ExecutionIndex",
+            "MyPipelines": "PipelineIndex"
+        }
+    },
     "ArchEditor": {
         "hotkeys": "none",
-	"LayerColors": {}
+        "LayerColors": {}
     },
-    "CHFLayout": {
+    "BreadcrumbHeader": {
+        "pathRule": "history",
+        "cachePrefix": "deepforge-header"
+    },
+    "FloatingActionButton": {
+        "hideOnEmpty": true,
+        "pluginUIConfigs": {
+            "GenerateArchitecture": {
+                "icon": "description",
+                "hotkey": "shift enter",
+                "priority": -1
+            },
+            "ExecutePipeline": {
+                "icon": "play_arrow",
+                "hotkey": "shift enter",
+                "color": "green",
+                "priority": 1
+            },
+            "ImportTorch": {
+                "icon": "import_export",
+                "priority": -1
+            },
+            "GenerateExecFile": {
+                "icon": "play_for_work",
+                "priority": -1
+            }
+        }
+    },
+    "GenericUIProjectNavigatorController": {
+        "rootMenuClass": "deepforge-logo",
+        "rootDisplayName": "DeepForge"
+    },
+    "SidebarLayout": {
         "panels": [
             {
-                "id": "Header",
-                "panel": "BreadcrumbHeader/BreadcrumbHeaderPanel",
+                "id": "WorkerHeader",
+                "panel": "WorkerHeader/WorkerHeaderPanel",
                 "container": "header",
                 "DEBUG_ONLY": false
             },
             {
                 "id": "Footer",
-                "panel": "FooterControls/FooterControlsPanel",
+                "panel": "Footer/FooterPanel",
                 "container": "footer",
                 "DEBUG_ONLY": false
             },
@@ -23,6 +72,12 @@
                 "container": "center",
                 "DEBUG_ONLY": false
             },
+            {
+                "id": "Sidebar",
+                "panel": "Sidebar/SidebarPanel",
+                "container": "sidebar",
+                "DEBUG_ONLY": false
+            },
             {
                 "id": "ForgeActionButton",
                 "panel": "ForgeActionButton/ForgeActionButton",
@@ -31,5 +86,32 @@
             }
             
         ]
+    },
+    "RootViz": {
+        "nodes": [
+            {
+                "nodeName": "MyArchitectures",
+                "title": "Architectures",
+                "icon": "shuffle",
+                "rank": 1,
+                "description": "Neural network architectures are stored here and can be used in pipelines."
+            },
+            {
+                "nodeName": "MyPipelines",
+                "title": "Pipelines",
+                "color": "blue-grey",
+                "icon": "input",
+                "rank": 3,
+                "description": "Pipelines compose operations together to effectively train, test and/or ensemble models."
+            },
+            {
+                "nodeName": "MyArtifacts",
+                "title": "Artifacts",
+                "icon": "view_quilt",
+                "color": "blue-grey",
+                "rank": 5,
+                "description": "Artifacts from pipeline executions are stored here."
+            }
+        ]
     }
 }

config/config.base.js

@@ -1,15 +1,28 @@
+/*globals require, module, process*/
 'use strict';
 
 var config = require('./config.webgme'),
     validateConfig = require('webgme/config/validator');
 
+require('dotenv').load({silent: true});
+
 // Add/overwrite any additional settings here
-// config.server.port = 8080;
-// config.mongo.uri = 'mongodb://127.0.0.1:27017/webgme_my_app';
+config.server.port = +process.env.PORT || config.server.port;
+config.mongo.uri = process.env.MONGO_URI || config.mongo.uri;
+config.blob.fsDir = process.env.DEEPFORGE_BLOB_DIR || config.blob.fsDir;
+
 config.requirejsPaths.deepforge = './src/common';
-config.seedProjects.defaultProject = 'nn';
+config.requirejsPaths.ace = './src/visualizers/widgets/TextEditor/lib/ace';
+config.seedProjects.defaultProject = 'project';
+
+config.plugin.allowBrowserExecution = true;
+config.plugin.allowServerExecution = true;
 
 config.executor.enable = true;
 
+config.visualization.extraCss.push('deepforge/styles/global.css');
+
+config.storage.autoMerge.enable = true;
+
 validateConfig(config);
 module.exports = config;

config/config.default.js

@@ -9,8 +9,7 @@ var config = require('./config.base'),
 // config.mongo.uri = 'mongodb://127.0.0.1:27017/webgme_my_app';
 
 // Seeds for development are prefixed with 'dev'
-config.seedProjects.basePaths = config.seedProjects.basePaths
-    .filter(path => path.indexOf('dev') === -1);
+config.seedProjects.basePaths = ['src/seeds/project', 'src/seeds/cifar10'];
 
 validateConfig(config);
 module.exports = config;

config/config.local.js

@@ -0,0 +1,12 @@
+// Config for running deepforge w/ one local worker
+// jshint node: true
+'use strict';
+
+var config = require('./config.default'),
+    validateConfig = require('webgme/config/validator');
+
+// Turn up the worker polling rate
+config.executor.workerRefreshInterval = 150;
+
+validateConfig(config);
+module.exports = config;

config/config.test.js

@@ -5,9 +5,11 @@
 
 'use strict';
 
-var config = require('./config.default');
+var config = require('./config.default'),
+    path = require('path');
 
 config.server.port = 9001;
 config.mongo.uri = 'mongodb://127.0.0.1:27017/webgme_tests';
+config.blob.fsDir = path.join(__dirname, '..', 'test-tmp', 'blob');
 
-module.exports = config;
+module.exports = config;

config/config.webgme.js

@@ -6,31 +6,38 @@
 var config = require('webgme/config/config.default'),
     validateConfig = require('webgme/config/validator');
 
-
 // The paths can be loaded from the webgme-setup.json
-config.plugin.basePaths.push('src/plugins');
-config.plugin.basePaths.push('node_modules/webgme-simple-nodes/src/plugins');
-config.visualization.layout.basePaths.push('node_modules/webgme-chflayout/src/layouts');
-config.visualization.decoratorPaths.push('node_modules/webgme-easydag/src/decorators');
-config.seedProjects.basePaths.push('src/seeds/nn');
-config.seedProjects.basePaths.push('src/seeds/devTests');
-config.seedProjects.basePaths.push('src/seeds/devMinimal');
-config.seedProjects.basePaths.push('src/seeds/devUtilTests');
-config.seedProjects.basePaths.push('src/seeds/pipeline');
-config.seedProjects.basePaths.push('src/seeds/devPipelineTests');
-config.seedProjects.basePaths.push('src/seeds/demo');
+config.plugin.basePaths.push(__dirname + '/../src/plugins');
+config.plugin.basePaths.push(__dirname + '/../node_modules/webgme-simple-nodes/src/plugins');
+config.visualization.layout.basePaths.push(__dirname + '/../src/layouts');
+config.visualization.layout.basePaths.push(__dirname + '/../node_modules/webgme-chflayout/src/layouts');
+config.visualization.decoratorPaths.push(__dirname + '/../src/decorators');
+config.visualization.decoratorPaths.push(__dirname + '/../node_modules/webgme-easydag/src/decorators');
+config.seedProjects.basePaths.push(__dirname + '/../src/seeds/nn');
+config.seedProjects.basePaths.push(__dirname + '/../src/seeds/devTests');
+config.seedProjects.basePaths.push(__dirname + '/../src/seeds/devUtilTests');
+config.seedProjects.basePaths.push(__dirname + '/../src/seeds/pipeline');
+config.seedProjects.basePaths.push(__dirname + '/../src/seeds/devPipelineTests');
+config.seedProjects.basePaths.push(__dirname + '/../src/seeds/project');
+config.seedProjects.basePaths.push(__dirname + '/../src/seeds/cifar10');
+config.seedProjects.basePaths.push(__dirname + '/../src/seeds/xor');
+config.seedProjects.basePaths.push(__dirname + '/../src/seeds/devProject');
 
 
 
-config.visualization.panelPaths.push('node_modules/webgme-fab/src/visualizers/panels');
-config.visualization.panelPaths.push('node_modules/webgme-breadcrumbheader/src/visualizers/panels');
-config.visualization.panelPaths.push('node_modules/webgme-autoviz/src/visualizers/panels');
-config.visualization.panelPaths.push('node_modules/webgme-easydag/src/visualizers/panels');
-config.visualization.panelPaths.push('src/visualizers/panels');
+config.visualization.panelPaths.push(__dirname + '/../node_modules/webgme-fab/src/visualizers/panels');
+config.visualization.panelPaths.push(__dirname + '/../node_modules/webgme-breadcrumbheader/src/visualizers/panels');
+config.visualization.panelPaths.push(__dirname + '/../node_modules/webgme-autoviz/src/visualizers/panels');
+config.visualization.panelPaths.push(__dirname + '/../node_modules/webgme-easydag/src/visualizers/panels');
+config.visualization.panelPaths.push(__dirname + '/../src/visualizers/panels');
 
 
+config.rest.components['execution/logs'] = __dirname + '/../src/routers/JobLogsAPI/JobLogsAPI.js';
+config.rest.components['job/origins'] = __dirname + '/../src/routers/JobOriginAPI/JobOriginAPI.js';
+config.rest.components['execution/pulse'] = __dirname + '/../src/routers/ExecPulse/ExecPulse.js';
+
 // Visualizer descriptors
-config.visualization.visualizerDescriptors.push('./src/visualizers/Visualizers.json');
+config.visualization.visualizerDescriptors.push(__dirname + '/../src/visualizers/Visualizers.json');
 // Add requirejs paths
 config.requirejsPaths = {
   'EllipseDecorator': 'node_modules/webgme-easydag/src/decorators/EllipseDecorator',
@@ -52,7 +59,7 @@ config.requirejsPaths = {
   'widgets/FloatingActionButton': './node_modules/webgme-fab/src/visualizers/widgets/FloatingActionButton'
 };
 
-config.visualization.layout.default = 'CHFLayout';
+config.visualization.layout.default = 'SidebarLayout';
 config.mongo.uri = 'mongodb://127.0.0.1:27017/deepforge';
 validateConfig(config);
 module.exports = config;

install.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+# Things to install:
+#   - nvm
+
+command -v git >/dev/null 2>&1 || { echo >&2 "I require git but it's not installed.  Aborting."; exit 1; }
+
+echo >&2 "Checking DeepForge dependencies...";
+
+# profile (bash, zsh, profile, etc) borrowed from nvm's installer
+detect_profile() {
+  if [ -n "$PROFILE" -a -f "$PROFILE" ]; then
+    echo "$PROFILE"
+    return
+  fi
+
+  DETECTED_PROFILE=''
+  local SHELLTYPE
+  SHELLTYPE="$(basename "/$SHELL")"
+
+  if [ "$SHELLTYPE" = "bash" ]; then
+    if [ -f "$HOME/.bashrc" ]; then
+      DETECTED_PROFILE="$HOME/.bashrc"
+    elif [ -f "$HOME/.bash_profile" ]; then
+      DETECTED_PROFILE="$HOME/.bash_profile"
+    fi
+  elif [ "$SHELLTYPE" = "zsh" ]; then
+    DETECTED_PROFILE="$HOME/.zshrc"
+  fi
+
+  if [ -z "$DETECTED_PROFILE" ]; then
+    if [ -f "$HOME/.profile" ]; then
+      DETECTED_PROFILE="$HOME/.profile"
+    elif [ -f "$HOME/.bashrc" ]; then
+      DETECTED_PROFILE="$HOME/.bashrc"
+    elif [ -f "$HOME/.bash_profile" ]; then
+      DETECTED_PROFILE="$HOME/.bash_profile"
+    elif [ -f "$HOME/.zshrc" ]; then
+      DETECTED_PROFILE="$HOME/.zshrc"
+    fi
+  fi
+}
+detect_profile
+
+set_node_version() {
+    # Install nodejs v6.2.1
+    echo "Installing NodeJS v6.2.1"
+    nvm install v6.2.1
+    nvm alias default v6.2.1
+
+    # Install npm@2
+    npm install npm@2 -g
+}
+
+command -v node >/dev/null 2>&1 || {
+    # No node! Install nvm
+    echo >&2 "NodeJS is not found. Installing (using nvm)...";
+    curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.31.1/install.sh | bash;
+    source $DETECTED_PROFILE
+    . $NVM_DIR/nvm.sh
+
+    set_node_version
+}
+
+# Check node version supports arrow fns and string templates
+node -e '() => console.log(`print "3": ${1+2}`)' >/dev/null 2>&1 || {
+    echo "Unsupported version of NodeJS."
+    echo ""
+    echo "Please update NodeJS to version 4.x.x or later (6.x.x recommended)"
+    exit 1
+}
+
+echo >&2 "Installing DeepForge...";
+
+# Clone deepforge into ~/deepforge
+npm install -g deepforge
+
+echo "Final Installation Steps:"
+echo "  1) Close and re-open your terminal"
+echo "     (or run \"source $DETECTED_PROFILE\")"
+
+if [[ $NEEDS_MONGO ]]; then
+    echo "  2) Install MongoDB for your OS"
+    echo "     (available at https://www.mongodb.com/download-center)"
+    echo "     Note: for custom installs, this may not be required"
+fi
+
+echo ""
+echo "Then run DeepForge!"
+echo "  1) run \"deepforge start\""
+echo "  2) open a browser to http://localhost:8888"
+echo "  3) start building neural nets!"

package.json

@@ -1,26 +1,41 @@
 {
   "name": "deepforge",
+  "bin": {
+    "deepforge": "./bin/deepforge"
+  },
   "scripts": {
     "start": "node app.js",
     "start-dev": "NODE_ENV=dev node app.js",
+    "local": "node ./bin/start-local.js",
     "worker": "node ./bin/start-worker.js",
-    "test": "node ./node_modules/mocha/bin/mocha --recursive test"
+    "test": "mkdir ./test-tmp; node ./node_modules/mocha/bin/mocha --recursive test",
+    "watch-test": "./node_modules/nodemon/bin/nodemon.js --exec 'node ./node_modules/mocha/bin/mocha --recursive test'",
+    "build-nn": "node ./utils/nn-parser.js"
   },
-  "version": "0.1.0",
+  "version": "0.19.0",
   "dependencies": {
+    "commander": "^2.9.0",
+    "dotenv": "^2.0.0",
+    "exists-file": "^2.1.0",
+    "express": "^4.14.0",
     "lodash.difference": "^4.1.2",
-    "webgme": "^2.0.0",
-    "webgme-autoviz": "^2.0.0",
-    "webgme-breadcrumbheader": "^2.0.0",
+    "lodash.merge": "^4.5.1",
+    "mongodb": "^2.2.10",
+    "nodemon": "^1.9.2",
+    "q": "1.4.1",
+    "rimraf": "^2.4.0",
+    "webgme": "^2.6.0",
+    "webgme-autoviz": "^2.2.0",
+    "webgme-breadcrumbheader": "^2.1.1",
     "webgme-chflayout": "^2.0.0",
     "webgme-easydag": "dfst/webgme-easydag",
-    "webgme-fab": "^2.0.0",
-    "webgme-simple-nodes": "^2.0.0"
+    "webgme-fab": "dfst/webgme-fab",
+    "webgme-simple-nodes": "^2.1.0"
   },
   "devDependencies": {
+    "chai": "^3.0.0",
     "jszip": "^2.5.0",
     "mocha": "^2.2.5",
-    "rimraf": "^2.4.0",
-    "chai": "^3.0.0"
+    "mockery": "^1.7.0"
   }
 }

src/common/Constants.js

@@ -0,0 +1,49 @@
+/* globals define */
+(function(root, factory){
+    if(typeof define === 'function' && define.amd) {
+        define([], function(){
+            return factory();
+        });
+    } else if(typeof module === 'object' && module.exports) {
+        module.exports = factory();
+    } else {
+        root.CONSTANTS = factory();
+    }
+}(this, function() {
+    return {
+        LINE_OFFSET: 'lineOffset',
+
+        // DeepForge metadata creation in dist execution
+        START_CMD: 'deepforge-cmd',
+
+        IMAGE: {  // all prefixed w/ 'IMG' for simple upload detection
+            PREFIX: 'IMG',
+            BASIC: 'IMG-B',
+            CREATE: 'IMG-C',
+            UPDATE: 'IMG-U',
+            NAME: 'IMAGE-N'  // No upload required
+        },
+
+        GRAPH_CREATE: 'GRAPH',
+        GRAPH_PLOT: 'PLOT',
+        GRAPH_CREATE_LINE: 'LINE',
+
+        // Code Generation Constants
+        CTOR_ARGS_ATTR: 'ctor_arg_order',
+
+        // Operation types
+        OP: {
+            INPUT: 'Input',
+            OUTPUT: 'Output'
+        },
+
+        PULSE: {
+            DEAD: 0,
+            ALIVE: 1,
+            DOESNT_EXIST: 2
+        },
+
+        // Job stdout update
+        STDOUT_UPDATE: 'stdout_update'
+    };
+}));

src/common/GraphChecker.js

@@ -1,3 +1,4 @@
+/* globals define */
 define([
     'deepforge/js-yaml.min'
 ], function(
@@ -148,28 +149,11 @@ define([
 
     GraphChecker.prototype = new Importer();
 
-    // Check if two models are isomorphic
-    var modelMatches = function(gmeNodes, text) {
-        var nodes = convertNodes(gmeNodes),
-            soln = yaml.load(text);
-
-        nodes.concat(soln).forEach(node => {
-            node.next = node.next || [];
-            node.attributes = node.attributes || {};
-        });
-
-        return _modelMatches(nodes, soln);
-    };
-
     //////////////// Operators ////////////////
+    // Check if two models are isomorphic
     var _modelMatches = function(soln, nodes) {
         var nodeMap = createMap(nodes),
-            solnMap = createMap(soln),
-            sInits,  // soln start nodes
-            nInits,  // 'nodes' start nodes
-            resMap = {},  // soln node id to node id
-            solnIds = soln.map(n => n.id),
-            nodeIds = nodes.map(n => n.id);
+            solnMap = createMap(soln);
 
         if (nodes.length !== soln.length) {
             return false;
@@ -187,8 +171,7 @@ define([
     };
 
     var getMostConstrained = function(soln, nodes) {
-        var options = soln.map(sn => [sn, nodes.filter(n => nodesMatch(sn, n))]),
-            startId;
+        var options = soln.map(sn => [sn, nodes.filter(n => nodesMatch(sn, n))]);
 
         options.sort((a, b) => b[1].length < a[1].length);
         return options[0];
@@ -199,8 +182,7 @@ define([
             nodes = Object.keys(nodeMap).map(id => nodeMap[id]),
             snode = solnMap[id],
             options,
-            used,
-            node;
+            used;
 
         mappings = mappings || {};
 
@@ -247,14 +229,16 @@ define([
             snext.push(startId);
         }
 
+        var inferNext = (prev, curr) => {
+            mappings2 = inferGraph(curr, solnMap, nodeMap, mappings2);
+            return prev && mappings;
+        };
+
         for (var i = options.length; i--;) {
             mappings[id] = options[i].id;  // need to clone the object
 
             mappings2 = clone(mappings);
-            result = snext.reduce((prev, curr) => {
-                mappings2 = inferGraph(curr, solnMap, nodeMap, mappings2);
-                return prev && mappings;
-            }, true);
+            result = snext.reduce(inferNext, true);
 
             if (result) {
                 return mappings2;

src/common/LayerParser.js

@@ -0,0 +1,382 @@
+/* globals define*/
+(function(root, factory){
+    if(typeof define === 'function' && define.amd) {
+        define(['./lua'], function(luajs){
+            return (root.LayerParser = factory(luajs));
+        });
+    } else if(typeof module === 'object' && module.exports) {
+        var luajs = require('./lua');
+        module.exports = (root.LayerParser = factory(luajs));
+    }
+}(this, function(luajs) {
+    var LayerParser = {};
+
+    //////////////////////// Setters //////////////////////// 
+    var returnsSelf = function(fnNode){
+        var stats = fnNode.block.stats,
+            last = stats[stats.length-1];
+
+        if (last.type === 'stat.return') {
+            return last.nret[0].type === 'variable' && last.nret[0].val === 'self';
+        }
+        return false;
+    };
+
+    var isAttrSetter = function(node){
+        if (node.type === 'stat.assignment' && node.lefts.length === 1) {
+            var left = node.lefts[0];
+            return left.type === 'expr.index' && left.self.val === 'self';
+        }
+        return false;
+    };
+
+    var getSettingAttrName = function(node){
+        if (isAttrSetter(node)) {
+            var left = node.lefts[0];
+            return left.key.val;
+        }
+        return null;
+    };
+
+    var getSettingAttrValue = function(node){
+        if (isAttrSetter(node)) {
+            return node.right;
+        }
+        return null;
+    };
+
+    var isSetterMethod = function(curr, parent, className){
+        if (parent && parent.type === 'stat.method') {
+            // is it a fn w/ two statements (stats)
+            if (parent.self.val === className && curr.type === 'function' &&
+                curr.block.stats.length === 2) {
+                // Is the first statement setting a value?
+                return returnsSelf(curr) && getSettingAttrName(curr.block.stats[0]);  // does it return itself?
+            }
+        }
+        return false;
+    };
+
+    var isFnArg = function(method, name) {
+        return method.args.indexOf(name) !== -1;
+    };
+
+    var getSetterSchema = function(node, method) {
+        var setterType,
+            setterFn,
+            value = getSettingAttrValue(node);
+
+        if (value[0].type === 'variable' && isFnArg(method.func, value[0].val)) {
+            setterType = 'arg';
+            setterFn = method.key.val;
+        } else {
+            setterType = 'const';
+            setterFn = {};
+            setterFn[value[0].val] = method.key.val;
+        }
+
+        return {
+            setterType,
+            setterFn
+        };
+    };
+
+    //////////////////////// Setters END //////////////////////// 
+
+    var isInitFn = function(node, className) {
+        if (node.type === 'stat.method' && node.self.val === className) {
+            return node.key.val === '__init';
+        }
+        return false;
+    };
+
+    var getClassAttrDefs = function(method) {
+        var fn = method.func,
+            dict = {},
+            attr,
+            right,
+            value;
+
+        luajs.codegen.traverse(curr => {
+            if (isAttrSetter(curr)) {
+                // Store the value if it is set to a constant
+                attr = curr.lefts[0].key.val;
+                right = curr.right[0];
+                if (right.type.indexOf('const.') !== -1) {
+                    value = right.val;
+
+                    if (right.type === 'const.nil') {
+                        value = null;
+                    }
+
+                    dict[attr] = value;
+                }
+            }
+        })(fn);
+
+        return dict;
+    };
+
+    var getAttrsAndVals = function(method) {
+        // Given a method, get the 'self' attributes and the default values
+        var fn = method.func,
+            dict = {},
+            varName,
+            value,
+            varUsageCnt = {};
+
+        // Get the variables that are used only once (or updating themselves)
+        luajs.codegen.traverse(curr => {
+            if (curr.type === 'variable') {
+                varUsageCnt[curr.val] = varUsageCnt[curr.val] ?
+                    varUsageCnt[curr.val] + 1 : 1;
+            }
+        })(method);
+
+        luajs.codegen.traverse(curr => {
+            // If the variable is only used once and is 'or'-ed w/ a constant
+            // during this use, we can infer that this is the default value
+            if (curr.type === 'expr.op' && curr.op === 'op.or' &&
+                curr.left.type === 'variable' && curr.right.type.indexOf('const') !== -1) {
+                varName = curr.left.val;
+                if (varUsageCnt[varName] === 1) {
+                    value = curr.right.type === 'const.nil' ? null : curr.right;
+                    dict[varName] = value;
+                }
+            }
+        })(fn);
+
+        return dict;
+    };
+
+    var copyNodeValues = function(attrs, from, to) {
+        var value;
+        for (var i = attrs.length; i--;) {
+            value = from[attrs[i]] || null;
+            if (value) {
+                value = (value && value.hasOwnProperty('val')) ? value.val : value;
+                to[attrs[i]] = value;
+            }
+        }
+        return to;
+    };
+
+    var getTypeCheckInfo = function(cond) {
+        var caller,
+            method,
+            target,
+            expType;
+
+        // Check for torch.isTypeOf:
+        if (cond.type === 'expr.call' && cond.func.type === 'expr.index') {
+            caller = cond.func.self.val;
+            method = cond.func.key.val;
+
+            if (cond.type === 'expr.call' && caller === 'torch') {
+                target = cond.args[0].val;
+                if (method === 'isTypeOf' && target) {
+                    expType = cond.args[1].val;
+                    return {
+                        target,
+                        type: expType
+                    };
+                }
+            }
+        } else if (cond.type === 'expr.op') {  // torch.type() === ''
+            // Check right side, too!
+            var sides = [cond.left, cond.right],
+                side,
+                otherSide;
+
+            for (var i = sides.length; i--;) {
+                side = sides[i];
+                otherSide = sides[(i+1)%2];
+                if (side.type === 'expr.call' && side.func.type === 'expr.index') {
+                    // Is it torch?
+                    caller = side.func.self.val;
+                    method = side.func.key.val;
+                    if (caller === 'torch' && method === 'type') {
+                        if (side.args[0].type === 'variable') {
+                            target = side.args[0].val;
+                            if (otherSide.type === 'const.string') {
+                                expType = otherSide.val;
+
+                                return {
+                                    target: target,
+                                    type: expType
+                                };
+                            }
+                        }
+                        
+                    }
+                }
+            }
+            return null;
+        }
+    };
+
+    var isError = function(stat) {
+        var fn;
+        if (stat.type === 'stat.expr' && stat.expr.type === 'expr.call') {
+            fn = stat.expr.func.val;
+            return fn === 'error';
+        }
+        return false;
+    };
+
+    var inferParamTypes = function(node, paramDefs) {
+        var types = {},
+            check,
+            cond;
+
+        // Infer from assertions
+        luajs.codegen.traverse(curr => {
+            // check for 'assert's that check type
+            if (curr.type === 'expr.call' && curr.func.val === 'assert') {
+                cond = curr.args[0];
+                check = getTypeCheckInfo(cond);
+                if (check) {
+                    types[check.target] = check.type;
+                }
+            } else if (curr.type === 'stat.if' && curr.cond.op === 'uop.not') {
+                // if statements throwing errors on type mismatch
+                cond = curr.cond.operand;  // non-negated version
+                // Check that it throws an error on true
+                if (curr.tblock.stats.some(isError)) {
+                    check = getTypeCheckInfo(cond);
+                    if (check) {
+                        types[check.target] = check.type;
+                    }
+                }
+            }
+        })(node);
+
+        // Infer from defaults
+        Object.keys(paramDefs).forEach(param => {
+            var val = paramDefs[param];
+            if (val) {  // initialized to 'null' doesn't help us...
+                types[param] = val.type.replace('const.', '');
+            }
+        });
+
+        return types;
+    };
+
+    var findTorchClass = function(ast){ 
+        var torchClassArgs,  // args for `torch.class(...)`
+            name = '',
+            alias,
+            baseType,
+            params,
+            setters = {},
+            defaults = {},
+            paramDefs,
+            attrDefs;
+
+        if(ast.type == 'function'){
+            ast.block.stats.forEach(function(func){
+                if(func.type == 'stat.local' && func.right && func.right[0] &&
+                  func.right[0].func && func.right[0].func.self &&
+                  func.right[0].func.self.val == 'torch' &&
+                  func.right[0].func.key.val == 'class'){
+
+                    torchClassArgs = func.right[0].args.map(arg => arg.val);
+                    name = torchClassArgs[0];
+                    if(name !== ''){
+                        name = name.replace('nn.', '');
+                        alias = func.names[0] || name;
+                        if (torchClassArgs.length > 1) {
+                            baseType = torchClassArgs[1].replace('nn.', '');
+                        }
+                    }
+                }
+            });
+        }
+
+        // Get the setters, defaults and type info (inferred)
+        var setterNames,
+            schema,
+            types,
+            values;
+
+        luajs.codegen.traverse((curr, parent) => {
+            var firstLine,
+                attrName;
+
+            // Record the setter functions
+            if (isSetterMethod(curr, parent, alias)) {
+                firstLine = curr.block.stats[0];
+                // just use the attribute attrName for now...
+                attrName = getSettingAttrName(firstLine);
+
+                // merge schemas
+                schema = getSetterSchema(firstLine, parent);
+                if (setters[attrName] && setters[attrName].setterType === 'const') {  // merge
+                    for (var val in schema.setterFn) {
+                        setters[attrName].setterFn[val] = schema.setterFn[val];
+                    }
+                } else {
+                    setters[attrName] = schema;
+                }
+            } else if (isInitFn(curr, alias)) {  // Record the defaults
+                paramDefs = getAttrsAndVals(curr);
+                attrDefs = getClassAttrDefs(curr);
+                types = inferParamTypes(curr, paramDefs);
+
+                // get ctor args
+                params = curr.func.args;
+                if(params.length === 0 && curr.func.varargs){
+                    params.push('params');
+                }
+            }
+
+        })(ast);
+
+        // Get the defaults for the params from defs
+        if (paramDefs && params) {
+            copyNodeValues(params, paramDefs, defaults);
+        }
+
+        // Get the defaults for the setters from attrDefs
+        if (attrDefs) {
+            setterNames = Object.keys(setters);
+            copyNodeValues(setterNames, attrDefs, defaults);
+        }
+
+        // Remove any const setters w/ only one value and no default
+        setterNames = Object.keys(setters);
+        for (var i = setterNames.length; i--;) {
+            schema = setters[setterNames[i]];
+            if (schema.setterType === 'const') {
+                values = Object.keys(schema.setterFn);
+                if (values.length === 1 &&
+                    // boolean setters can have the default value inferred
+                    values[0] !== 'true' && values[0] !== 'false' &&
+                    !defaults[setterNames[i]]) {
+
+                    delete setters[setterNames[i]];
+                }
+            }
+        }
+
+        return {
+            name,
+            baseType,
+            params,
+            setters,
+            types,
+            defaults
+        };
+    };
+
+    LayerParser.parse = function(text) {
+        try {
+            var ast = luajs.parser.parse(text);
+            return findTorchClass(ast);
+        } catch (e) {
+            return null;
+        }
+    };
+
+    return LayerParser;
+}));

src/common/api/APIClient.js

@@ -0,0 +1,75 @@
+/*globals define*/
+define([
+    'q',
+    'superagent'
+], function(
+    Q,
+    superagent
+) {
+    'use strict';
+
+    // Wrap the ability to read, update, and delete logs using the JobLogsAPI
+    var APIClient = function(params) {
+        params = params || {};
+
+        this.logger = this.logger || params.logger.fork('APIClient');
+
+        // Get the server url
+        this.token = params.token;
+        this.origin = this._getServerUrl(params);
+        this.relativeUrl = this.relativeUrl || '';
+        this.url = this.origin + this.relativeUrl;
+
+        this.logger.debug(`Setting url to ${this.url}`);
+
+        this.branch = params.branchName;
+        this.project = params.projectId;
+        this._modifiedJobs = [];
+
+        this.logger.debug(`Using <project>:<branch>: "${this.project}"/"${this.branch}"`);
+        this.logger.info('ctor finished');
+    };
+
+    APIClient.prototype._getServerUrl = function(params) {
+        if (typeof window !== 'undefined') {
+            return window.location.origin;
+        }
+
+        // If not in browser, set using the params
+        var server = params.server || '127.0.0.1',
+            port = params.port || '80',
+            protocol = params.httpsecure ? 'https' : 'http';  // default is http
+
+        return params.origin || `${protocol}://${server}:${port}`;
+    };
+
+    APIClient.prototype.getUrl = function() {
+        return this.url;
+    };
+
+    APIClient.prototype._request = function(method, jobId, content) {
+        var deferred = Q.defer(),
+            req = superagent[method](this.getUrl(jobId));
+
+        this.logger.debug(`sending ${method} request to ${this.getUrl(jobId)}`);
+        if (this.token) {
+            req.set('Authorization', 'Bearer ' + this.token);
+        }
+
+        if (content) {
+            req = req.send(content);
+        }
+
+        req.end((err, res) => {
+            if (err || res.status > 399) {
+                return deferred.reject(res || err);
+            }
+
+            return deferred.resolve(res);
+        });
+
+        return deferred.promise;
+    };
+
+    return APIClient;
+});

src/common/api/ExecPulseClient.js

@@ -0,0 +1,44 @@
+/* globals define */
+define([
+    './APIClient'
+], function(
+    APIClient
+) {
+    'use strict';
+
+    var ExecPulseClient = function(params) {
+        this.relativeUrl = '/execution/pulse/';
+        this.logger = params.logger.fork('ExecPulseClient');
+        APIClient.call(this, params);
+    };
+
+    ExecPulseClient.prototype = Object.create(APIClient.prototype);
+
+    ExecPulseClient.prototype.getUrl = function(hash) {
+        return this.url + hash;
+    };
+
+    // - update the heartbeat
+    // - check the heartbeat
+    // - delete the heartbeat
+    ExecPulseClient.prototype.update = function(hash) {
+        return this._request('post', hash)
+            .catch(err => {
+                throw err.text || err;
+            });
+    };
+
+    ExecPulseClient.prototype.check = function(hash) {
+        return this._request('get', hash)
+            .then(res => JSON.parse(res.text))
+            .catch(err => {
+                throw err.text || err;
+            });
+    };
+
+    ExecPulseClient.prototype.clear = function(hash) {
+        return this._request('delete', hash);
+    };
+
+    return ExecPulseClient;
+});

src/common/api/JobLogsClient.js

@@ -0,0 +1,120 @@
+/* globals define */
+define([
+    './APIClient',
+    'q',
+    'superagent'
+], function(
+    APIClient,
+    Q,
+    superagent
+) {
+    'use strict';
+
+    // Wrap the ability to read, update, and delete logs using the JobLogsAPI
+    var METADATA_FIELDS = [
+        'lineCount'
+    ];
+    var JobLogsClient = function(params) {
+        params = params || {};
+
+        this.relativeUrl = '/execution/logs';
+        this.logger = params.logger.fork('JobLogsClient');
+        APIClient.call(this, params);
+
+        // Get the project, branch name
+        if (!(params.branchName && params.projectId)) {
+            throw Error('"branchName" and "projectId" required');
+        }
+
+        this._modifiedJobs = [];
+
+        this.logger.debug(`Using <project>:<branch>: "${this.project}"/"${this.branch}"`);
+        this.logger.info('ctor finished');
+    };
+
+    JobLogsClient.prototype = Object.create(APIClient.prototype);
+
+    // This method could be optimized - it could make a log of requests
+    JobLogsClient.prototype.fork = function(forkName) {
+        var jobIds = this._modifiedJobs,
+            deferred = Q.defer(),
+            url = [
+                this.url,
+                'migrate',
+                encodeURIComponent(this.project),
+                encodeURIComponent(this.branch),
+                encodeURIComponent(forkName)
+            ].join('/'),
+            req = superagent.post(url);
+
+        this.logger.info(`migrating ${jobIds.length} jobs from ${this.branch} to ${forkName} in ${this.project}`);
+        if (this.token) {
+            req.set('Authorization', 'Bearer ' + this.token);
+        }
+
+        req.send({jobs: jobIds})
+            .end((err, res) => {
+                if (err || res.status > 399) {
+                    return deferred.reject(err || res.status);
+                }
+
+                return deferred.resolve(res);
+            });
+
+        this.branch = forkName;
+        return deferred.promise;
+    };
+
+    JobLogsClient.prototype.getUrl = function(jobId) {
+        var url = this.url;
+            
+        if (typeof jobId !== 'string') {
+            url = this.url + jobId.route;
+            jobId = jobId.jobId;
+        }
+
+        return [
+            url,
+            encodeURIComponent(this.project),
+            encodeURIComponent(this.branch),
+            encodeURIComponent(jobId)
+        ].join('/');
+    };
+
+    var hasRequiredFields = function(md) {
+        return METADATA_FIELDS.reduce((passing, nextField) => {
+            return passing && md.hasOwnProperty(nextField);
+        }, true);
+    };
+
+    JobLogsClient.prototype.appendTo = function(jobId, text, metadata) {
+        this._modifiedJobs.push(jobId);
+        this.logger.info(`Appending logs to ${jobId}`);
+
+        if (metadata && !hasRequiredFields(metadata)) {
+            throw Error(`Required metadata fields: ${METADATA_FIELDS.join(', ')}`);
+        }
+        metadata = metadata || {};
+        metadata.patch = text;
+        return this._request('patch', jobId, metadata);
+    };
+
+    JobLogsClient.prototype.getLog = function(jobId) {
+        this.logger.info(`Getting logs for ${jobId}`);
+        return this._request('get', jobId)
+            .then(res => res.text);
+    };
+
+    JobLogsClient.prototype.deleteLog = function(jobId) {
+        this.logger.info(`Deleting logs for ${jobId}`);
+        return this._request('delete', jobId);
+    };
+
+    JobLogsClient.prototype.getMetadata = function(jobId) {
+        this.logger.info(`Getting line count for ${jobId}`);
+        return this._request('get', {jobId: jobId, route: '/metadata'})
+            .then(res => JSON.parse(res.text));
+    };
+
+    return JobLogsClient;
+});

src/common/api/JobOriginClient.js

@@ -0,0 +1,60 @@
+/* globals define */
+define([
+    './APIClient'
+], function(
+    APIClient
+) {
+    'use strict';
+
+    var JobOriginClient = function(params) {
+        this.relativeUrl = '/job/origins/';
+        this.logger = params.logger.fork('JobOriginClient');
+        APIClient.call(this, params);
+    };
+
+    JobOriginClient.prototype = Object.create(APIClient.prototype);
+
+    // - Record the origin
+    // - Look up the origin
+    // - Delete record
+    JobOriginClient.prototype.getUrl = function(hash) {
+        return this.url + hash;
+    };
+
+    JobOriginClient.prototype.record = function(hash, info) {
+        var jobInfo = {
+            hash: hash,
+            nodeId: info.nodeId,
+            job: info.job,
+            project: info.project || this.project,
+            branch: info.branch || this.branch,
+            execution: info.execution
+        };
+
+        return this._request('post', hash, jobInfo)
+            .catch(err => {
+                throw err.text || err;
+            });
+    };
+
+    JobOriginClient.prototype.getOrigin = function(hash) {
+        return this._request('get', hash)
+            .then(res => JSON.parse(res.text))
+            .catch(res => {
+                if (res.status && res.status === 404) {
+                    return null;
+                }
+                throw res;
+            });
+    };
+
+    JobOriginClient.prototype.fork = function(hash, forkName) {
+        return this._request('patch', hash, {branch: forkName});
+    };
+
+    JobOriginClient.prototype.deleteRecord = function(hash) {
+        return this._request('delete', hash);
+    };
+
+    return JobOriginClient;
+});

src/common/globals.js

@@ -0,0 +1,325 @@
+/* globals WebGMEGlobal, define*/
+// This file creates the DeepForge namespace and defines basic actions
+define([
+    'panel/FloatingActionButton/styles/Materialize',
+    'js/RegistryKeys',
+    'js/Panels/MetaEditor/MetaEditorConstants',
+    'js/Constants',
+    'q'
+], function(
+    Materialize,
+    REGISTRY_KEYS,
+    META_CONSTANTS,
+    CONSTANTS,
+    Q
+) {
+    var DeepForge = {},
+        placesTerritoryId,
+        client = WebGMEGlobal.Client,
+        PLACE_NAMES;
+
+    // Helper functions
+    var addToMetaSheet = function(nodeId, metasheetName) {
+        var root = client.getNode(CONSTANTS.PROJECT_ROOT_ID),
+            metatabs = root.getRegistry(REGISTRY_KEYS.META_SHEETS),
+            metatab = metatabs.find(tab => tab.title === metasheetName) || metatabs[0],
+            metatabId = metatab.SetID;
+
+        // Add to the general meta
+        client.addMember(
+            CONSTANTS.PROJECT_ROOT_ID,
+            nodeId,
+            META_CONSTANTS.META_ASPECT_SET_NAME
+        );
+        client.setMemberRegistry(
+            CONSTANTS.PROJECT_ROOT_ID,
+            nodeId,
+            META_CONSTANTS.META_ASPECT_SET_NAME,
+            REGISTRY_KEYS.POSITION,
+            {
+                x: 100,
+                y: 100
+            }
+        );
+
+        // Add to the specific sheet
+        client.addMember(CONSTANTS.PROJECT_ROOT_ID, nodeId, metatabId);
+        client.setMemberRegistry(
+            CONSTANTS.PROJECT_ROOT_ID,
+            nodeId,
+            metatabId,
+            REGISTRY_KEYS.POSITION,
+            {
+                x: 100,
+                y: 100
+            }
+        );
+    };
+
+    var createNamedNode = function(baseId, parentId, isMeta) {
+        var newId = client.createNode({parentId, baseId}),
+            baseNode = client.getNode(baseId),
+            basename = 'New' + baseNode.getAttribute('name'),
+            newName = getUniqueName(parentId, basename);
+
+        // If instance, make the first char lowercase
+        if (!isMeta) {
+            newName = newName.substring(0, 1).toLowerCase() + newName.substring(1);
+        }
+
+        // Set isAbstract false, if needed
+        if (baseNode.getRegistry('isAbstract')) {
+            client.setRegistry(newId, 'isAbstract', false);
+        }
+
+        client.setAttribute(newId, 'name', newName);
+        return newId;
+    };
+
+    var getUniqueName = function(parentId, basename) {
+        var pNode = client.getNode(parentId),
+            children = pNode.getChildrenIds().map(id => client.getNode(id)),
+            name = basename,
+            exists = {},
+            i = 2;
+
+        children.forEach(child => exists[child.getAttribute('name')] = true);
+
+        while (exists[name]) {
+            name = basename + '_' + i;
+            i++;
+        }
+
+        return name;
+    };
+
+    //////////////////// DeepForge places detection ////////////////////
+    DeepForge.places = {};
+    var TYPE_TO_CONTAINER = {
+        
+        Architecture: 'MyArchitectures',
+        Pipeline: 'MyPipelines',
+        Execution: 'MyExecutions',
+        Layer: 'MyLayers',
+        Artifact: 'MyArtifacts',
+        Operation: 'MyOperations',
+        Primitive: 'MyDataTypes',
+        Complex: 'MyDataTypes'
+    };
+
+    PLACE_NAMES = Object.keys(TYPE_TO_CONTAINER).map(key => TYPE_TO_CONTAINER[key]);
+
+    // Add DeepForge directories
+    var placePromises = {},
+        setPlaceId = {},
+        firstProject = true;
+
+    var getPlace = function(name) {
+        return placePromises[name];
+    };
+
+    var initializePlaces = function() {
+        PLACE_NAMES.forEach(name => {
+            var deferred = Q.defer();
+            placePromises[name] = deferred.promise;
+            setPlaceId[name] = deferred.resolve;
+        });
+    };
+
+    var updateDeepForgeNamespace = function() {
+        var territory = {};
+
+        if (!firstProject) {
+            initializePlaces();
+        }
+        firstProject = false;
+
+        // Create a territory
+        if (placesTerritoryId) {
+            client.removeUI(placesTerritoryId);
+        }
+
+        territory[CONSTANTS.PROJECT_ROOT_ID] = {children: 1};
+        placesTerritoryId = client.addUI(null, updateDeepForgePlaces);
+
+        // Update the territory (load the main places)
+        client.updateTerritory(placesTerritoryId, territory);
+    };
+
+    var updateDeepForgePlaces = function(events) {
+        var nodeIdsByName = {},
+            nodes;
+
+        nodes = events
+            // Remove root node, complete event and update/unload events
+            .filter(event => event.eid && event.eid !== CONSTANTS.PROJECT_ROOT_ID)
+            .filter(event => event.etype === CONSTANTS.TERRITORY_EVENT_LOAD)
+            .map(event => client.getNode(event.eid));
+
+        nodes.forEach(node =>
+            nodeIdsByName[node.getAttribute('name')] = node.getId());
+
+        PLACE_NAMES.forEach(name => setPlaceId[name](nodeIdsByName[name]));
+        
+        // Remove the territory
+        client.removeUI(placesTerritoryId);
+        placesTerritoryId = null;
+    };
+
+    initializePlaces();
+    PLACE_NAMES.forEach(name => DeepForge.places[name] = getPlace.bind(null, name));
+
+    //////////////////// DeepForge creation actions ////////////////////
+    var instances = [
+            'Architecture',
+            'Pipeline'
+        ],
+        metaNodes = [
+            'Operation',
+            'Primitive',
+            'Complex'
+        ];
+
+    var createNew = function(type, metasheetName) {
+        var placeName = TYPE_TO_CONTAINER[type],
+            newId,
+            baseId,
+            msg = `Created new ${type + (metasheetName ? ' prototype' : '')}`;
+
+        baseId = client.getAllMetaNodes()
+                .find(node => node.getAttribute('name') === type)
+                .getId();
+
+        // Look up the parent container
+        DeepForge.places[placeName]().then(parentId => {
+
+            client.startTransaction(msg);
+            newId = createNamedNode(baseId, parentId, !!metasheetName);
+
+            if (metasheetName) {
+                addToMetaSheet(newId, metasheetName);
+            }
+
+            client.completeTransaction();
+
+            WebGMEGlobal.State.registerActiveObject(newId);
+            return newId;
+        });
+    };
+
+    var createCustomLayer = function(typeName) {
+        var metanodes = client.getAllMetaNodes(),
+            msg = `Created new custom ${typeName} layer`,
+            newId,
+            customLayerId,
+            baseId,
+            name,
+            i = metanodes.length;
+
+        while (i-- && !(baseId && customLayerId)) {
+            name = metanodes[i].getAttribute('name');
+            if (name === 'CustomLayer') {
+                customLayerId = metanodes[i].getId();
+            } else if (name === typeName) {
+                baseId = metanodes[i].getId();
+            }
+        }
+
+        return DeepForge.places.MyLayers()
+            .then(id => {
+
+                client.startTransaction(msg);
+
+                newId = createNamedNode(baseId, id, true);
+                addToMetaSheet(newId, 'CustomLayers');
+                client.addMixin(newId, customLayerId);
+                client.setRegistry(newId, REGISTRY_KEYS.IS_ABSTRACT, false);
+
+                client.completeTransaction();
+
+                WebGMEGlobal.State.registerActiveObject(newId);
+            });
+    };
+
+    // Creating Artifacts
+    var UPLOAD_PLUGIN = 'ImportArtifact',
+        DATA_TYPE_CONFIG = {
+            name: 'dataTypeId',
+            displayName: 'Data Type Id',
+            valueType: 'string',
+            valueItems: []
+        };
+
+    var uploadArtifact = function() {
+        // Get the data types
+        var dataBase,
+            dataBaseId,
+            metanodes = client.getAllMetaNodes(),
+            dataTypes = [];
+
+        dataBase = metanodes.find(n => n.getAttribute('name') === 'Data');
+
+        if (!dataBase) {
+            this.logger.error('Could not find the base Data node!');
+            return;
+        }
+
+        dataBaseId = dataBase.getId();
+        dataTypes = metanodes.filter(n => n.isTypeOf(dataBaseId))
+            .filter(n => !n.getRegistry('isAbstract'))
+            .map(node => node.getAttribute('name'));
+
+        //this.logger.info(`Found ${dataTypes.length} data types`);
+
+        // Add the target type to the pluginMetadata... hacky :/
+        var metadata = WebGMEGlobal.allPluginsMetadata[UPLOAD_PLUGIN], 
+            config = metadata.configStructure
+                .find(opt => opt.name === DATA_TYPE_CONFIG.name);
+
+        if (!config) {
+            config = DATA_TYPE_CONFIG;
+            WebGMEGlobal.allPluginsMetadata[UPLOAD_PLUGIN].configStructure.push(config);
+        }
+
+        config.valueItems = dataTypes;
+        config.value = dataTypes[0];
+
+        WebGMEGlobal.InterpreterManager.configureAndRun(metadata, (result) => {
+            var msg = 'Artifact upload complete!';
+            if (!result) {
+                return;
+            }
+            if (!result.success) {
+                msg = `Artifact upload failed: ${result.error}`;
+            }
+            Materialize.toast(msg, 2000);
+        });
+    };
+
+    DeepForge.last = {};
+    DeepForge.create  = {};
+    instances.forEach(type => {
+        DeepForge.create[type] = function() {
+            return createNew.call(null, type);
+        };
+    });
+
+    metaNodes.forEach(type => {
+        DeepForge.create[type] = function() {
+            return createNew.call(null, type, type);
+        };
+    });
+
+    DeepForge.create.Layer = createCustomLayer;
+    DeepForge.create.Artifact = uploadArtifact;
+
+    //////////////////// DeepForge prev locations ////////////////////
+    // Update DeepForge on project changed
+    WebGMEGlobal.State.on('change:' + CONSTANTS.STATE_ACTIVE_PROJECT_NAME,
+        updateDeepForgeNamespace, null);
+
+    // define DeepForge globally
+    window.DeepForge = DeepForge;
+
+    return DeepForge;
+});

src/common/layer-args.js

@@ -1,5 +1,8 @@
+/* globals define*/
 define([
+    'deepforge/Constants'
 ], function(
+    Constants
 ) {
     'use strict';
 
@@ -14,33 +17,49 @@ define([
         return result;
     };
 
-    var isArgument = function(arg) {
-        return arg.hasOwnProperty('argindex');
-    };
-
-    var sortByIndex = function(a, b) {
-        return a.argindex > b.argindex;
+    var isSetter = function(arg) {
+        return arg.hasOwnProperty('setterType');
     };
 
     var createLayerDict = function(core, meta) {
         var node,
             names = Object.keys(meta),
-            attributes,
-            layers = {};
+            layers = {},
+            setters,
+            ctorData,
+            ctorArgs,
+            attrs;
 
         for (var i = names.length; i--;) {
             node = meta[names[i]];
-            layers[names[i]] = core.getValidAttributeNames(node)
+            ctorData = core.getAttribute(node, Constants.CTOR_ARGS_ATTR);
+            attrs = core.getValidAttributeNames(node);
+
+            layers[names[i]] = {};
+            if (ctorData) {
+                ctorArgs = ctorData.split(',')
+                    .map(attr => prepAttribute(core, node, attr));
+
+                // Get the constructor args
+                layers[names[i]].args = ctorArgs;
+            } else {
+                layers[names[i]].args = [];
+            }
+
+            layers[names[i]].setters = {};
+            setters = attrs
                 .map(attr => prepAttribute(core, node, attr))
-                .filter(isArgument)
-                .sort(sortByIndex);
+                .filter(isSetter);
+            for (var j = setters.length; j--;) {
+                layers[names[i]].setters[setters[j].name] = setters[j];
+            }
         }
 
         return layers;
     };
 
     // When provided with the META, create the given LayerDict object
-    //  - Sort (and filter) by argindex
+    //  - Filter out the ctor args (in order)
     //  - add name attribute to schema
     //  - store this array under the META name
 

src/common/layers.yml

@@ -1,546 +0,0 @@
-# This file should actually be an alternative way of viewing the metamodel. 
-#
-# This contains metadata about the Torch nn library used for
-# creating the metamodel
-#
-# By default...
-#   - all attributes are a number
-#   - default values are optional
-#       - all booleans default to false
-#   - list attributes are specified with WORD...WORD
-#   - if `ignore` is set, the attribute is not added to the metamodel
-
-# This should have tests to verify that this document is up to date...
-# TODO
-
-Containers:
-    - Concat:
-        - dim:
-            min: 1  # TODO: Figure out exactly how this works
-
-Module:
-    - SpatialBatchNormalization:
-        - input:
-            infer: dimensionality  # change this to `infer: 'dimensionality'`
-        - eps:
-            default: 0.00001
-        - momentum:
-            default: 0.1
-        - affine:
-            default: true
-
-    - BatchNormalization:
-        - input:
-            infer: dimensionality  # change this to `infer: 'dimensionality'`
-        - eps:
-            default: 0.00001
-        - momentum:
-            default: 0.1
-        - affine:
-            default: true
-
-    - Threshold:
-        - threshold:
-            type: float
-            default: 1e-6
-        - value:
-            type: float
-            default: 0
-        - inplace:
-            type: boolean
-            default: false
-
-ConvLayer:
-    - TemporalConvolution:
-        - inputFrameSize:
-            min: 1
-        - outputFrameSize:
-            min: 1
-        - kernelWidth:
-            min: 1
-        - step:
-            default: 1
-
-    - TemporalMaxPooling:
-        - kernelWidth:
-            min: 1
-        - step:  # FIXME: defaults to 'kernelWidth'
-            min: 1
-
-    - TemporalSubSampling:
-        - inputFrameSize:
-            min: 1
-        - kernelWidth:
-            min: 1
-        - step:
-            min: 1
-            # TODO: What is the default?
-
-    - LookupTable:
-        - nIndex:
-            min: 1
-        - sizes:
-            min: 1
-
-# Spatial Modules
-    - SpatialConvolutionMM:
-        - nInputPlane:  # TODO: Infer this
-            min: 1
-        - nOutputPlane:
-            min: 1
-        - kernelWidth:
-            min: 1
-        - kernelHeight:
-            min: 1
-
-        - strideWidth:
-            min: 1
-            default: 1
-        - strideHeight:
-            min: 1
-            default: 1
-        - padWidth:
-            min: 0
-            default: 0
-        - padHeight:  # FIXME: this defaults to padWidth - not 0
-            min: 0
-            default: 0
-
-    - SpatialConvolution:
-        - nInputPlane:
-            min: 1
-        - nOutputPlane:
-            min: 1
-        - kernelWidth:
-            min: 1
-        - kernelHeight:
-            min: 1
-        - strideWidth:
-            min: 1
-            default: 1
-        - strideHeight:
-            min: 1
-            default: 1
-
-    - SpatialConvolutionMap:
-        - connectionMatrix:
-            min: 1
-        - kernelWidth:
-            min: 1
-        - kernelHeight:
-            min: 1
-        - strideWidth:
-            min: 1
-            default: 1
-        - strideHeight:
-            min: 1
-            default: 1
-
-    - SpatialLPPooling:
-        - nInputPlane:
-            min: 1
-        - norm:
-            min: 1
-        - kernelWidth:
-            min: 1
-        - kernelHeight:
-            min: 1
-        - strideWidth:
-            min: 1
-            default: 1
-        - strideHeight:
-            min: 1
-            default: 1
-
-    - SpatialMaxPooling:
-        - kernelWidth:
-            min: 1
-        - kernelHeight:
-            min: 1
-        - strideWidth:
-            min: 1
-            default: 1
-        - strideHeight:
-            min: 1
-            default: 1
-
-    - SpatialAveragePooling:
-        - kernelWidth:
-            min: 1
-        - kernelHeight:
-            min: 1
-        - strideWidth:
-            min: 1
-            default: 1
-        - strideHeight:
-            min: 1
-            default: 1
-
-    - SpatialAdaptiveMaxPooling:  # output is width x height
-        - width:
-            min: 1
-        - height:
-            min: 1
-
-    - SpatialSubSampling:
-        - nInputPlane:
-            min: 1
-        - kernelWidth:
-            min: 1
-        - kernelHeight:
-            min: 1
-        - strideWidth:
-            min: 1
-            default: 1
-        - strideHeight:
-            min: 1
-            default: 1
-
-    - SpatialUpSamplingNearest:
-        - scale:  # upscale ratio
-            min: 1
-
-    - SpatialZeroPadding:
-        - left:
-            min: 0
-        - right:
-            min: 0
-        - top:
-            min: 0
-        - bottom:
-            min: 0
-
-    - SpatialSubtractiveNormalization:
-        - nInputPlane:
-            min: 1
-        - kernel:
-            min: 1
-
-    - SpatialCrossMapLRN:
-        - size:
-            min: 1
-        - alpha:
-            default: 0.0001
-        - beta:
-            default: 0.75
-        - k:
-            default: 1
-
-    - SpatialConvolutionLocal:
-        - nInputPlane:
-            min: 1
-        - nOutputPlane:
-            min: 1
-        - inputWidth:  # TODO: infer this
-            min: 1
-        - inputHeight:  # TODO: infer this
-            min: 1
-        - kernelWidth:
-            min: 1
-        - kernelHeight:
-            min: 1
-        - strideWidth:
-            min: 1
-            default: 1
-        - strideHeight:
-            min: 1
-            default: 1
-        - padWidth:
-            min: 0
-            default: 0
-        - padHeight:  # FIXME: this defaults to padWidth - not 0
-            min: 0
-            default: 0
-
-    - SpatialDropout:
-        - probability:
-            default: 0.5
-
-    - SpatialFractionalMaxPooling:
-        - poolWidth:
-            - min: 2
-        - poolHeight:
-            - min: 2
-        - outWidth:  # Optionally, these could be ratioW/H FIXME
-            - min: 1
-        - outHeight:
-            - min: 1
-
-    - SpatialDivisiveNormalization:
-        - nInputPlane:  # TODO: infer this
-            - default: 1
-        - kernel:  # TODO: this is a tensor type...
-        - threshold:
-            - default: 0.0001
-        - thresval:
-            - default: 0.0001  # FIXME: this defaults to "threshold"
-
-    - SpatialContrastiveNormalization:
-        - nInputPlane:  # TODO: infer this
-            - default: 1
-        - kernel:  # TODO: this is a tensor type...
-        - threshold:
-            - default: 0.0001
-        - thresval:
-            - default: 0.0001  # FIXME: this defaults to "threshold"
-
-    - SpatialFullConvolution:
-        - nInputPlane:  # TODO: should infer this
-            min: 1
-        - nOutputPlane:
-            min: 1
-        - kernelWidth:
-            min: 1
-        - kernelHeight:
-            min: 1
-        - strideWidth:
-            min: 1
-            default: 1
-        - strideHeight:
-            min: 1
-            default: 1
-        - padWidth:
-            min: 0
-            default: 0
-        - padHeight:
-            min: 0
-            default: 0
-        - adjWidth:
-            min: 0
-            default: 0
-        - adjHeight:
-            min: 0
-            default: 0
-        # Additional constraint:
-
-# Volumetric Modules
-    - VolumetricConvolution:
-        - nInputPlane:
-            min: 1
-        - nOutputPlane:
-            min: 1
-        - kernelTime:
-            min: 1
-        - kernelWidth:
-            min: 1
-        - kernelHeight:
-            min: 1
-        - strideTime:
-            min: 1
-            default: 1
-        - strideWidth:
-            min: 1
-            default: 1
-        - strideHeight:
-            min: 1
-            default: 1
-    - VolumetricMaxPooling:
-        - kernelTime:
-            min: 1
-        - kernelWidth:
-            min: 1
-        - kernelHeight:
-            min: 1
-        - strideTime:
-            min: 1
-            default: 1
-        - strideWidth:
-            min: 1
-            default: 1
-        - strideHeight:
-            min: 1
-            default: 1
-
-SimpleLayer:
-    - Linear:  # FIXME: These should contain the actual args
-        - input:
-            infer: dimensionality
-        - output:
-            min: 1
-    - SparseLinear: 
-        - input: 
-            infer: dimensionality
-        - output:
-            min: 1
-
-    - Dropout: 
-        - probability:
-            type: float
-    - Abs:
-    - Add:
-        - isScalar:
-            type: boolean
-    - Mul:
-    - CMul:
-        - size: null
-    - Max:
-        - dimension:
-            min: 0
-    - Min:
-        - dimension:
-            min: 0
-    - Mean:
-        - dimension:
-            min: 0
-    - Sum:
-        - dimension:
-            min: 0
-
-    - Euclidean:
-        - output:
-            min: 0
-    - WeightedEuclidean:
-        - output:
-            min: 0
-    - Identity:
-    - Copy:  # Casts types
-        - inputType:
-            type: string
-        - outputType:
-            type: string
-        - forceCopy:
-            type: boolean
-    - Narrow:
-        - dimension:
-            min: 0
-        - offset:
-            min: 0
-        - length:
-            min: 0
-    - Replicate:
-        - nFeature:
-            min: 0
-    - Reshape:
-        - dimensions:
-            type: list
-    - View:
-        - sizes:  # list
-            type: list
-            min: 0
-    - Select:
-        - dimensions:
-            type: list
-    - Exp
-    - Square
-    - Sqrt
-    - Power:
-        - p: null
-    - MM:
-        - transA:
-            type: boolean
-        - transB:
-            type: boolean
-
-TransferLayer:
-    - HardTanh
-    - HardShrink:
-        - lambda:
-            type: float
-    - SoftShrink:
-        - lambda:
-            type: float
-    - SoftMax
-    - SoftMin
-    - SoftPlus
-    - SoftSign  # Typo in the docs on this one
-    - LogSigmoid
-    - LogSoftMax  # Also in Criterion?
-    - Sigmoid
-    - Tanh
-    - ReLU
-    - PReLU  # Missing from docs
-    - RReLU  # Missing from docs
-    - LeakyReLU  # Missing from docs
-    - AddConstant:
-        - scalar:
-            type: float
-    - MulConstant:
-        - scalar:
-            type: float
-            min: 1
-        - inplace:
-            default: false
-
-Criterion:
-    - BCECriterion
-    - WeightedMSECriterion
-    - SmoothL1Criterion
-    - MSECriterion
-    - AbsCriterion
-    - MultiCriterion
-    - DistKLDivCriterion
-    - HingeEmbeddingCriterion
-    - CriterionTable
-    - MultiMarginCriterion
-    - MultiLabelMarginCriterion
-    - L1HingeEmbeddingCriterion
-    - CosineEmbeddingCriterion
-    - MarginRankingCriterion
-    - CrossEntropyCriterion
-    - MarginCriterion
-    - ClassNLLCriterion
-    - ParallelCriterion
-
-MiscLayers:
-    - Jacobian
-    - ConcatTable
-    - CMulTable
-    - CAddTable
-    - TanhShrink
-    - Padding:
-        - dim:
-        - pad:
-            min: 0
-        - nInputDim:  #  TODO: infer?
-            min: 1
-        - value:
-            min: 0
-            default: 0
-
-# TODO: Add the following layers
-#VolumetricMaxUnpooling
-    # Takes a poolingModule as an arg...
-
-#MixtureTable
-#NarrowTable
-#SplitTable
-#DotProduct
-#DepthConcat
-#Parallel
-#Log
-#hessian
-#ELU
-#CSubTable
-#VolumetricAveragePooling
-#StochasticGradient
-#Bilinear
-#VolumetricFullConvolution
-#SparseJacobian
-#Contiguous
-#L1Cost
-#JoinTable
-#CosineDistance
-#Index
-#L1Penalty
-#Cosine
-#Clamp
-#SpatialConvolutionMM
-#LogSigmoid
-#ParallelTable
-#CDivTable
-#SpatialFullConvolutionMap
-#GradientReversal
-#SpatialMaxUnpooling
-#Transpose
-#Normalize
-#SpatialSoftMax
-#SelectTable
-#FlattenTable
-
-# CONTAINERS and TableLayouts
-# Some of these are captured by the visual structure of the architecture and are not needed
-# as explicit layers in the metamodel
-#TableLayer:
-#- ConcatTable
-#Container:
-

src/common/lua.js

@@ -2687,6 +2687,7 @@ function LuaContext(){
         }
         exports.stdlib(_G, helpers)();
     }
+    this.__helpers = helpers;
 }
 
 LuaContext.prototype = {}

src/common/plugin/LocalExecutor.js

@@ -0,0 +1,197 @@
+/* globals define*/
+// This is an 'executor' containing the implementations of all local operations
+// These are all primitives in DeepForge
+define([
+    'deepforge/Constants'
+], function(
+    CONSTANTS
+) {
+    'use strict';
+    var LocalExecutor = function() {
+    };
+
+    // Should these be in lua?
+    LocalExecutor.prototype[CONSTANTS.OP.INPUT] = function(node) {
+        // Get the hash from the output node
+        var hash;
+        return this.core.loadChildren(node)
+            .then(cntrs => {
+                // Get the output container and load it's children
+                var output = cntrs
+                    .find(cntr => {
+                        var metaNode = this.core.getMetaType(cntr),
+                            metaName = this.getAttribute(metaNode, 'name');
+                        return metaName === 'Outputs';
+                    });
+                return this.core.loadChildren(output);
+            })
+            .then(dataNodes => {
+                hash = this.getAttribute(dataNodes[0], 'data');
+                return this.getOutputs(node);
+            })
+            .then(outputTuples => {
+                var outputs = outputTuples.map(tuple => tuple[2]),
+                    paths;
+
+                paths = outputs.map(output => this.core.getPath(output));
+                // Get the 'data' hash and store it in the output data ports
+                this.logger.info(`Loading blob data (${hash}) to ${paths.map(p => `"${p}"`)}`);
+                outputs.forEach(output => this.core.setAttribute(output, 'data', hash));
+
+                this.onOperationComplete(node);
+            });
+    };
+
+    LocalExecutor.prototype.ArtifactFinder = function(node) {
+        // Check the save dir for a node with the given name
+        // that has the given type
+        var hash,
+            typeId = this.core.getPointerPath(node, 'type'),
+            type,
+            artifactName = this.getAttribute(node, 'artifactName');
+
+        return this.core.loadByPath(this.rootNode, typeId)
+            .then(_type => {
+                type = _type;
+                return this._getSaveDir();
+            })
+            .then(saveDir => this.core.loadChildren(saveDir))
+            .then(artifacts => {
+                return artifacts.find(artifact =>
+                    this.getAttribute(artifact, 'name') === artifactName &&
+                        this.isMetaTypeOf(artifact, type));
+            })
+            .then(matchingArtifact => {
+                hash = matchingArtifact && this.getAttribute(matchingArtifact, 'data');
+                // If no hash, just continue (the subsequent ops will receive 'nil')
+                if (!hash) {
+                    return this.onOperationComplete(node);
+                } else {
+                    return this.getOutputs(node)
+                        .then(outputPairs => {
+                            var outputs = outputPairs.map(pair => pair[2]);
+                            // Get the 'data' hash and store it in the output data ports
+                            outputs.forEach(output => this.setAttribute(output, 'data', hash));
+
+                            this.onOperationComplete(node);
+                        });
+                }
+            });
+    };
+
+    LocalExecutor.prototype._getSaveDir = function () {
+        return this.core.loadChildren(this.rootNode)
+            .then(children => {
+                var execPath = this.core.getPath(this.META.Data),
+                    containers,
+                    saveDir;
+
+                // Find a node in the root that can contain only executions
+                containers = children.filter(child => {
+                    var metarule = this.core.getChildrenMeta(child);
+                    return metarule && metarule[execPath];
+                });
+
+                if (containers.length > 1) {
+                    saveDir = containers.find(c =>
+                        this.getAttribute(c, 'name').toLowerCase().indexOf('artifacts') > -1
+                    ) || containers[0];
+                }
+
+                return saveDir || this.rootNode;  // default to rootNode
+            });
+    };
+
+    LocalExecutor.prototype[CONSTANTS.OP.OUTPUT] = function(node) {
+        var parentNode,
+            currNameHashPairs;
+        
+        // Get the input node
+        this.logger.info('Calling save operation!');
+        return this._getSaveDir()
+            .then(_saveDir => {
+                parentNode = _saveDir;
+                return this.core.loadChildren(_saveDir);
+            })
+            .then(artifacts => {
+                currNameHashPairs = artifacts
+                    .map(node => [
+                        this.getAttribute(node, 'name'),
+                        this.getAttribute(node, 'data')
+                    ]);
+                return this.getInputs(node);
+            })
+            .then(inputs => {
+                var ids = inputs.map(i => this.core.getPath(i[2])),
+                    allDataNodes,
+                    dataNodes;
+
+                allDataNodes = Object.keys(this.nodes)
+                    .map(id => this.nodes[id])
+                    .filter(node => this.isMetaTypeOf(node, this.META.Transporter))
+                    .filter(node => 
+                        ids.indexOf(this.core.getPointerPath(node, 'dst')) > -1
+                    )
+                    .map(node => this.core.getPointerPath(node, 'src'))
+                    .map(id => this.nodes[id]);
+
+                // Remove nodes that already exist
+                dataNodes = allDataNodes.filter(dataNode => {
+                    var hash = this.getAttribute(dataNode, 'data'),
+                        name = this.core.getOwnAttribute(node, 'saveName') ||
+                            this.getAttribute(dataNode, 'name');
+
+                    return !(currNameHashPairs
+                        .find(pair => pair[0] === name && pair[1] === hash));
+                });
+
+                // get the input node
+                if (dataNodes.length !== 0) {
+                    var newNodes = this.core.copyNodes(dataNodes, parentNode),
+                        newName = this.core.getOwnAttribute(node, 'saveName'),
+                        createdAt = Date.now();
+
+                    if (newName) {
+                        newNodes.forEach(node => {
+                            this.setAttribute(node, 'name', newName);
+                            this.setAttribute(node, 'createdAt', createdAt);
+                        });
+                    }
+                    var hashes = dataNodes.map(n => this.getAttribute(n, 'data'));
+                    this.logger.info(`saving hashes: ${hashes.map(h => `"${h}"`)}`);
+                } else if (allDataNodes.length === 0) {
+                    this.logger.warn('No data nodes found!');
+                } else {
+                    this.logger.info('Using cached artifact(s)');
+                }
+
+                this.onOperationComplete(node);
+            });
+    };
+
+    // Helper methods
+    LocalExecutor.prototype.getLocalOperationType = function(node) {
+        var type;
+        for (var i = LocalExecutor.OPERATIONS.length; i--;) {
+            type = LocalExecutor.OPERATIONS[i];
+            if (!this.META[type]) {
+                this.logger.warn(`Missing local operation: ${type}`);
+                continue;
+            }
+            if (this.isMetaTypeOf(node, this.META[type])) {
+                return type;
+            }
+        }
+        return null;
+    };
+
+    LocalExecutor.prototype.isLocalOperation = function(node) {
+        return !!this.getLocalOperationType(node);
+    };
+
+    LocalExecutor.OPERATIONS = Object.keys(LocalExecutor.prototype)
+        .filter(name => name.indexOf('_') !== 0)
+        .filter(name => name !== 'isLocalOperation' && name !== 'getLocalOperationType');
+    
+    return LocalExecutor;
+});

src/common/plugin/PtrCodeGen.js

@@ -0,0 +1,83 @@
+/*globals define, requirejs*/
+define([
+    'plugin/util',
+    'q'
+], function(
+    PluginUtils,
+    Q
+) {
+    var PtrCodeGen = function() {
+    };
+
+    PtrCodeGen.prototype.getPtrCodeHash = function(ptrId) {
+        return this.core.loadByPath(this.rootNode, ptrId)
+            .then(ptrNode => {
+                // Look up the plugin to use
+                var metanode = this.core.getMetaType(ptrNode),
+                    pluginId;
+
+                this.logger.debug(`loaded pointer target of ${ptrId}: ${ptrNode}`);
+                pluginId = this.core.getRegistry(ptrNode, 'validPlugins').split(' ').shift();
+                this.logger.info(`generating code for ${this.core.getAttribute(ptrNode, 'name')} using ${pluginId}`);
+
+                var context = {
+                    namespace: this.core.getNamespace(metanode),
+                    activeNode: this.core.getPath(ptrNode)
+                };
+
+                // Load and run the plugin
+                return this.executePlugin(pluginId, context);
+            })
+            .then(hashes => hashes[0]);  // Grab the first asset for now
+    };
+
+    PtrCodeGen.prototype.createPlugin = function(pluginId) {
+        var deferred = Q.defer(),
+            pluginPath = [
+                'plugin',
+                pluginId,
+                pluginId,
+                pluginId
+            ].join('/');
+
+        requirejs([pluginPath], Plugin => {
+            var plugin = new Plugin();
+            deferred.resolve(plugin);
+        }, err => {
+            this.logger.error(`Could not load ${pluginId}: ${err}`);
+            deferred.reject(err);
+        });
+        return deferred.promise;
+    };
+
+    PtrCodeGen.prototype.configurePlugin = function(plugin, opts) {
+        var logger = this.logger.fork(plugin.getName());
+
+        return PluginUtils.loadNodesAtCommitHash(
+            this.project,
+            this.core,
+            this.commitHash,
+            this.logger,
+            opts
+        ).then(config => {
+            plugin.initialize(logger, this.blobClient, this.gmeConfig);
+            config.core = this.core;
+            plugin.configure(config);
+            return plugin;
+        });
+    };
+
+    PtrCodeGen.prototype.executePlugin = function(pluginId, config) {
+        return this.createPlugin(pluginId)
+            .then(plugin => this.configurePlugin(plugin, config))
+            .then(plugin => {
+                return Q.ninvoke(plugin, 'main');
+            })
+            .then(result => {
+                this.logger.info('Finished calling ' + pluginId);
+                return result.artifacts;
+            });
+    };
+
+    return PtrCodeGen;
+});

src/common/styles/global.css

@@ -0,0 +1,24 @@
+li.deepforge-logo {
+    background-image: url(img/deepforge-logo.png);
+    background-repeat: no-repeat;
+    background-position: center;
+    width: 100px;
+    background-size: 95px;
+}
+
+li.deepforge-logo span {
+    visibility: hidden;
+}
+
+i.gme-icon {
+    background-image: url(img/deepforge-icon.png);
+    background-size: 15.20px 18px;
+}
+
+.create-node text {
+    font-style: italic;
+}
+
+.job-canceled {
+    background-color: #ffe0b2;
+}

src/common/utils.js

@@ -0,0 +1,93 @@
+/* globals define*/
+(function(root, factory){
+    if(typeof define === 'function' && define.amd) {
+        define([], function(){
+            return (root.utils = factory());
+        });
+    } else if(typeof module === 'object' && module.exports) {
+        module.exports = (root.utils = factory());
+    }
+}(this, function() {
+    var isBoolean = txt => {
+        return typeof txt === 'boolean' || (txt === 'false' || txt === 'true');
+    };
+
+    var getSetterSchema = function(name, setters, defaults) {
+        var values,
+            schema = setters[name];
+
+        if (defaults.hasOwnProperty(name)) {
+            schema.default = defaults[name];
+        }
+        schema.type = 'string';
+        if (schema.setterType === 'const') {
+            values = Object.keys(schema.setterFn);
+            schema.isEnum = true;
+            schema.enumValues = values;
+            if (values.every(isBoolean)) {
+                if (!defaults.hasOwnProperty(name) && values.length === 1) {
+                    // there is only a method to toggle the flag to true/false, 
+                    // then the default must be the other one
+                    schema.default = values[0] === 'true' ? false : true;
+                }
+
+                if (isBoolean(schema.default)) {
+                    schema.type = 'boolean';
+                }
+            }
+        }
+        return schema;
+    };
+
+    var abbrWord = function(word) {  // camelcase
+        word = word.substring(0, 1).toUpperCase() + word.substring(1);
+        return word.split(/[a-z]+/g).join('').toLowerCase();
+    };
+
+    var abbrPhrase = function(words) {  // dashes, spaces, underscores, etc
+        return words.map(word => word[0]).join('');
+    };
+
+    var abbr = function(phrase) {
+        var words = phrase.split(/[^a-zA-Z0-9]+/g);
+        if (words.length === 1) {
+            return abbrWord(phrase);
+        } else {
+            return abbrPhrase(words);
+        }
+    };
+
+    // Resolving stdout
+    var resolveCarriageReturns = function(text) {
+        // resolve \r
+        var lines,
+            chars,
+            result,
+            i = 0;
+
+        text = text.replace(/\u0000/g, '');
+        lines = text.split('\n');
+        for (var l = lines.length-1; l >= 0; l--) {
+            i = 0;
+            chars = lines[l].split('');
+            result = [];
+
+            for (var c = 0; c < chars.length; c++) {
+                if (chars[c] === '\r') {
+                    i = 0;
+                }
+                result[i] = chars[c];
+                i++;
+            }
+            lines[l] = result.join('');
+        }
+        return lines;
+    };
+
+
+    return {
+        getSetterSchema: getSetterSchema,
+        resolveCarriageReturns: resolveCarriageReturns,
+        abbr: abbr
+    };
+}));

src/common/viz/Buttons.js

@@ -0,0 +1,65 @@
+/*globals define, WebGMEGlobal*/
+define([
+    'widgets/EasyDAG/Buttons',
+    'widgets/EasyDAG/Icons'
+], function(
+    EasyDAGButtons,
+    Icons
+) {
+
+    // Create a GoToBase button
+    var client = WebGMEGlobal.Client;
+
+    var GoToBase = function(params) {
+        // Check if it should be disabled
+        var baseId = this._getBaseId(params.item),
+            base = baseId && client.getNode(baseId);
+
+        if (!params.disabled) {
+            params.disabled = base ? base.isLibraryElement() : true;
+        }
+        EasyDAGButtons.ButtonBase.call(this, params);
+    };
+
+    GoToBase.SIZE = 10;
+    GoToBase.BORDER = 1;
+    GoToBase.prototype.BTN_CLASS = 'go-to-base';
+    GoToBase.prototype = new EasyDAGButtons.ButtonBase();
+
+    GoToBase.prototype._render = function() {
+        var lineRadius = GoToBase.SIZE - GoToBase.BORDER,
+            btnColor = '#90caf9';
+
+        if (this.disabled) {
+            btnColor = '#e0e0e0';
+        }
+
+        this.$el
+            .append('circle')
+            .attr('r', GoToBase.SIZE)
+            .attr('fill', btnColor);
+
+        // Show the 'code' icon
+        Icons.addIcon('code', this.$el, {
+            radius: lineRadius
+        });
+    };
+
+    GoToBase.prototype._onClick = function(item) {
+        var node = client.getNode(item.id),
+            baseId = node.getBaseId();
+
+        WebGMEGlobal.State.registerActiveObject(baseId);
+    };
+
+    GoToBase.prototype._getBaseId = function(item) {
+        var n = client.getNode(item.id);
+        return n && n.getBaseId();
+    };
+
+    return {
+        DeleteOne: EasyDAGButtons.DeleteOne,
+        GoToBase: GoToBase
+    };
+});
+

src/common/viz/Execute.js

@@ -0,0 +1,285 @@
+/* globals define, WebGMEGlobal */
+// Mixin for executing jobs and pipelines
+define([
+    'q',
+    'executor/ExecutorClient',
+    'deepforge/api/ExecPulseClient',
+    'deepforge/api/JobOriginClient',
+    'deepforge/Constants',
+    'panel/FloatingActionButton/styles/Materialize'
+], function(
+    Q,
+    ExecutorClient,
+    ExecPulseClient,
+    JobOriginClient,
+    CONSTANTS,
+    Materialize
+) {
+
+    var Execute = function(client, logger) {
+        this.client = this.client || client;
+        this.logger = this.logger || logger;
+        this.pulseClient = new ExecPulseClient({
+            logger: this.logger
+        });
+        this._executor = new ExecutorClient({
+            logger: this.logger.fork('ExecutorClient'),
+            serverPort: WebGMEGlobal.gmeConfig.server.port,
+            httpsecure: window.location.protocol === 'https:'
+        });
+        this.originManager = new JobOriginClient({logger: this.logger});
+    };
+
+    Execute.prototype.executeJob = function(node) {
+        return this.runExecutionPlugin('ExecuteJob', {node: node});
+    };
+
+    Execute.prototype.executePipeline = function(node) {
+        return this.runExecutionPlugin('ExecutePipeline', {node: node});
+    };
+
+    Execute.prototype.runExecutionPlugin = function(pluginId, opts) {
+        var context = this.client.getCurrentPluginContext(pluginId),
+            node = opts.node || this.client.getNode(this._currentNodeId),
+            name = node.getAttribute('name'),
+            method;
+
+        // Set the activeNode
+        context.managerConfig.namespace = 'pipeline';
+        context.managerConfig.activeNode = node.getId();
+        method = opts.useSecondary ? 'runBrowserPlugin' : 'runServerPlugin';
+
+        if (method === 'runServerPlugin' &&
+            this.client.getBranchStatus() !== this.client.CONSTANTS.BRANCH_STATUS.SYNC) {
+
+            Materialize.toast('Cannot execute operations when client is out-of-sync', 2000);
+            return;
+        }
+
+        this.client[method](pluginId, context, (err, result) => {
+            var msg = err ? `${name} failed!` : `${name} executed successfully!`,
+                duration = err ? 4000 : 2000;
+
+            // Check if it was canceled - if so, show that type of message
+            if (result && result.messages.length) {
+                msg = result.messages[0].message;
+                duration = 4000;
+            }
+
+            Materialize.toast(msg, duration);
+        });
+    };
+
+    Execute.prototype.isRunning = function(node) {
+        var baseId,
+            base,
+            type;
+
+        node = node || this.client.getNode(this._currentNodeId);
+        baseId = node.getBaseId();
+        base = this.client.getNode(baseId);
+        type = base.getAttribute('name');
+
+        if (type === 'Execution') {
+            return node.getAttribute('status') === 'running';
+        } else if (type === 'Job') {
+            return this.isRunningJob(node);
+        }
+        return false;
+    };
+
+    Execute.prototype.isRunningJob = function(job) {
+        var status = job.getAttribute('status');
+
+        return (status === 'running' || status === 'pending') &&
+            job.getAttribute('secret') && job.getAttribute('jobId');
+    };
+
+    Execute.prototype.silentStopJob = function(job) {
+        var jobHash,
+            secret;
+
+        job = job || this.client.getNode(this._currentNodeId);
+        jobHash = job.getAttribute('jobId');
+        secret = job.getAttribute('secret');
+        if (!jobHash || !secret) {
+            this.logger.error('Cannot stop job. Missing jobHash or secret');
+            return;
+        }
+
+        return this._executor.cancelJob(jobHash, secret)
+            .then(() => this.logger.info(`${jobHash} has been cancelled!`))
+            .fail(err => this.logger.error(`Job cancel failed: ${err}`));
+    };
+
+    Execute.prototype._setJobStopped = function(jobId, silent) {
+        if (!silent) {
+            var name = this.client.getNode(jobId).getAttribute('name');
+            this.client.startTransaction(`Stopping "${name}" job`);
+        }
+
+        this.client.delAttribute(jobId, 'jobId');
+        this.client.delAttribute(jobId, 'secret');
+        this.client.setAttribute(jobId, 'status', 'canceled');
+
+        if (!silent) {
+            this.client.completeTransaction();
+        }
+    };
+
+    Execute.prototype.stopJob = function(job, silent) {
+        var jobId;
+
+        job = job || this.client.getNode(this._currentNodeId);
+        jobId = job.getId();
+
+        this.silentStopJob(job);
+        this._setJobStopped(jobId, silent);
+    };
+
+
+    Execute.prototype.loadChildren = function(id) {
+        var deferred = Q.defer(),
+            execNode = this.client.getNode(id || this._currentNodeId),
+            jobIds = execNode.getChildrenIds(),
+            jobsLoaded = !jobIds.length || this.client.getNode(jobIds[0]);
+
+        // May need to load the jobs...
+        if (!jobsLoaded) {
+            // Create a territory and load the nodes
+            var territory = {},
+                ui;
+
+            territory[id] = {children: 1};
+            ui = this.client.addUI(this, () => {
+                this.client.removeUI(ui);
+                deferred.resolve();
+            });
+            this.client.updateTerritory(ui, territory);
+        } else {
+            deferred.resolve();
+        }
+
+        return deferred.promise;
+    };
+
+    Execute.prototype.stopExecution = function(id, inTransaction) {
+        var execNode = this.client.getNode(id || this._currentNodeId);
+
+        return this.loadChildren(id)
+            .then(() => this._stopExecution(execNode, inTransaction));
+    };
+
+    Execute.prototype.silentStopExecution = function(id) {
+        var execNode = this.client.getNode(id || this._currentNodeId);
+
+        // Stop the execution w/o setting any attributes
+        return this.loadChildren(id)
+            .then(() => this._silentStopExecution(execNode));
+    };
+
+    Execute.prototype._stopExecution = function(execNode, inTransaction) {
+        var msg = `Canceling ${execNode.getAttribute('name')} execution`,
+            jobIds;
+
+        if (!inTransaction) {
+            this.client.startTransaction(msg);
+        }
+
+        jobIds = this._silentStopExecution(execNode);
+
+        this.client.setAttribute(execNode.getId(), 'status', 'canceled');
+        jobIds.forEach(jobId => this._setJobStopped(jobId, true));
+
+        if (!inTransaction) {
+            this.client.completeTransaction();
+        }
+    };
+
+    Execute.prototype._silentStopExecution = function(execNode) {
+        var runningJobIds = execNode.getChildrenIds()
+            .map(id => this.client.getNode(id))
+            .filter(job => this.isRunning(job));  // get running jobs
+
+        runningJobIds.forEach(job => this.silentStopJob(job));  // stop them
+
+        return runningJobIds;
+    };
+
+    // Resuming Executions
+    Execute.prototype.checkJobExecution= function (job) {
+        var pipelineId = job.getParentId(),
+            pipeline = this.client.getNode(pipelineId);
+
+        // First check the parent execution. If it doesn't exist, then check the job
+        return this.checkPipelineExecution(pipeline)
+            .then(tryToStartJob => {
+                if (tryToStartJob) {
+                    return this._checkJobExecution(job);
+                }
+            });
+    };
+
+    Execute.prototype._checkJobExecution = function (job) {
+        var jobId = job.getAttribute('jobId'),
+            status = job.getAttribute('status');
+
+        if (status === 'running' && jobId) {
+            return this.pulseClient.check(jobId)
+                .then(status => {
+                    if (status !== CONSTANTS.PULSE.DOESNT_EXIST) {
+                        return this._onOriginBranch(jobId).then(onBranch => {
+                            if (onBranch) {
+                                this.runExecutionPlugin('ExecuteJob', {
+                                    node: job
+                                });
+                            }
+                        });
+                    } else {
+                        this.logger.warn(`Could not restart job: ${job.getId()}`);
+                    }
+                });
+        }
+        return Q();
+    };
+
+    Execute.prototype._onOriginBranch = function (hash) {
+        return this.originManager.getOrigin(hash)
+            .then(origin => {
+                var currentBranch = this.client.getActiveBranchName();
+                if (origin && origin.branch) {
+                    return origin.branch === currentBranch;
+                }
+                return false;
+            });
+    };
+
+    Execute.prototype.checkPipelineExecution = function (pipeline) {
+        var runId = pipeline.getAttribute('runId'),
+            status = pipeline.getAttribute('status'),
+            tryToStartJob = true;
+
+        if (status === 'running' && runId) {
+            return this.pulseClient.check(runId)
+                .then(status => {
+                    if (status === CONSTANTS.PULSE.DEAD) {
+                        // Check the origin branch
+                        return this._onOriginBranch(runId).then(onBranch => {
+                            if (onBranch) {
+                                this.runExecutionPlugin('ExecutePipeline', {
+                                    node: pipeline
+                                });
+                            }
+                        });
+                    }
+                    // only try to start if the pulse info doesn't exist
+                    tryToStartJob = status === CONSTANTS.PULSE.DOESNT_EXIST;
+                    return tryToStartJob;
+                });
+        } else {
+            return Q().then(() => tryToStartJob);
+        }
+    };
+
+    return Execute;
+});

src/common/viz/NodePrompter.css

@@ -0,0 +1,3 @@
+.node-prompter .scrollbar {
+    fill: #bfbfbf;
+}

src/common/viz/NodePrompter.js

@@ -0,0 +1,318 @@
+/*globals define, d3 */
+// Given a container and a set of nodes, this will prompt the user
+// to select one of the set of nodes. This will also need to support
+// adding a "plus" button for creating new objects in line
+
+define([
+    'q',
+    'css!./NodePrompter.css'
+], function(
+    Q
+) {
+
+    var MARGIN = 15,
+        CLOSING_GRACE = 400,
+        TRANSITION_DURATION = 400;
+
+    var NodePrompter = function(rect, opts) {
+        opts = opts || {};
+        // default options
+        opts.padding = opts.padding || 0;
+
+        this.left = rect.left-opts.padding;
+        this.top = rect.top-opts.padding;
+        this.width = rect.width + 2*opts.padding;
+        this.actualHeight = rect.height + 2*opts.padding;
+        this.height = this.actualHeight;  // scroll height
+        this.cx = opts.cx || rect.left + rect.width/2;
+        this.cy = opts.cy || rect.top + rect.height/2;
+        this.active = true;
+        this.onNode = false;
+        this.scrollbar = null;
+        this.scrollPosition = 0;
+
+        var container = document.createElement('div');
+        container.setAttribute('class', 'node-prompter');
+        this.container = container;
+        container.style.width = this.width + 'px';
+        container.style.height = this.height+'px';
+        container.style.position = 'absolute';
+
+    };
+
+    NodePrompter.prototype.prompt = function(nodes, selectFn) {
+        var deferred = Q.defer(),
+            size,
+            cornerRadius = 10;
+
+        this.selectHandler = selectFn;
+        this.svg = d3.select(this.container).append('svg')
+            .attr('width', this.width)
+            .attr('height', this.height)
+            .attr('overflow', 'hidden');
+
+        document.body.appendChild(this.container);
+
+        // Expand the panel
+        this.panel = this.svg.append('rect');
+        this.nodeContainer = this.svg.append('g');
+
+        size = this.initNodes(nodes);
+        this.resize(size.width, size.height);
+
+        // Create the panel
+        this.panel
+            .attr('x', this.cx)
+            .attr('y', this.cy)
+            .attr('rx', 1)
+            .attr('ry', 1)
+            .attr('height', 1)
+            .attr('width', 1)
+            .attr('fill', '#f44336');
+
+
+        this.panel.transition()
+            .delay(50)
+            .duration(TRANSITION_DURATION)
+            .attr('x', 0)
+            .attr('y', 0)
+            .attr('rx', cornerRadius)
+            .attr('ry', cornerRadius)
+            .attr('height', this.actualHeight)
+            .attr('width', this.width)
+            .attr('fill', '#e0e0e0')
+            .each('end', () => {
+                // Add the given nodes to the panel
+                this.showNodes(nodes, deferred.resolve);
+                // Add scrollbar if height is too large
+                if (this.height > this.actualHeight) {
+                    this.createScrollbar(cornerRadius);
+                }
+            });
+
+        // Event handling
+        this.svg.on('mouseout', () => {
+            this.active = false;
+            setTimeout(this.destroyIfInactive.bind(this), CLOSING_GRACE);
+        });
+        this.svg.on('mouseover', () => this.active = true);
+
+        // Return a promise called on 'selected'
+        return deferred.promise;
+    };
+
+    NodePrompter.prototype.resize = function(width, height) {
+        var dx = this.width - width,
+            maxHeight = this.height,
+            dy;
+
+        this.actualHeight = Math.min(maxHeight, height);
+        dy = this.height - this.actualHeight;
+
+        this.nodes.forEach(node => node.moveBy(-dx/2, 0));
+        this.left += dx;
+        this.top += dy;
+        this.cx -= dx;
+        this.cy -= dy;
+
+        this.container.style.left = this.left + 'px';
+        this.container.style.top = this.top + 'px';
+
+        this.width = width;
+        this.height = height;
+    };
+
+    NodePrompter.prototype.createScrollbar = function(yMargin) {
+        var width = 4,
+            actualHeight = this.actualHeight-2*yMargin,
+            updateScroll = this.updateScroll.bind(this);
+
+        // Create the scrollbar 
+        this.scrollBarHeight = actualHeight/this.height*actualHeight;
+        this.scrollHeight = actualHeight;
+        this.scrollbar = this.svg.append('rect')
+            .attr('class', 'scrollbar')
+            .attr('x', this.width - width)
+            .attr('y', yMargin)
+            .attr('rx', 2)
+            .attr('ry', 2)
+            .attr('height', this.scrollBarHeight)
+            .attr('width', width);
+
+        // Attach scroll handler to the 'panel' rect
+        this.svg
+          .on('zoom', updateScroll)
+          .on('wheel.zoom', updateScroll) 
+          .on('mousewheel.zoom',  updateScroll)
+          .on('DOMMouseScroll.zoom',  updateScroll);
+    };
+
+    NodePrompter.prototype.updateScroll = function() {
+        var delta = d3.event.deltaY,
+            sensitivity = 1,
+            maxScroll = this.scrollHeight - this.scrollBarHeight,
+            containerY,
+            relView;
+
+        this.scrollPosition += delta*sensitivity;
+        this.scrollPosition = Math.max(this.scrollPosition, 0);
+        this.scrollPosition = Math.min(this.scrollPosition, maxScroll);
+
+        this.scrollbar
+            .attr('transform', `translate(0, ${this.scrollPosition})`);
+
+        // Update the translation on the nodeContainer
+        relView = this.scrollPosition/maxScroll;
+        containerY = relView * (this.height-this.actualHeight);
+        this.nodeContainer
+            .attr('transform', `translate(0, -${containerY})`);
+    };
+
+    NodePrompter.prototype.destroyIfInactive = function() {
+        // Verify that is not over any of the displayed nodes
+        if (!this.active && !this.onNode) {
+            this.destroy();
+        }
+    };
+
+    NodePrompter.prototype.destroy = function() {
+        this.hideNodes();
+        if (this.scrollbar) {
+            this.scrollbar.remove();
+        }
+        this.panel.transition()
+            .duration(TRANSITION_DURATION)
+            .attr('x', this.cx)
+            .attr('y', this.cy)
+            .attr('rx', 1)
+            .attr('ry', 1)
+            .attr('height', 1)
+            .attr('width', 1)
+            .attr('fill', '#f44336')
+            .each('end', () => {
+                this.container.remove();
+            });
+    };
+
+    NodePrompter.prototype.onSelected = function(container, callback) {
+        // Return the id
+        if (this.selectHandler) {
+            this.selectHandler(container.node, this);
+        } else {
+            this.destroy();
+            return callback(container.node);
+        }
+    };
+
+    NodePrompter.prototype.initNodes = function(nodes) {
+        // For each node, create the containers and position them
+        var decorators = nodes.map(node => new Container(this.nodeContainer, node)),
+            lineGroup,
+            maxLineWidth = this.width - 2*MARGIN,
+            totalWidth = 0,
+            lineWidth,
+            lineStartHeight = MARGIN,
+            lineHeight,
+            cntr,
+            x,y,
+            i = 0;
+
+        // Position the nodes. while we can fit the node on the given line, add it
+        decorators.forEach(d => {
+            d.computeSize(0.25);
+        });
+
+        while (i < decorators.length) {
+            lineGroup = [decorators[i]];
+            lineWidth = decorators[i].width() + MARGIN;
+            lineHeight = decorators[i].height();
+            i++;
+            while (i < decorators.length &&
+                lineWidth + decorators[i].width() + MARGIN < maxLineWidth) {
+
+                lineGroup.push(decorators[i]);
+                lineWidth += decorators[i].width() + MARGIN;
+                lineHeight = Math.max(lineHeight, decorators[i].height());
+                i++;
+            }
+
+            // Get the positions for each
+            lineWidth += MARGIN;
+            totalWidth = Math.max(lineWidth, totalWidth);
+            x = (this.width-lineWidth)/2 + MARGIN;
+            for (var g = 0; g < lineGroup.length; g++) {
+                cntr = lineGroup[g];
+                y = (lineHeight - cntr.height())/2 + lineStartHeight;
+                cntr.goTo(x, y);
+                x += cntr.width() + MARGIN;
+            }
+
+            lineStartHeight += lineHeight + MARGIN;
+        }
+        
+        this.nodes = decorators;
+        return {
+            height: lineStartHeight,
+            width: totalWidth
+        };
+    };
+
+    NodePrompter.prototype.showNodes = function(nodes, callback) {
+        this.nodes.forEach(d => {
+            d.render(0.25);
+            d.$el.on('mouseover', () => this.onNode = true);
+            d.$el.on('mouseout', () => this.onNode = false);
+            d.$el.on('click', () => this.onSelected(d, callback));
+        });
+    };
+
+    NodePrompter.prototype.hideNodes = function() {
+        this.nodes.forEach(node => node.$el.remove());
+    };
+
+    var Container = function(svg, node) {  // used for positioning
+        this.$el = svg.append('g');
+        this.x = 0;
+        this.y = 0;
+        this.node = node;
+        this.decorator = new node.Decorator({
+            node: node,
+            parentEl: this.$el
+        });
+    };
+
+    Container.prototype.moveBy = function(dx, dy) {
+        dx = dx || 0;
+        dy = dy || 0;
+        this.x += dx;
+        this.y += dy;
+    };
+
+    Container.prototype.goTo = function(x, y) {
+        this.x = x;
+        this.y = y;
+    };
+
+    Container.prototype.computeSize = function(zoom) {
+        this.$el.attr('opacity', 0);
+        this.decorator.render(zoom);
+    };
+
+    Container.prototype.render = function(zoom) {
+        this.$el.attr('transform', `translate(${this.x}, ${this.y})`);
+        this.$el
+            .transition()
+            .attr('opacity', 1);
+        this.decorator.render(zoom);
+    };
+
+    Container.prototype.width = function() {
+        return this.decorator.width;
+    };
+
+    Container.prototype.height = function() {
+        return this.decorator.height;
+    };
+
+    return NodePrompter;
+});

src/common/viz/OperationControl.js

@@ -0,0 +1,42 @@
+/* globals define */
+// A mixin containing helpers for working with operations
+define([
+], function(
+) {
+    'use strict';
+    var OperationControl = function() {
+    };
+
+    OperationControl.prototype.hasMetaName = function(id, name, inclusive) {
+        var node = this._client.getNode(id),
+            bId = inclusive ? id : node.getBaseId(),
+            baseName;
+
+        while (bId) {
+            node = this._client.getNode(bId);
+            baseName = node.getAttribute('name');
+            if (baseName === name) {
+                return true;
+            }
+            bId = node.getBaseId();
+        }
+        return false;
+    };
+
+    OperationControl.prototype.getOperationInputs = function(node) {
+        return this.getOperationData(node, 'Inputs');
+    };
+
+    OperationControl.prototype.getOperationOutputs = function(node) {
+        return this.getOperationData(node, 'Outputs');
+    };
+
+    OperationControl.prototype.getOperationData = function(node, type) {
+        var childrenIds = node.getChildrenIds(),
+            typeId = childrenIds.find(cId => this.hasMetaName(cId, type));
+
+        return typeId ? this._client.getNode(typeId).getChildrenIds() : [];
+    };
+
+    return OperationControl;
+});

src/common/viz/PipelineControl.js

@@ -0,0 +1,146 @@
+/* globals define */
+// Shared methods for editing pipelines
+define([
+    'panels/EasyDAG/EasyDAGControl',
+    'deepforge/viz/OperationControl',
+    'deepforge/Constants',
+    'widgets/EasyDAG/AddNodeDialog',
+    'underscore'
+], function(
+    EasyDAGControl,
+    OperationControl,
+    CONSTANTS,
+    AddNodeDialog,
+    _
+) {
+    'use strict';
+
+    var PipelineControl = function() {
+    };
+
+    _.extend(PipelineControl.prototype, OperationControl.prototype);
+
+    PipelineControl.prototype.DEFAULT_DECORATOR = 'OperationDecorator';
+
+    PipelineControl.prototype._getAllDescendentIds =
+        EasyDAGControl.prototype._getAllDescendentIds;
+    PipelineControl.prototype._getAllValidChildren =
+        EasyDAGControl.prototype._getAllValidChildren;
+    PipelineControl.prototype._getNodeDecorator =
+        EasyDAGControl.prototype._getNodeDecorator;
+
+    PipelineControl.prototype.onCreateInitialNode = function() {
+        var initialNodes = this.getValidInitialNodes(),
+            initialNode = initialNodes[0];
+
+        if (initialNodes.length > 1) {
+            // Create the modal view with all possible subsequent nodes
+            var dialog = new AddNodeDialog();
+
+            dialog.show(null, initialNodes.map(node => {
+                return {node};
+            }));
+            dialog.onSelect = nodeInfo => {
+                if (nodeInfo) {
+                    this.createNode(nodeInfo.node.id);
+                }
+            };
+        } else {
+            this.createNode(initialNode.id);
+        }
+    };
+
+    PipelineControl.prototype.getValidInitialNodes = function () {
+        // Get all nodes that have no inputs
+        return this._getAllValidChildren(this._currentNodeId)
+            .map(id => this._client.getNode(id))
+            .filter(node => !node.isAbstract() && !node.isConnection())
+            // Checking the name (below) is simply convenience so we can
+            // still create operation prototypes from Operation (which we
+            // wouldn't be able to do if it was abstract - which it probably
+            // should be)
+            .filter(node => node.getAttribute('name') !== 'Operation')
+            .map(node => this._getObjectDescriptor(node.getId()));
+    };
+
+    PipelineControl.prototype.createNode = function(baseId) {
+        var parentId = this._currentNodeId,
+            newNodeId = this._client.createNode({parentId, baseId});
+
+        return newNodeId;
+    };
+
+    PipelineControl.prototype._getObjectDescriptor = function(id) {
+        var desc = EasyDAGControl.prototype._getObjectDescriptor.call(this, id),
+            node = this._client.getNode(id);
+
+        desc.inputs = [];
+        desc.outputs = [];
+        if (this.hasMetaName(id, 'Operation')) {
+            // Only decorate operations in the currently active node
+            if (this._currentNodeId !== desc.parentId) {
+                return desc;
+            }
+
+            // Add inputs and outputs
+            var childrenIds = node.getChildrenIds(),
+                inputId = childrenIds.find(cId => this.hasMetaName(cId, 'Inputs')),
+                outputId = childrenIds.find(cId => this.hasMetaName(cId, 'Outputs')),
+                inputs,
+                outputs;
+
+            inputs = inputId ? this._client.getNode(inputId).getChildrenIds() : [];
+            outputs = outputId ? this._client.getNode(outputId).getChildrenIds() : [];
+
+            // Add the inputs, outputs in the form:
+            //   [ name, baseId ]
+            desc.inputs = inputs.map(id => this.formatIO(id));
+            desc.outputs = outputs.map(id => this.formatIO(id));
+
+            // Remove the 'code' attribute
+            if (desc.attributes.code) {
+                delete desc.attributes[CONSTANTS.LINE_OFFSET];
+                delete desc.attributes.code;
+            }
+
+        } else if (desc.isConnection) {
+            // Set src, dst to siblings and add srcPort, dstPort
+            desc.srcPort = desc.src;
+            desc.dstPort = desc.dst;
+
+            // Get the src/dst that are in the currentNode
+            desc.src = this.getSiblingContaining(desc.src);
+            desc.dst = this.getSiblingContaining(desc.dst);
+
+            if (desc.src === null || desc.dst === null) {
+                this._logger.warn(`Could not get src/dst for ${desc.id}`);
+            }
+        } else if (this.hasMetaName(desc.id, 'Data')) {  // port
+            // Add nodeId for container
+            desc.nodeId = this.getSiblingContaining(desc.id);
+            // It is a data port if it has a parentId and the parent is either
+            // 'Inputs' or 'Outputs'
+            desc.isDataPort = desc.parentId &&
+                (this.hasMetaName(desc.parentId, 'Inputs') || this.hasMetaName(desc.parentId, 'Outputs'));
+        }
+        return desc;
+    };
+
+    PipelineControl.prototype.getSiblingContaining = function(containedId) {
+        var n = this._client.getNode(containedId);
+        while (n && n.getParentId() !== this._currentNodeId) {
+            n = this._client.getNode(n.getParentId());
+        }
+        return n && n.getId();
+    };
+
+    PipelineControl.prototype.formatIO = function(id) {
+        var node = this._client.getNode(id);
+        return {
+            id: id,
+            name: node.getAttribute('name')
+        };
+    };
+
+    return PipelineControl;
+});

src/common/viz/RenameablePanel.js

@@ -0,0 +1,60 @@
+/*globals define*/
+define([
+    'js/PanelBase/PanelBaseWithHeader',
+    'js/PanelManager/IActivePanel',
+    'underscore'
+], function(
+    PanelBaseWithHeader,
+    IActivePanel,
+    _
+) {
+
+    var RenameablePanel = function() {
+        PanelBaseWithHeader.apply(this, arguments);
+    };
+
+    _.extend(
+        RenameablePanel.prototype,
+        PanelBaseWithHeader.prototype,
+        IActivePanel.prototype
+    );
+
+    RenameablePanel.OPTIONS = PanelBaseWithHeader.OPTIONS;
+    RenameablePanel.prototype.initializeRenameable = function () {
+        this.$panelHeaderTitle.on('click', this.editTitle.bind(this));
+    };
+
+    RenameablePanel.prototype.currentNodeId = function () {
+        return this.control._currentNodeId;
+    };
+
+    RenameablePanel.prototype.currentBaseName = function () {
+        var currentId = this.currentNodeId(),
+            node = this._client.getNode(currentId),
+            baseId = node.getBaseId(),
+            base = this._client.getNode(baseId);
+
+        return base.getAttribute('name');
+    };
+
+    RenameablePanel.prototype.editTitle = function () {
+        this.$panelHeaderTitle.editInPlace({
+            css: {
+                'z-index': 1000
+            },
+            onChange: (oldValue, newValue) => {
+                var nodeId = this.currentNodeId(),
+                    type = this.currentBaseName(),
+                    msg = `Renamed ${type}: ${oldValue} -> ${newValue}`;
+
+                if (!/^\s*$/.test(newValue)) {
+                    this._client.startTransaction(msg);
+                    this._client.setAttribute(nodeId, 'name', newValue);
+                    this._client.completeTransaction();
+                }
+            }
+        });
+    };
+
+    return RenameablePanel;
+});

src/common/viz/Utils.js

@@ -0,0 +1,19 @@
+/* globals define*/
+define({
+    getDisplayTime: timestamp => {
+        var today = new Date().toLocaleDateString(),
+            date = new Date(timestamp).toLocaleDateString();
+
+        if (date === today) {
+            date = `Today (${new Date(timestamp).toLocaleTimeString()})`;
+        }
+        return date;
+    },
+    ClassForJobStatus: {
+        success: 'success',
+        canceled: 'job-canceled',
+        failed: 'danger',
+        pending: '',
+        running: 'warning'
+    }
+});

src/common/viz/panels/ThumbnailControl.js

@@ -0,0 +1,41 @@
+/* globals define */
+define([
+    'panels/EasyDAG/EasyDAGControl'
+], function(
+    EasyDAGControl
+) {
+    var ThumbnailControl = function() {
+        EasyDAGControl.apply(this, arguments);
+    };
+
+    ThumbnailControl.prototype = Object.create(EasyDAGControl.prototype);
+
+    ThumbnailControl.prototype._initWidgetEventHandlers = function () {
+        EasyDAGControl.prototype._initWidgetEventHandlers.call(this);
+        this._widget.updateThumbnail = this.updateThumbnail.bind(this);
+    };
+
+    ThumbnailControl.prototype.updateThumbnail = function (svg) {
+        var node = this._client.getNode(this._currentNodeId),
+            name,
+            attrs,
+            currentThumbnail,
+            attrName = 'thumbnail',
+            msg;
+
+        if (node) {  // may have been deleted
+            name = node.getAttribute('name');
+            attrs = node.getValidAttributeNames();
+            currentThumbnail = node.getAttribute(attrName);
+            msg = `Updating pipeline thumbnail for "${name}"`;
+
+            if (attrs.indexOf(attrName) > -1 && currentThumbnail !== svg) {
+                this._client.startTransaction(msg);
+                this._client.setAttribute(this._currentNodeId, attrName, svg);
+                this._client.completeTransaction();
+            }
+        }
+    };
+
+    return ThumbnailControl;
+});

src/common/viz/widgets/Thumbnail.js

@@ -0,0 +1,84 @@
+/* globals define, $, _ */
+define([
+    'widgets/EasyDAG/EasyDAGWidget'
+], function(
+    EasyDAGWidget
+) {
+
+    var ThumbnailWidget = function() {
+        EasyDAGWidget.apply(this, arguments);
+    };
+
+    ThumbnailWidget.prototype = Object.create(EasyDAGWidget.prototype);
+
+    ThumbnailWidget.prototype.addNode = function() {
+        var result = EasyDAGWidget.prototype.addNode.apply(this, arguments);
+
+        this.refreshThumbnail();
+        return result;
+    };
+
+    ThumbnailWidget.prototype.removeNode = function() {
+        var result = EasyDAGWidget.prototype.removeNode.apply(this, arguments);
+
+        this.refreshThumbnail();
+        return result;
+    };
+
+    ThumbnailWidget.prototype._removeConnection = function() {
+        var result = EasyDAGWidget.prototype._removeConnection.apply(this, arguments);
+
+        this.refreshThumbnail();
+        return result;
+    };
+
+    ThumbnailWidget.prototype.addConnection = function() {
+        var result = EasyDAGWidget.prototype.addConnection.apply(this, arguments);
+
+        this.refreshThumbnail();
+        return result;
+    };
+
+    ////////////////////////// Thumbnail updates //////////////////////////
+    ThumbnailWidget.prototype.getSvgDistanceDim = function(dim) {
+        var maxValue = this._getMaxAlongAxis(dim),
+            nodes,
+            minValue;
+
+        nodes = this.graph.nodes().map(id => this.graph.node(id));
+        minValue = Math.min.apply(null, nodes.map(node => node[dim]));
+        return maxValue-minValue;
+    };
+
+    ThumbnailWidget.prototype.getSvgWidth = function() {
+        return this.getSvgDistanceDim('x');
+    };
+
+    ThumbnailWidget.prototype.getSvgHeight = function() {
+        return this.getSvgDistanceDim('y');
+    };
+
+    ThumbnailWidget.prototype.getViewBox = function() {
+        var maxX = this.getSvgWidth('x'),
+            maxY = this.getSvgHeight('y');
+
+        return `0 0 ${maxX} ${maxY}`;
+    };
+
+    ThumbnailWidget.prototype.refreshThumbnail = _.debounce(function() {
+        // Get the svg...
+        var svg = document.createElement('svg'),
+            group = this.$svg.node(),
+            child;
+
+        svg.setAttribute('viewBox', this.getViewBox());
+        for (var i = 0; i < group.children.length; i++) {
+            child = $(group.children[i]);
+            svg.appendChild(child.clone()[0]);
+        }
+
+        this.updateThumbnail(svg.outerHTML);
+    }, 1000);
+
+    return ThumbnailWidget;
+});

src/decorators/ArtifactOpDecorator/ArtifactOpDecorator.js

@@ -0,0 +1,43 @@
+/*globals define, _*/
+/*jshint browser: true, camelcase: false*/
+
+/**
+ * @author brollb / https://github.com/brollb
+ */
+
+define([
+    'js/Decorators/DecoratorBase',
+    './EasyDAG/ArtifactOpDecorator.EasyDAGWidget'
+], function (
+    DecoratorBase,
+    ArtifactOpDecoratorEasyDAGWidget
+) {
+
+    'use strict';
+
+    var ArtifactOpDecorator,
+        __parent__ = DecoratorBase,
+        __parent_proto__ = DecoratorBase.prototype,
+        DECORATOR_ID = 'ArtifactOpDecorator';
+
+    ArtifactOpDecorator = function (params) {
+        var opts = _.extend({loggerName: this.DECORATORID}, params);
+
+        __parent__.apply(this, [opts]);
+
+        this.logger.debug('ArtifactOpDecorator ctor');
+    };
+
+    _.extend(ArtifactOpDecorator.prototype, __parent_proto__);
+    ArtifactOpDecorator.prototype.DECORATORID = DECORATOR_ID;
+
+    /*********************** OVERRIDE DecoratorBase MEMBERS **************************/
+
+    ArtifactOpDecorator.prototype.initializeSupportedWidgetMap = function () {
+        this.supportedWidgetMap = {
+            EasyDAG: ArtifactOpDecoratorEasyDAGWidget
+        };
+    };
+
+    return ArtifactOpDecorator;
+});

src/decorators/ArtifactOpDecorator/EasyDAG/ArtifactOpDecorator.EasyDAGWidget.css

@@ -0,0 +1,43 @@
+/**
+ * @author brollb / https://github.com/brollb
+ */
+.artifactop-decorator {
+  min-width: 65px;
+  height: 40px;
+  border: 1px solid black;
+  background-color: #dedede;
+  padding: 3px;
+  text-align: center; }
+  .artifactop-decorator .attr-title {
+      font-style: italic;
+  }
+  .artifactop-decorator .name {
+    margin-top: 10px;
+    white-space: nowrap;
+    font-size: 16px;
+    font-weight: bold;
+    font-family: "Helvetica Neue",Helvetica,Arial,sans-serif;
+    text-align: center; }
+  .artifactop-decorator .connector {
+    background-color: #fefefe;
+    height: 10px;
+    width: 10px;
+    position: absolute;
+    cursor: pointer;
+    border: 1px solid blue;
+    z-index: 10;
+    margin-left: -6px;
+    left: 50%; }
+    .artifactop-decorator .connector:hover {
+      border-color: rgba(82, 168, 236, 0.8);
+      -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6);
+      -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6);
+      box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6); }
+    .artifactop-decorator .connector.top {
+      top: -6px; }
+    .artifactop-decorator .connector.bottom {
+      bottom: -6px; }
+
+.selected .artifactop-decorator {
+  border: 1px solid #52a8ec;
+  background-color: #dbeafc; }

src/decorators/ArtifactOpDecorator/EasyDAG/ArtifactOpDecorator.EasyDAGWidget.js

@@ -0,0 +1,109 @@
+/*globals define, _*/
+/*jshint browser: true, camelcase: false*/
+
+define([
+    'deepforge/Constants',
+    'decorators/DcOpDecorator/EasyDAG/DcOpDecorator.EasyDAGWidget',
+    'css!./ArtifactOpDecorator.EasyDAGWidget.css'
+], function (
+    CONSTANTS,
+    DecoratorBase
+) {
+
+    'use strict';
+
+    var ArtifactOpDecorator,
+        DECORATOR_ID = 'ArtifactOpDecorator',
+        CAST_OPTS = {
+            ArtifactFinder: {
+                ptr: 'type',
+                metaTgt: true
+            }
+        };
+
+    CAST_OPTS[CONSTANTS.OP.INPUT] = {
+        ptr: 'artifact',
+        metaTgt: false
+    };
+
+    // ArtifactOp nodes need to be able to...
+    //     - dynamically change their outputs (downcast)
+    ArtifactOpDecorator = function (options) {
+        options.color = options.color || '#b0bec5';
+        DecoratorBase.call(this, options);
+        // set the opts...
+        this.castOpts = CAST_OPTS[this._node.baseName];
+    };
+
+    _.extend(ArtifactOpDecorator.prototype, DecoratorBase.prototype);
+
+    ArtifactOpDecorator.prototype.DECORATOR_ID = DECORATOR_ID;
+
+    ArtifactOpDecorator.prototype.getTargetFilterFnFor = function() {
+        return id => {
+            var node = this.client.getNode(id),
+                isMetaTgt = node.getId() === node.getMetaTypeId();
+            return isMetaTgt === this.castOpts.metaTgt;
+        };
+    };
+
+    ArtifactOpDecorator.prototype.savePointer = function(name, to) {
+        // When the 'artifact' pointer is changed, we should change the base
+        // of the data output node to the target type
+        if (typeof to !== 'string') {
+            var outputId = this._node.outputs[0] && this._node.outputs[0].id;
+
+            // Clear the data handle of the output
+            this.client.startTransaction(`Removing output of ${this.name}`);
+            this.client.delPointer(this._node.id, name);
+            if (outputId) {
+                this.client.delAttribute(outputId, 'data');
+            }
+            this.client.completeTransaction();
+        } else if (name === this.castOpts.ptr) {  // set the casted value
+            this.client.startTransaction(`Setting output of ${this.name} to ${to}`);
+            this.castOutputType(to);
+            this.client.setPointer(this._node.id, name, to);
+            this.client.completeTransaction();
+        } else {
+            DecoratorBase.prototype.savePointer.call(this, name, to);
+        }
+    };
+
+    ArtifactOpDecorator.prototype.getDisplayName = function() {
+        var ptrName = this._node.baseName === CONSTANTS.OP.INPUT ? 'artifact' : 'type',
+            id = this._node.pointers[ptrName],
+            name = this.nameFor[id] || this._node.name;
+        return name;
+    };
+
+    ArtifactOpDecorator.prototype.updateDisplayName = function() {
+        var newName = this.getDisplayName();
+        if (this.name !== newName) {
+            this.name = newName;
+            this.nameWidth = null;
+        }
+    };
+
+    ArtifactOpDecorator.prototype.updateTargetName = function(id, name) {
+        DecoratorBase.prototype.updateTargetName.apply(this, arguments);
+        // Update name
+        var ptrName = this._node.baseName === CONSTANTS.OP.INPUT ? 'artifact' : 'type';
+        if (this._node.pointers[ptrName] === id) {
+            this._name = name;
+            this.onResize();
+        }
+    };
+
+    ArtifactOpDecorator.prototype.expand = function() {
+        this.updateDisplayName();
+        DecoratorBase.prototype.expand.call(this);
+    };
+
+    ArtifactOpDecorator.prototype.condense = function() {
+        this.updateDisplayName();
+        DecoratorBase.prototype.condense.call(this);
+    };
+
+    return ArtifactOpDecorator;
+});

src/decorators/DcOpDecorator/DcOpDecorator.js

@@ -0,0 +1,43 @@
+/*globals define, _*/
+/*jshint browser: true, camelcase: false*/
+
+/**
+ * @author brollb / https://github.com/brollb
+ */
+
+define([
+    'js/Decorators/DecoratorBase',
+    './EasyDAG/DcOpDecorator.EasyDAGWidget'
+], function (
+    DecoratorBase,
+    DcOpDecoratorEasyDAGWidget
+) {
+
+    'use strict';
+
+    var DcOpDecorator,
+        __parent__ = DecoratorBase,
+        __parent_proto__ = DecoratorBase.prototype,
+        DECORATOR_ID = 'DcOpDecorator';
+
+    DcOpDecorator = function (params) {
+        var opts = _.extend({loggerName: this.DECORATORID}, params);
+
+        __parent__.apply(this, [opts]);
+
+        this.logger.debug('DcOpDecorator ctor');
+    };
+
+    _.extend(DcOpDecorator.prototype, __parent_proto__);
+    DcOpDecorator.prototype.DECORATORID = DECORATOR_ID;
+
+    /*********************** OVERRIDE DecoratorBase MEMBERS **************************/
+
+    DcOpDecorator.prototype.initializeSupportedWidgetMap = function () {
+        this.supportedWidgetMap = {
+            EasyDAG: DcOpDecoratorEasyDAGWidget
+        };
+    };
+
+    return DcOpDecorator;
+});

src/decorators/DcOpDecorator/EasyDAG/DcOpDecorator.EasyDAGWidget.css

@@ -0,0 +1,43 @@
+/**
+ * @author brollb / https://github.com/brollb
+ */
+.dcop-decorator {
+  min-width: 65px;
+  height: 40px;
+  border: 1px solid black;
+  background-color: #dedede;
+  padding: 3px;
+  text-align: center; }
+  .dcop-decorator .attr-title {
+      font-style: italic;
+  }
+  .dcop-decorator .name {
+    margin-top: 10px;
+    white-space: nowrap;
+    font-size: 16px;
+    font-weight: bold;
+    font-family: "Helvetica Neue",Helvetica,Arial,sans-serif;
+    text-align: center; }
+  .dcop-decorator .connector {
+    background-color: #fefefe;
+    height: 10px;
+    width: 10px;
+    position: absolute;
+    cursor: pointer;
+    border: 1px solid blue;
+    z-index: 10;
+    margin-left: -6px;
+    left: 50%; }
+    .dcop-decorator .connector:hover {
+      border-color: rgba(82, 168, 236, 0.8);
+      -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6);
+      -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6);
+      box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6); }
+    .dcop-decorator .connector.top {
+      top: -6px; }
+    .dcop-decorator .connector.bottom {
+      bottom: -6px; }
+
+.selected .dcop-decorator {
+  border: 1px solid #52a8ec;
+  background-color: #dbeafc; }

src/decorators/DcOpDecorator/EasyDAG/DcOpDecorator.EasyDAGWidget.js

@@ -0,0 +1,79 @@
+/*globals define, _*/
+/*jshint browser: true, camelcase: false*/
+
+/**
+ * @author brollb / https://github.com/brollb
+ */
+
+define([
+    'js/Constants',
+    'decorators/OperationDecorator/EasyDAG/OperationDecorator.EasyDAGWidget',
+    'css!./DcOpDecorator.EasyDAGWidget.css'
+], function (
+    CONSTANTS,
+    DecoratorBase
+) {
+
+    'use strict';
+
+    var DcOpDecorator,
+        DECORATOR_ID = 'DcOpDecorator';
+
+    // DcOp nodes need to be able to...
+    //     - dynamically change their outputs (downcast)
+    DcOpDecorator = function (options) {
+        options.color = options.color || '#78909c';
+        DecoratorBase.call(this, options);
+    };
+
+    _.extend(DcOpDecorator.prototype, DecoratorBase.prototype);
+
+    DcOpDecorator.prototype.DECORATOR_ID = DECORATOR_ID;
+
+    DcOpDecorator.prototype.getTargetFilterFnFor = function() {
+        return id => {
+            var node = this.client.getNode(id);
+            return node.getId() !== node.getMetaTypeId();  // not meta node
+        };
+    };
+
+    DcOpDecorator.prototype.castOutputType = function(targetId) {
+        var target = this.client.getNode(targetId),
+            baseId = target.getBaseId(),
+            outputId = this._node.outputs[0] && this._node.outputs[0].id,
+            hash;
+
+        if (!outputId) {
+            // create the outputId node
+            outputId = this._createOutputNode(baseId);
+        } else {
+            this.client.setPointer(outputId, CONSTANTS.POINTER_BASE, baseId);
+        }
+        // Copy the data content to the output node
+        hash = target.getAttribute('data');
+        this.client.setAttribute(outputId, 'data', hash);
+    };
+
+    DcOpDecorator.prototype._createOutputNode = function(baseId) {
+        var n = this.client.getNode(this._node.id),
+            outputCntrId;
+
+        outputCntrId = n.getChildrenIds().find(id => {
+            var metaTypeId = this.client.getNode(id).getMetaTypeId(),
+                metaType = this.client.getNode(metaTypeId);
+
+            if (!metaType) {
+                this.logger.error(`Could not check the type of ${id}!`);
+                return false;
+            }
+            return metaType.getAttribute('name') === 'Outputs';
+        });
+
+        return this.client.createNode({
+            baseId: baseId,
+            parentId: outputCntrId
+        });
+    };
+
+    return DcOpDecorator;
+});

src/decorators/JobDecorator/EasyDAG/AttributeField.RO.js

@@ -0,0 +1,17 @@
+/*globals define*/
+define([
+    'decorators/EllipseDecorator/EasyDAG/AttributeField'
+], function(
+    BaseAttributeField
+) {
+    var AttributeField = function() {
+        BaseAttributeField.apply(this, arguments);
+    };
+
+    AttributeField.prototype = Object.create(BaseAttributeField.prototype);
+
+    AttributeField.prototype.onClick = function() {
+    };
+
+    return AttributeField;
+});

src/decorators/JobDecorator/EasyDAG/JobDecorator.EasyDAGWidget.css

@@ -0,0 +1,43 @@
+/**
+ * @author brollb / https://github.com/brollb
+ */
+.job-decorator {
+  min-width: 65px;
+  height: 40px;
+  border: 1px solid black;
+  background-color: #dedede;
+  padding: 3px;
+  text-align: center; }
+  .job-decorator .attr-title {
+      font-style: italic;
+  }
+  .job-decorator .name {
+    margin-top: 10px;
+    white-space: nowrap;
+    font-size: 16px;
+    font-weight: bold;
+    font-family: "Helvetica Neue",Helvetica,Arial,sans-serif;
+    text-align: center; }
+  .job-decorator .connector {
+    background-color: #fefefe;
+    height: 10px;
+    width: 10px;
+    position: absolute;
+    cursor: pointer;
+    border: 1px solid blue;
+    z-index: 10;
+    margin-left: -6px;
+    left: 50%; }
+    .job-decorator .connector:hover {
+      border-color: rgba(82, 168, 236, 0.8);
+      -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6);
+      -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6);
+      box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6); }
+    .job-decorator .connector.top {
+      top: -6px; }
+    .job-decorator .connector.bottom {
+      bottom: -6px; }
+
+.selected .job-decorator {
+  border: 1px solid #52a8ec;
+  background-color: #dbeafc; }

src/decorators/JobDecorator/EasyDAG/JobDecorator.EasyDAGWidget.js

@@ -0,0 +1,94 @@
+/*globals define, _*/
+/*jshint browser: true, camelcase: false*/
+
+define([
+    'deepforge/Constants',
+    'decorators/EllipseDecorator/EasyDAG/EllipseDecorator.EasyDAGWidget',
+    './PointerField.RO',
+    './AttributeField.RO',
+    'css!./JobDecorator.EasyDAGWidget.css'
+], function (
+    CONSTANTS,
+    EllipseDecorator,
+    PointerField,
+    AttributeField
+) {
+
+    'use strict';
+
+    var JobDecorator,
+        DECORATOR_ID = 'JobDecorator',
+        COLORS = {
+            pending: '#9e9e9e',
+            queued: '#cfd8dc',
+            running: '#fff59d',
+            canceled: '#ffcc80',
+            success: '#66bb6a',
+            fail: '#e57373'
+        };
+
+    // Job nodes need to be able to...
+    //     - show their ports
+    //     - highlight ports
+    //     - unhighlight ports
+    //     - report the location of specific ports
+    JobDecorator = function (options) {
+        options.skipAttributes = {
+            name: true,
+            status: true,
+            execFiles: true,
+            stdout: true,
+            secret: true,
+            jobId: true,
+            debug: true
+        };
+        EllipseDecorator.call(this, options);
+    };
+
+    _.extend(JobDecorator.prototype, EllipseDecorator.prototype);
+
+    JobDecorator.prototype.DECORATOR_ID = DECORATOR_ID;
+    JobDecorator.prototype.AttributeField = AttributeField;
+    JobDecorator.prototype.PointerField = PointerField;
+
+    JobDecorator.prototype.isInputOperation = function() {
+        return this._node.name === CONSTANTS.OP.INPUT;
+    };
+
+    JobDecorator.prototype.getDisplayName = function() {
+        if (this.isInputOperation()) {
+            var id = this._node.pointers.artifact;
+
+            // Try to look up the pointer name
+            return this.nameFor[id] || this._node.name;
+        }
+        return this._node.name;
+    };
+
+    JobDecorator.prototype.setAttributes = function() {
+        EllipseDecorator.prototype.setAttributes.call(this);
+        var attrs = this._node.attributes,
+            status = attrs.status && attrs.status.value,
+            opAttrs = Object.keys(this._node.opAttributes);
+
+        // Update the color based on the 'status' attr
+        this.color = COLORS[status] || COLORS.fail;
+
+        // Set _attributes from opAttributes
+        for (var i = opAttrs.length; i--;) {
+            this._attributes[opAttrs[i]] = this._node.opAttributes[opAttrs[i]];
+        }
+    };
+
+    JobDecorator.prototype.updateTargetName = function() {
+        EllipseDecorator.prototype.updateTargetName.apply(this, arguments);
+        var name = this.getDisplayName();
+
+        if (name !== this.name) {
+            this.name = name;
+            this.onResize();
+        }
+    };
+
+    return JobDecorator;
+});

src/decorators/JobDecorator/EasyDAG/PointerField.RO.js

@@ -0,0 +1,22 @@
+/*globals define*/
+define([
+    'decorators/EllipseDecorator/EasyDAG/PointerField'
+], function(
+    BasePointerField
+) {
+    var PointerField = function() {
+        BasePointerField.apply(this, arguments);
+    };
+
+    PointerField.prototype = Object.create(BasePointerField.prototype);
+
+    PointerField.prototype.onClick = function() {
+    };
+
+    // Remove the delete icon and adjust the text location
+    PointerField.prototype.hasIcon = function() {
+        return false;
+    };
+
+    return PointerField;
+});

src/decorators/JobDecorator/JobDecorator.js

@@ -0,0 +1,43 @@
+/*globals define, _*/
+/*jshint browser: true, camelcase: false*/
+
+/**
+ * @author brollb / https://github.com/brollb
+ */
+
+define([
+    'js/Decorators/DecoratorBase',
+    './EasyDAG/JobDecorator.EasyDAGWidget'
+], function (
+    DecoratorBase,
+    JobDecoratorEasyDAGWidget
+) {
+
+    'use strict';
+
+    var JobDecorator,
+        __parent__ = DecoratorBase,
+        __parent_proto__ = DecoratorBase.prototype,
+        DECORATOR_ID = 'JobDecorator';
+
+    JobDecorator = function (params) {
+        var opts = _.extend({loggerName: this.DECORATORID}, params);
+
+        __parent__.apply(this, [opts]);
+
+        this.logger.debug('JobDecorator ctor');
+    };
+
+    _.extend(JobDecorator.prototype, __parent_proto__);
+    JobDecorator.prototype.DECORATORID = DECORATOR_ID;
+
+    /*********************** OVERRIDE DecoratorBase MEMBERS **************************/
+
+    JobDecorator.prototype.initializeSupportedWidgetMap = function () {
+        this.supportedWidgetMap = {
+            EasyDAG: JobDecoratorEasyDAGWidget
+        };
+    };
+
+    return JobDecorator;
+});

src/decorators/LayerDecorator/EasyDAG/LayerDecorator.EasyDAGWidget.js

@@ -0,0 +1,110 @@
+/*globals define, _, WebGMEGlobal*/
+/*jshint browser: true, camelcase: false*/
+
+define([
+    'decorators/EllipseDecorator/EasyDAG/EllipseDecorator.EasyDAGWidget',
+    'deepforge/Constants',
+    './LayerField'
+], function (
+    EllipseDecorator,
+    Constants,
+    LayerField
+) {
+
+    'use strict';
+
+    var LayerDecorator,
+        DECORATOR_ID = 'LayerDecorator';
+
+    // Layer nodes need to be able to...
+    //     - show their ports
+    //     - highlight ports
+    //     - unhighlight ports
+    //     - report the location of specific ports
+    LayerDecorator = function (options) {
+        options.skipAttributes = {name: true};
+        options.skipAttributes[Constants.CTOR_ARGS_ATTR] = true;
+        EllipseDecorator.call(this, options);
+    };
+
+    _.extend(LayerDecorator.prototype, EllipseDecorator.prototype);
+
+    LayerDecorator.prototype.DECORATOR_ID = DECORATOR_ID;
+    LayerDecorator.prototype.PointerField = LayerField;
+    LayerDecorator.prototype.getDisplayName = function() {
+        return this._node.name;
+    };
+
+    // Create the pointer fields and change the event handlers
+    LayerDecorator.prototype.createPointerFields = function() {
+        var i = this.fields.length,
+            y,
+            ptr;
+
+        // Get the fields
+        y = EllipseDecorator.prototype.createPointerFields.apply(this, arguments);
+        while (i < this.fields.length) {
+            // Update the event handlers
+            ptr = this.fields[i].name;
+            // TODO: This should be changed in EasyDAG
+            this.fields[i].selectTarget = this.getValidNestedLayers.bind(this, ptr);
+            i++;
+        }
+        return y;
+    };
+
+    LayerDecorator.prototype.getValidNestedLayers = function(ptr) {
+        var tgtId = this._node.pointers[ptr];
+        if (tgtId) {
+            WebGMEGlobal.State.registerActiveObject(tgtId);
+        } else {
+            this.createLayerArg(ptr);
+        }
+    };
+
+    LayerDecorator.prototype.createLayerArg = function(ptr) {
+        // Find the Architecture node type
+        var metanodes = this.client.getAllMetaNodes(),
+            base = metanodes.find(node => node.getAttribute('name') === 'Architecture'),
+            baseId,
+            msg = `Creating layers for "${ptr}" of ${this._node.name}`,
+            tgtId;
+
+        if (!base) {
+            return this.logger.error('Could not find "Architecture" type!');
+        }
+
+        // Create a nested "architecture" node and set the ptr target to it
+        baseId = base.getId();
+        this.client.startTransaction(msg);
+        tgtId = this.client.createNode({
+            parentId: this._node.id,
+            baseId: baseId
+        });
+        this.client.setAttribute(tgtId, 'name', `${ptr} (${this._node.name})`);
+        this.savePointer(ptr, tgtId);
+        this.client.completeTransaction();
+        WebGMEGlobal.State.registerActiveObject(tgtId);
+    };
+
+    LayerDecorator.prototype.savePointer = function(ptr, to) {
+        if (!to) {  // delete the current target
+            var currentId = this._node.pointers[ptr],
+                name = this._node.name;
+
+            // If the target is contained in the current node, delete it!
+            if (currentId.indexOf(this._node.id) === 0) {
+                this.client.startTransaction(`Removing layer for ${ptr} of ${name}`);
+                this.client.deleteNode(currentId);
+                this.client.completeTransaction();
+                this.logger.info(`Removed ${ptr} and deleted target (${currentId})`);
+            } else {
+                this.logger.info(`Removed ${ptr} (external architecture)`);
+            }
+        } else {  // create and set the node
+            EllipseDecorator.prototype.savePointer.apply(this, arguments);
+        }
+    };
+
+    return LayerDecorator;
+});

src/decorators/LayerDecorator/EasyDAG/LayerField.js

@@ -0,0 +1,29 @@
+/* globals define, _ */
+define([
+    'decorators/EllipseDecorator/EasyDAG/PointerField'
+], function(
+    PointerField
+) {
+    // The LayerField behaves the same as PointerFields but it shows "click to view"
+    // if it has a value
+    var LayerField = function() {
+        PointerField.apply(this, arguments);
+    };
+
+    _.extend(LayerField.prototype, PointerField.prototype);
+
+    LayerField.prototype.getContent = function(content) {
+        return content && 'click to view';
+    };
+
+    LayerField.prototype.createContent = function(w, y, content) {
+        PointerField.prototype.createContent.call(this, w, y, this.getContent(content));
+        this.$content.attr('font-style', 'italic');
+    };
+
+    LayerField.prototype.setValue = function(content) {
+        PointerField.prototype.setValue.call(this, this.getContent(content));
+    };
+
+    return LayerField;
+});

src/decorators/LayerDecorator/LayerDecorator.js

@@ -0,0 +1,39 @@
+/*globals define, _*/
+/*jshint browser: true, camelcase: false*/
+
+define([
+    'js/Decorators/DecoratorBase',
+    './EasyDAG/LayerDecorator.EasyDAGWidget'
+], function (
+    DecoratorBase,
+    LayerDecoratorEasyDAGWidget
+) {
+
+    'use strict';
+
+    var LayerDecorator,
+        __parent__ = DecoratorBase,
+        __parent_proto__ = DecoratorBase.prototype,
+        DECORATOR_ID = 'LayerDecorator';
+
+    LayerDecorator = function (params) {
+        var opts = _.extend({loggerName: this.DECORATORID}, params);
+
+        __parent__.apply(this, [opts]);
+
+        this.logger.debug('LayerDecorator ctor');
+    };
+
+    _.extend(LayerDecorator.prototype, __parent_proto__);
+    LayerDecorator.prototype.DECORATORID = DECORATOR_ID;
+
+    /*********************** OVERRIDE DecoratorBase MEMBERS **************************/
+
+    LayerDecorator.prototype.initializeSupportedWidgetMap = function () {
+        this.supportedWidgetMap = {
+            EasyDAG: LayerDecoratorEasyDAGWidget
+        };
+    };
+
+    return LayerDecorator;
+});

src/decorators/OpIntDecorator/EasyDAG/AttributeField.js

@@ -0,0 +1,16 @@
+/* globals define, _*/
+define([
+    'decorators/EllipseDecorator/EasyDAG/AttributeField'
+], function(
+    AttributeFieldBase
+) {
+    // Attribute field in which the label is clickable and the attribute meta is editable
+    var AttributeField = function() {
+        AttributeFieldBase.apply(this, arguments);
+        this.$label.on('click', () => this.onLabelClick());
+    };
+
+    _.extend(AttributeField.prototype, AttributeFieldBase.prototype);
+
+    return AttributeField;
+});

src/decorators/OpIntDecorator/EasyDAG/CreateAttributeField.js

@@ -0,0 +1,26 @@
+/* globals define */
+define([
+], function(
+) {
+
+    var CreateAttrField = function(logger, pEl, y) {
+        this.$el = pEl.append('text')
+            .attr('y', y)
+            .attr('class', 'create-attr-field')
+            .attr('text-anchor', 'middle')
+            .attr('dominant-baseline', 'middle')
+            .attr('font-weight', 'bold')
+            .attr('font-style', 'italic')
+            .text('New Attribute')
+            .on('click', () => this.onClick());
+    };
+
+    CreateAttrField.prototype.render =
+    CreateAttrField.prototype.destroy = function() {};
+
+    CreateAttrField.prototype.width = function() {
+        return this.$el[0][0].getBoundingClientRect().width;
+    };
+
+    return CreateAttrField;
+});

src/decorators/OpIntDecorator/EasyDAG/OpIntDecorator.EasyDAGWidget.css

@@ -0,0 +1,43 @@
+/**
+ * @author brollb / https://github.com/brollb
+ */
+.op-int-decorator {
+  min-width: 65px;
+  height: 40px;
+  border: 1px solid black;
+  background-color: #dedede;
+  padding: 3px;
+  text-align: center; }
+  .op-int-decorator .attr-title {
+      font-style: italic;
+  }
+  .op-int-decorator .name {
+    margin-top: 10px;
+    white-space: nowrap;
+    font-size: 16px;
+    font-weight: bold;
+    font-family: "Helvetica Neue",Helvetica,Arial,sans-serif;
+    text-align: center; }
+  .op-int-decorator .connector {
+    background-color: #fefefe;
+    height: 10px;
+    width: 10px;
+    position: absolute;
+    cursor: pointer;
+    border: 1px solid blue;
+    z-index: 10;
+    margin-left: -6px;
+    left: 50%; }
+    .op-int-decorator .connector:hover {
+      border-color: rgba(82, 168, 236, 0.8);
+      -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6);
+      -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6);
+      box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6); }
+    .op-int-decorator .connector.top {
+      top: -6px; }
+    .op-int-decorator .connector.bottom {
+      bottom: -6px; }
+
+.selected .op-int-decorator {
+  border: 1px solid #52a8ec;
+  background-color: #dbeafc; }

src/decorators/OpIntDecorator/EasyDAG/OpIntDecorator.EasyDAGWidget.js

@@ -0,0 +1,193 @@
+/*globals define, $,_*/
+/*jshint browser: true, camelcase: false*/
+
+define([
+    'decorators/EllipseDecorator/EasyDAG/EllipseDecorator.EasyDAGWidget',
+    './AttributeField',
+    './CreateAttributeField',
+    'decorators/MetaDecorator/DiagramDesigner/AttributeDetailsDialog',
+    'css!./OpIntDecorator.EasyDAGWidget.css'
+], function (
+    DecoratorBase,
+    AttributeField,
+    CreateAttributeField,
+    AttributeDetailsDialog
+) {
+
+    'use strict';
+
+    var OpIntDecorator,
+        DECORATOR_ID = 'OpIntDecorator';
+
+    OpIntDecorator = function (options) {
+        DecoratorBase.call(this, options);
+    };
+
+    _.extend(OpIntDecorator.prototype, DecoratorBase.prototype);
+
+    OpIntDecorator.prototype.DECORATOR_ID = DECORATOR_ID;
+    OpIntDecorator.prototype.initialize = function() {
+        if (this.isOperation()) {
+            this.color = '#2196f3';
+        } else if (this._node.baseName) {
+            // On hover, show the type
+            this.enableTooltip(this._node.baseName, 'dark');
+        }
+        DecoratorBase.prototype.initialize.call(this);
+        this.$name.on('dblclick', this.editName.bind(this));
+    };
+
+    OpIntDecorator.prototype.AttributeField = AttributeField;
+    OpIntDecorator.prototype.isOperation = function() {
+        return this._node.baseName === 'Operation';
+    };
+
+    OpIntDecorator.prototype.createAttributeFields = function(y, width) {
+        var field,
+            initialY = y;
+
+        if (!this.isOperation()) {
+            return y;
+        }
+
+        y = DecoratorBase.prototype.createAttributeFields.call(this, y, width);
+        // Change attribute field so clicking allows user to edit/delete the field
+        this.fields.forEach(field =>
+            field.onLabelClick = this.editAttributeMeta.bind(this, field.name));
+
+        // Add the 'create new attribute' field
+        y += this.ROW_HEIGHT + (y === initialY ? 0 : 10);
+        field = new CreateAttributeField(this.logger, this.$attributes, y, width);
+        field.onClick = this.newAttribute.bind(this);
+        this.fields.push(field);
+        return y;
+    };
+
+    OpIntDecorator.prototype.newAttribute = function() {
+        var defSchema = {
+            type: 'string'
+        };
+
+        this.editAttributeMeta(null, defSchema);
+    };
+
+    OpIntDecorator.prototype.expand = function() {
+        DecoratorBase.prototype.expand.call(this, this.isOperation());
+    };
+
+    OpIntDecorator.prototype.editAttributeMeta = function(name, defSchema) {
+        var dialog = new AttributeDetailsDialog(),
+            node = this.client.getNode(this._node.id),
+            attrNames = node.getValidAttributeNames(),
+            attrInfo = this._node.attributes[name] || defSchema,
+            schema,
+            i;
+
+        // Open the dialog for editing the attribute
+        schema = _.extend({defaultValue: attrInfo.value}, attrInfo);
+
+        // Remove the current name
+        i = attrNames.indexOf(name);
+        if (i !== -1) {
+            attrNames.splice(i, 1);
+        }
+
+        dialog.show(schema, attrNames,
+            desc => this.setAttributeMeta(name, desc),
+            () => this.deleteAttribute(name));
+    };
+
+    OpIntDecorator.prototype.deleteAttribute = function(name) {
+        var opName = this._node.attributes.name.value,
+            msg = `Deleting "${name}" attribute from "${opName}" operation`;
+
+        this.client.startTransaction(msg);
+        this.client.delAttributeMeta(this._node.id, name);
+        this.client.delAttribute(this._node.id, name);
+        this.client.completeTransaction();
+    };
+
+    OpIntDecorator.prototype.setAttributeMeta = function(name, desc) {
+        var schema,
+            opName = this._node.attributes.name.value,
+            msg = `Updating "${name}" attribute in "${opName}" operation`;
+
+        // Create the schema from the desc
+        schema = {
+            type: desc.type,
+            min: desc.min,
+            max: desc.max,
+            regexp: desc.regexp
+        };
+
+        if (desc.isEnum) {
+            schema.enum = desc.enumValues;
+        }
+
+        // Update the operation's attribute
+        this.client.startTransaction(msg);
+
+        if (name !== desc.name) {  // Renaming attribute
+            if (name) {
+                this.client.delAttributeMeta(this._node.id, name);
+                this.client.delAttribute(this._node.id, name);
+            }
+            name = desc.name;
+        }
+
+        this.client.setAttributeMeta(this._node.id, name, schema);
+        this.client.setAttribute(this._node.id, name, desc.defaultValue);
+        this.client.completeTransaction();
+    };
+
+    OpIntDecorator.prototype.editName = function() {
+        var html = this.$name[0][0],
+            position = html.getBoundingClientRect(),
+
+            width = Math.max(position.right-position.left, 15),
+            container = $('<div>'),
+            parentHtml = $('body');
+
+        // foreignObject was not working so we are using a tmp container
+        // instead
+        container.css('top', position.top);
+        container.css('left', position.left);
+        container.css('position', 'absolute');
+        container.css('width', width);
+        container.attr('id', 'CONTAINER-TMP');
+
+        $(parentHtml).append(container);
+
+        container.editInPlace({
+            enableEmpty: true,
+            value: this.name,
+            css: {
+                'z-index': 10000,
+                'id': 'asdf',
+                'width': width,
+                'xmlns': 'http://www.w3.org/1999/xhtml'
+            },
+            onChange: this.onNameChanged.bind(this),
+            onFinish: function () {
+                $(this).remove();
+            }
+        });
+    };
+
+    OpIntDecorator.prototype.onNameChanged = function(oldVal, newValue) {
+        var whitespace = /^\s*$/;
+        if (newValue !== oldVal && !whitespace.test(newValue)) {
+            this.onValidNameChange(newValue);
+        }
+    };
+
+    OpIntDecorator.prototype.onValidNameChange = function(newValue) {
+        this.saveAttribute('name', newValue);
+    };
+
+    OpIntDecorator.prototype.getDisplayName = function() {
+        return this._node.name;
+    };
+
+    return OpIntDecorator;
+});

src/decorators/OpIntDecorator/OpIntDecorator.js

@@ -0,0 +1,43 @@
+/*globals define, _*/
+/*jshint browser: true, camelcase: false*/
+
+/**
+ * @author brollb / https://github.com/brollb
+ */
+
+define([
+    'js/Decorators/DecoratorBase',
+    './EasyDAG/OpIntDecorator.EasyDAGWidget'
+], function (
+    DecoratorBase,
+    OpIntDecoratorEasyDAGWidget
+) {
+
+    'use strict';
+
+    var OpIntDecorator,
+        __parent__ = DecoratorBase,
+        __parent_proto__ = DecoratorBase.prototype,
+        DECORATOR_ID = 'OpIntDecorator';
+
+    OpIntDecorator = function (params) {
+        var opts = _.extend({loggerName: this.DECORATORID}, params);
+
+        __parent__.apply(this, [opts]);
+
+        this.logger.debug('OpIntDecorator ctor');
+    };
+
+    _.extend(OpIntDecorator.prototype, __parent_proto__);
+    OpIntDecorator.prototype.DECORATORID = DECORATOR_ID;
+
+    /*********************** OVERRIDE DecoratorBase MEMBERS **************************/
+
+    OpIntDecorator.prototype.initializeSupportedWidgetMap = function () {
+        this.supportedWidgetMap = {
+            EasyDAG: OpIntDecoratorEasyDAGWidget
+        };
+    };
+
+    return OpIntDecorator;
+});

src/decorators/OpIntPtrDecorator/EasyDAG/OpIntPtrDecorator.EasyDAGWidget.css

@@ -0,0 +1,43 @@
+/**
+ * @author brollb / https://github.com/brollb
+ */
+.op-int-decorator {
+  min-width: 65px;
+  height: 40px;
+  border: 1px solid black;
+  background-color: #dedede;
+  padding: 3px;
+  text-align: center; }
+  .op-int-decorator .attr-title {
+      font-style: italic;
+  }
+  .op-int-decorator .name {
+    margin-top: 10px;
+    white-space: nowrap;
+    font-size: 16px;
+    font-weight: bold;
+    font-family: "Helvetica Neue",Helvetica,Arial,sans-serif;
+    text-align: center; }
+  .op-int-decorator .connector {
+    background-color: #fefefe;
+    height: 10px;
+    width: 10px;
+    position: absolute;
+    cursor: pointer;
+    border: 1px solid blue;
+    z-index: 10;
+    margin-left: -6px;
+    left: 50%; }
+    .op-int-decorator .connector:hover {
+      border-color: rgba(82, 168, 236, 0.8);
+      -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6);
+      -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6);
+      box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6); }
+    .op-int-decorator .connector.top {
+      top: -6px; }
+    .op-int-decorator .connector.bottom {
+      bottom: -6px; }
+
+.selected .op-int-decorator {
+  border: 1px solid #52a8ec;
+  background-color: #dbeafc; }

src/decorators/OpIntPtrDecorator/EasyDAG/OpIntPtrDecorator.EasyDAGWidget.js

@@ -0,0 +1,39 @@
+/*globals define, _*/
+/*jshint browser: true, camelcase: false*/
+
+/**
+ * @author brollb / https://github.com/brollb
+ */
+
+define([
+    'decorators/OpIntDecorator/EasyDAG/OpIntDecorator.EasyDAGWidget',
+    'css!./OpIntPtrDecorator.EasyDAGWidget.css'
+], function (
+    DecoratorBase
+) {
+
+    'use strict';
+
+    var OpIntPtrDecorator,
+        DECORATOR_ID = 'OpIntPtrDecorator';
+
+    // OpInt nodes need to be able to...
+    //     - show their ports
+    //     - highlight ports
+    //     - unhighlight ports
+    //     - report the location of specific ports
+    OpIntPtrDecorator = function (options) {
+        this.color = '#80deea';
+        DecoratorBase.call(this, options);
+    };
+
+    _.extend(OpIntPtrDecorator.prototype, DecoratorBase.prototype);
+
+    OpIntPtrDecorator.prototype.DECORATOR_ID = DECORATOR_ID;
+
+    OpIntPtrDecorator.prototype.onValidNameChange  = function(newValue) {
+        return this.changePtrName(this.name, newValue);
+    };
+
+    return OpIntPtrDecorator;
+});

src/decorators/OpIntPtrDecorator/OpIntPtrDecorator.js

@@ -0,0 +1,43 @@
+/*globals define, _*/
+/*jshint browser: true, camelcase: false*/
+
+/**
+ * @author brollb / https://github.com/brollb
+ */
+
+define([
+    'js/Decorators/DecoratorBase',
+    './EasyDAG/OpIntPtrDecorator.EasyDAGWidget'
+], function (
+    DecoratorBase,
+    OpIntPtrDecoratorEasyDAGWidget
+) {
+
+    'use strict';
+
+    var OpIntPtrDecorator,
+        __parent__ = DecoratorBase,
+        __parent_proto__ = DecoratorBase.prototype,
+        DECORATOR_ID = 'OpIntPtrDecorator';
+
+    OpIntPtrDecorator = function (params) {
+        var opts = _.extend({loggerName: this.DECORATORID}, params);
+
+        __parent__.apply(this, [opts]);
+
+        this.logger.debug('OpIntPtrDecorator ctor');
+    };
+
+    _.extend(OpIntPtrDecorator.prototype, __parent_proto__);
+    OpIntPtrDecorator.prototype.DECORATORID = DECORATOR_ID;
+
+    /*********************** OVERRIDE DecoratorBase MEMBERS **************************/
+
+    OpIntPtrDecorator.prototype.initializeSupportedWidgetMap = function () {
+        this.supportedWidgetMap = {
+            EasyDAG: OpIntPtrDecoratorEasyDAGWidget
+        };
+    };
+
+    return OpIntPtrDecorator;
+});

src/decorators/OperationDecorator/EasyDAG/OperationDecorator.EasyDAGWidget.css

@@ -0,0 +1,43 @@
+/**
+ * @author brollb / https://github.com/brollb
+ */
+.operation-decorator {
+  min-width: 65px;
+  height: 40px;
+  border: 1px solid black;
+  background-color: #dedede;
+  padding: 3px;
+  text-align: center; }
+  .operation-decorator .attr-title {
+      font-style: italic;
+  }
+  .operation-decorator .name {
+    margin-top: 10px;
+    white-space: nowrap;
+    font-size: 16px;
+    font-weight: bold;
+    font-family: "Helvetica Neue",Helvetica,Arial,sans-serif;
+    text-align: center; }
+  .operation-decorator .connector {
+    background-color: #fefefe;
+    height: 10px;
+    width: 10px;
+    position: absolute;
+    cursor: pointer;
+    border: 1px solid blue;
+    z-index: 10;
+    margin-left: -6px;
+    left: 50%; }
+    .operation-decorator .connector:hover {
+      border-color: rgba(82, 168, 236, 0.8);
+      -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6);
+      -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6);
+      box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6); }
+    .operation-decorator .connector.top {
+      top: -6px; }
+    .operation-decorator .connector.bottom {
+      bottom: -6px; }
+
+.selected .operation-decorator {
+  border: 1px solid #52a8ec;
+  background-color: #dbeafc; }

src/decorators/OperationDecorator/EasyDAG/OperationDecorator.EasyDAGWidget.js

@@ -0,0 +1,166 @@
+/*globals define, _, Opentip*/
+/*jshint browser: true, camelcase: false*/
+
+define([
+    'decorators/EllipseDecorator/EasyDAG/EllipseDecorator.EasyDAGWidget',
+    'css!./OperationDecorator.EasyDAGWidget.css'
+], function (
+    DecoratorBase
+) {
+
+    'use strict';
+
+    var OperationDecorator,
+        NAME_MARGIN = 25,
+        DECORATOR_ID = 'OperationDecorator',
+        PORT_TOOLTIP_OPTS = {
+            tipJoint: 'left',
+            removeElementsOnHide: true,
+            style: 'dark'
+        };
+
+    // Operation nodes need to be able to...
+    //     - show their ports
+    //     - highlight ports
+    //     - unhighlight ports
+    //     - report the location of specific ports
+    OperationDecorator = function (options) {
+        options.color = options.color || '#78909c';
+        DecoratorBase.call(this, options);
+
+        this.id = this._node.id;
+        this.$ports = this.$el.append('g')
+            .attr('id', 'ports');
+        this.$portTooltips = {};
+    };
+
+    _.extend(OperationDecorator.prototype, DecoratorBase.prototype);
+
+    OperationDecorator.prototype.DECORATOR_ID = DECORATOR_ID;
+    OperationDecorator.prototype.PORT_COLOR = {
+        OPEN: '#90caf9',
+        OCCUPIED: '#e57373'
+    };
+
+    OperationDecorator.prototype.condense = function() {
+        var width = Math.max(this.nameWidth + 2 * NAME_MARGIN, this.dense.width);
+
+        this.$body
+            .transition()
+            .attr('x', -width/2)
+            .attr('y', 0)
+            .attr('width', width)
+            .attr('height', this.dense.height);
+
+        // Clear the attributes
+        this.clearFields();
+        this.$attributes = this.$el.append('g')
+            .attr('fill', 'none');
+
+        this.createAttributeFields(0, width);
+        this.createPointerFields(0, width, true);
+
+        this.height = this.dense.height;
+        this.width = width;
+
+        this.$name.attr('y', this.height/2);
+
+        this.$el
+            .attr('transform', `translate(${this.width/2}, 0)`);
+        this.expanded = false;
+        this.onResize();
+    };
+
+    OperationDecorator.prototype.showPorts = function(ids, areInputs) {
+        var allPorts = areInputs ? this._node.inputs : this._node.outputs,
+            x = -this.width/2,
+            dx = this.width/(allPorts.length+1),
+            y = areInputs ? 0 : this.height;  // (this.height/2);
+
+        allPorts.forEach(port => {
+            x += dx;
+            if (!ids || ids.indexOf(port.id) > -1) {
+                this.renderPort(port, x, y, areInputs);
+            }
+        });
+    };
+
+    OperationDecorator.prototype.renderPort = function(port, x, y, isInput) {
+        var color = this.PORT_COLOR.OPEN,
+            portIcon = this.$ports.append('g'),
+            tooltip;
+
+        // If the port is incoming and occupied, render it differently
+        if (isInput && port.connection) {
+            color = this.PORT_COLOR.OCCUPIED;
+        }
+
+        portIcon.append('circle')
+            .attr('cx', x)
+            .attr('cy', y)
+            .attr('r', 10)
+            .attr('fill', color);
+            
+        portIcon.append('text')
+                .attr('x', x)
+                .attr('y', y)
+                .attr('text-anchor', 'middle')
+                .attr('dominant-baseline', 'middle')
+                .attr('fill', 'black')
+                .text(port.name[0]);
+
+        portIcon.on('click', this.onPortClick.bind(this, this.id, port.id, !isInput));
+
+        // Add tooltip with whole name
+        if (this.$portTooltips[port.id]) {
+            this.$portTooltips[port.id].hide();
+        }
+        tooltip = new Opentip(portIcon[0][0], PORT_TOOLTIP_OPTS);
+        tooltip.setContent(port.name);
+        portIcon.on('mouseenter', () => tooltip.show());
+        portIcon.on('mouseleave', () => tooltip.hide());
+        this.$portTooltips[port.id] = tooltip;
+    };
+
+    OperationDecorator.prototype.hidePorts = function() {
+        var visiblePortIds = Object.keys(this.$portTooltips);
+        this.logger.info(`hiding ports for ${this.name} (${this.id})`);
+        this.$ports.remove();
+        this.$ports = this.$el.append('g')
+            .attr('id', 'ports');
+        
+        for (var i = visiblePortIds.length; i--;) {
+            this.$portTooltips[visiblePortIds[i]].hide();
+        }
+    };
+
+    OperationDecorator.prototype.getPortLocation = function(id, isInput) {
+        // Report location of given port
+        var ports = isInput ? this._node.inputs : this._node.outputs,
+            i = ports.length-1,
+            y;
+
+        while (i >= 0 && ports[i].id !== id) {
+            i--;
+        }
+        if (i !== -1) {
+            i += 1;
+            y = (this.height/2);
+            return {
+                x: i * this.width/(ports.length+1),
+                y: isInput ? y * -1 : y
+            };
+        }
+        return null;
+    };
+
+    OperationDecorator.prototype.onPortClick = function() {
+        // Overridden in the widget
+    };
+
+    OperationDecorator.prototype.getDisplayName = function() {
+        return this._node.name;
+    };
+
+    return OperationDecorator;
+});

src/decorators/OperationDecorator/OperationDecorator.js

@@ -0,0 +1,43 @@
+/*globals define, _*/
+/*jshint browser: true, camelcase: false*/
+
+/**
+ * @author brollb / https://github.com/brollb
+ */
+
+define([
+    'js/Decorators/DecoratorBase',
+    './EasyDAG/OperationDecorator.EasyDAGWidget'
+], function (
+    DecoratorBase,
+    OperationDecoratorEasyDAGWidget
+) {
+
+    'use strict';
+
+    var OperationDecorator,
+        __parent__ = DecoratorBase,
+        __parent_proto__ = DecoratorBase.prototype,
+        DECORATOR_ID = 'OperationDecorator';
+
+    OperationDecorator = function (params) {
+        var opts = _.extend({loggerName: this.DECORATORID}, params);
+
+        __parent__.apply(this, [opts]);
+
+        this.logger.debug('OperationDecorator ctor');
+    };
+
+    _.extend(OperationDecorator.prototype, __parent_proto__);
+    OperationDecorator.prototype.DECORATORID = DECORATOR_ID;
+
+    /*********************** OVERRIDE DecoratorBase MEMBERS **************************/
+
+    OperationDecorator.prototype.initializeSupportedWidgetMap = function () {
+        this.supportedWidgetMap = {
+            EasyDAG: OperationDecoratorEasyDAGWidget
+        };
+    };
+
+    return OperationDecorator;
+});

src/layouts/SidebarLayout/SidebarLayout.css

@@ -0,0 +1,12 @@
+.ui-layout-center .layout-center {
+    left: 40px;
+    height: 100%;
+    width: 100%;
+    position: absolute;
+}
+
+.ui-layout-sidebar {
+    top: 64px;
+    width: 40px;
+    bottom: 27px;
+}

src/layouts/SidebarLayout/SidebarLayout.js

@@ -0,0 +1,52 @@
+/*globals define, */
+define([
+    'layout/CHFLayout/CHFLayout/CHFLayout',
+    'text!./templates/SidebarLayout.html',
+    'css!./SidebarLayout.css'
+], function(
+    CHFLayout,
+    SidebarTemplate
+) {
+    'use strict';
+    
+    var SidebarLayout = function(params) {
+        params = params || {};
+        params.template = SidebarTemplate;
+        CHFLayout.call(this, params);
+    };
+
+    SidebarLayout.prototype = Object.create(CHFLayout.prototype);
+
+    SidebarLayout.prototype.getComponentId = function () {
+        return 'SidebarLayout';
+    };
+
+    /**
+     * Initialize the html page. This example is using the jQuery Layout plugin.
+     *
+     * @return {undefined}
+     */
+    SidebarLayout.prototype.init = function() {
+        CHFLayout.prototype.init.apply(this, arguments);
+        this._sidebarPanel = this._body.find('div.ui-layout-sidebar');
+        this._centerPanel = this._body.find('div.layout-center');
+    };
+
+    /**
+     * Add a panel to a given container. This is defined in the corresponding
+     * layout config JSON file.
+     *
+     * @param {Panel} panel
+     * @param {String} container
+     * @return {undefined}
+     */
+    SidebarLayout.prototype.addToContainer = function(panel, container) {
+        if (container === 'sidebar') {
+            this._sidebarPanel.append(panel.$pEl);
+        } else {
+            CHFLayout.prototype.addToContainer.apply(this, arguments);
+        }
+    };
+
+    return SidebarLayout;
+});

src/layouts/SidebarLayout/templates/SidebarLayout.html

@@ -0,0 +1,7 @@
+<div class="ui-layout-center" style="position: relative;">
+    <div class="layout-center"></div>
+    <div class="float"></div>
+</div>
+<div class="ui-layout-sidebar"></div>
+<div class="ui-layout-north"></div>
+<div class="ui-layout-south"></div>

src/plugins/CheckLibraries/CheckLibraries.js

@@ -0,0 +1,185 @@
+/*globals define*/
+/*jshint node:true, browser:true*/
+
+define([
+    'text!./metadata.json',
+    'module',
+    'path',
+    'fs',
+    'q',
+    'plugin/PluginBase'
+], function (
+    pluginMetadata,
+    module,
+    path,
+    fs,
+    Q,
+    PluginBase
+) {
+    'use strict';
+
+    pluginMetadata = JSON.parse(pluginMetadata);
+    var __dirname = path.dirname(module.uri),
+        SEEDS_DIR = path.join(__dirname, '..', '..', 'seeds');
+
+    /**
+     * Initializes a new instance of CheckLibraries.
+     * @class
+     * @augments {PluginBase}
+     * @classdesc This class represents the plugin CheckLibraries.
+     * @constructor
+     */
+    var CheckLibraries = function () {
+        // Call base class' constructor.
+        PluginBase.call(this);
+        this.pluginMetadata = pluginMetadata;
+        this.libraries = {};
+    };
+
+    /**
+     * Metadata associated with the plugin. Contains id, name, version, description, icon, configStructue etc.
+     * This is also available at the instance at this.pluginMetadata.
+     * @type {object}
+     */
+    CheckLibraries.metadata = pluginMetadata;
+
+    // Prototypical inheritance from PluginBase.
+    CheckLibraries.prototype = Object.create(PluginBase.prototype);
+    CheckLibraries.prototype.constructor = CheckLibraries;
+
+    /**
+     * Main function for the plugin to execute. This will perform the execution.
+     * Notes:
+     * - Always log with the provided logger.[error,warning,info,debug].
+     * - Do NOT put any user interaction logic UI, etc. inside this method.
+     * - callback always has to be called even if error happened.
+     *
+     * @param {function(string, plugin.PluginResult)} callback - the result callback
+     */
+    CheckLibraries.prototype.main = function (callback) {
+        var tuples;
+
+        return this.getAllLibraries()
+            .then(libs => {
+                tuples = libs.map(lib => {  // map to [name, version, dir]
+                    var version,
+                        hash,
+                        data,
+                        versionPath = this.getSeedVersionPath(lib);
+
+                    try {
+                        this.logger.info(`Checking for version info at ${versionPath}`);
+                        version = fs.readFileSync(versionPath, 'utf8');
+                        this.logger.debug(`${lib} version is ${version}`);
+                        data = fs.readFileSync(this.getSeedHashPath(lib), 'utf8').split(' ');
+                        if (data[1] === version) {
+                            hash = data[0];
+                            this.logger.debug(`${lib} hash is ${hash}`);
+                        }
+                    } catch (e) {
+                        if (!version) {
+                            this.logger.warn(`Could not find library version for ${lib}`);
+                        } else {
+                            this.logger.warn(`Could not find library hash for ${lib}`);
+                        }
+                    }
+
+                    return [lib, version, hash];
+                })
+                .filter(tuple => {
+                    var projVersion = this.getLoadedVersion(tuple[0]),
+                        latest = tuple[1].replace(/\s+/g, '');
+
+                    this.logger.info(`${tuple[0]} version info:\n${projVersion} ` +
+                        `(project)\n${latest} (latest)`);
+                    return latest !== projVersion;
+                });
+
+                return Q.all(tuples.map(tuple => this.uploadSeed.apply(this, tuple)));
+            })
+            .then(hashes => {
+                var name;
+
+                for (var i = hashes.length; i--;) {
+                    name = tuples[i][0];
+                    this.createMessage(this.libraries[name], `${name} ${hashes[i]}`);
+                }
+
+                this.logger.info(`Found ${hashes.length} out of date libraries`);
+                this.result.setSuccess(true);
+                callback(null, this.result);
+            })
+            .fail(err => {
+                this.logger.error(`Could not check the libraries: ${err}`);
+                callback(err, this.result);
+            });
+    };
+
+    CheckLibraries.prototype.getSeedDir = function (name) {
+        return path.join(SEEDS_DIR, name);
+    };
+
+    CheckLibraries.prototype.getSeedDataPath = function (name) {
+        return path.join(this.getSeedDir(name), name + '.webgmex');
+    };
+
+    CheckLibraries.prototype.getSeedHashPath = function (name) {
+        return path.join(this.getSeedDir(name), 'hash.txt');
+    };
+
+    CheckLibraries.prototype.getSeedVersionPath = function (name) {
+        return path.join(this.getSeedDir(name), 'version.txt');
+    };
+
+    CheckLibraries.prototype.uploadSeed = function (name, version, hash) {
+        if (!hash) {  // Upload the seed
+            // Get the data
+            return Q.nfcall(fs.readFile, this.getSeedDataPath(name))
+                .then(data => {
+                    this.logger.info(`Uploading new version of ${name} (${version})`);
+                    return this.blobClient.putFile(`${name}.webgmex`, data);
+                })
+                .then(newHash => {  // Store the new hash
+                    this.logger.info(`Upload of ${name} finished!`);
+                    hash = newHash;
+                    return Q.nfcall(
+                        fs.writeFile,
+                        this.getSeedHashPath(name),
+                        `${hash} ${version}`
+                    );
+                }).then(() => hash);
+        }
+        return hash;
+    };
+
+    CheckLibraries.prototype.getAllLibraries = function () {
+        var name,
+            names = [];
+
+        return this.core.loadChildren(this.rootNode)
+            .then(children => {
+                for (var i = children.length; i--;) {
+                    if (this.core.isLibraryRoot(children[i])) {
+                        name = this.core.getAttribute(children[i], 'name');
+                        this.libraries[name] = children[i];
+                        names.push(name);
+                    }
+                }
+                if (names.length) {
+                    this.logger.debug(`Found libraries: ${names.join(', ')}`);
+                } else {
+                    this.logger.debug('Found no libraries!');
+                }
+                return names;
+            });
+    };
+
+    CheckLibraries.prototype.getLoadedVersion = function (libName) {
+        var node = this.libraries[libName],
+            version = this.core.getAttribute(node, 'version');  // using library root hash
+
+        return version;
+    };
+
+    return CheckLibraries;
+});

src/plugins/CheckLibraries/metadata.json

@@ -0,0 +1,14 @@
+{
+  "id": "CheckLibraries",
+  "name": "CheckLibraries",
+  "version": "0.1.0",
+  "description": "",
+  "icon": {
+    "class": "glyphicon glyphicon-cog",
+    "src": ""
+  },
+  "disableServerSideExecution": false,
+  "disableBrowserSideExecution": true,
+  "writeAccessRequired": false,
+  "configStructure": []
+}

src/plugins/CreateExecution/CreateExecution.js

@@ -0,0 +1,393 @@
+/*globals define*/
+/*jshint node:true, browser:true*/
+
+define([
+    'q',
+    'deepforge/plugin/LocalExecutor',
+    'text!./metadata.json',
+    'underscore',
+    'plugin/PluginBase'
+], function (
+    Q,
+    LocalExecutor,
+    pluginMetadata,
+    _,
+    PluginBase
+) {
+    'use strict';
+
+    pluginMetadata = JSON.parse(pluginMetadata);
+
+    /**
+     * Initializes a new instance of CreateExecution.
+     * @class
+     * @augments {PluginBase}
+     * @classdesc This class represents the plugin CreateExecution.
+     * @constructor
+     */
+    var CreateExecution = function () {
+        // Call base class' constructor.
+        PluginBase.call(this);
+        this.pluginMetadata = pluginMetadata;
+    };
+
+    /**
+     * Metadata associated with the plugin. Contains id, name, version, description, icon, configStructue etc.
+     * This is also available at the instance at this.pluginMetadata.
+     * @type {object}
+     */
+    CreateExecution.metadata = pluginMetadata;
+
+    // Prototypical inheritance from PluginBase.
+    CreateExecution.prototype = Object.create(PluginBase.prototype);
+    CreateExecution.prototype.constructor = CreateExecution;
+
+    /**
+     * Main function for the plugin to execute. This will perform the execution.
+     * Notes:
+     * - Always log with the provided logger.[error,warning,info,debug].
+     * - Do NOT put any user interaction logic UI, etc. inside this method.
+     * - callback always has to be called even if error happened.
+     *
+     * @param {function(string, plugin.PluginResult)} callback - the result callback
+     */
+    CreateExecution.prototype.main = function (callback) {
+        // Verify that the node is a pipeline
+        if (!this.core.isTypeOf(this.activeNode, this.META.Pipeline)) {
+            return callback('Current node is not a Pipeline!', this.result);
+        }
+
+        return this.createExecution(this.activeNode)
+            .then(() => {
+                this.result.setSuccess(true);
+                callback(null, this.result);
+            })
+            .catch(err => callback(err, this.result));
+    };
+
+    CreateExecution.prototype.getExecutionDir = function () {
+        return this.core.loadChildren(this.rootNode)
+            .then(children => {
+                var execPath = this.core.getPath(this.META.Execution);
+
+                // Find a node in the root that can contain only executions
+                return children.find(child => {
+                    var metarule = this.core.getChildrenMeta(child);
+                    return metarule && metarule[execPath];
+                }) || this.rootNode;  // default to rootNode
+            });
+    };
+
+    CreateExecution.prototype.createExecution = function (node) {
+        // Get the user supplied name
+        var name = this.core.getAttribute(node, 'name'),
+            config = this.getCurrentConfig(),
+            basename = config.name || (name + '_execution');
+            
+
+        // Given a pipeline, copy all the operations to a custom job
+        //   - Copy the operations 
+        //   - Wrap the operations in "Job" boxes which contain running info
+        //   - Update the references
+        var tgtNode,
+            execName,
+            copies,
+            opTuples,  // [[op, index], [op, index], ...]
+            dataMapping = {};
+
+        return this.getExecutionDir()
+            .then(execDir => {
+                var execDirId = this.core.getPath(execDir),
+                    execTypeId = this.core.getPath(this.META.Execution);
+
+                this.logger.debug(`Creating execution node in ${execDirId} (type is ${execTypeId})`);
+                tgtNode = this.core.createNode({
+                    base: this.META.Execution,
+                    parent: execDir
+                });
+                this.logger.debug(`New execution created w/ id: ${this.core.getPath(tgtNode)}`);
+
+                // Get a unique name
+                this.logger.debug(`About to get a unique name starting w/ ${basename}`);
+                return this.getUniqueExecName(basename);
+            })
+            .then(_execName => {
+                var isSnapshot = !this.getCurrentConfig().debug,
+                    originName = this.core.getAttribute(this.activeNode, 'name'),
+                    oId = this.core.getPath(this.activeNode),
+                    tgtId = this.core.getPath(tgtNode);
+
+                execName = _execName;
+                this.logger.debug(`Configuring execution attributes (${execName})`);
+
+                // Set all the metadata for the new execution
+                this.core.setAttribute(tgtNode, 'name', execName);
+                this.core.setAttribute(tgtNode, 'snapshot', isSnapshot);
+                this.core.setAttribute(tgtNode, 'tagname', execName);
+                this.core.setAttribute(tgtNode, 'createdAt', Date.now());
+                this.logger.debug(`Setting origin pipeline to ${originName} (${oId})`);
+                this.core.setPointer(tgtNode, 'origin', this.activeNode);
+                this.logger.debug(`Adding ${tgtId} to execution list of ${originName} (${oId})`);
+                this.core.addMember(this.activeNode, 'executions', tgtNode);
+
+                this.logger.debug(`Creating tag "${execName}"`);
+            })
+            .then(() => this.core.loadChildren(node))
+            .then(children => {
+                if (!children.length) {
+                    this.logger.warn('No children in pipeline. Will proceed anyway');
+                }
+
+                this.logger.debug(`Copying operations to "${execName}"`);
+                return this.copyOperations(children, tgtNode);
+            })
+            .then(copiedPairs => {
+                var originals = copiedPairs.map(pair => pair[0]);
+                copies = copiedPairs.map(pair => pair[1]);
+                opTuples = copies
+                    .map((copy, i) => [copy, i])  // zip w/ index
+                    .filter(pair => this.core.isTypeOf(pair[0], this.META.Operation));
+
+                // Create a mapping of old names to new names
+                this.logger.debug('Creating mapping of old->new');
+                return Q.all(opTuples.map(pair =>
+                        // Add the input/output mappings to the dataMapping
+                        this.addDataToMap(originals[pair[1]], pair[0], dataMapping)
+                    )
+                );
+            })
+            .then(() => {  // datamapping is set!
+                this.logger.debug('Updating references...');
+                this.updateReferences(copies, dataMapping);
+                this.logger.debug('Placing operations in Job containers');
+                this.boxOperations(opTuples.map(o => o[0]), tgtNode);
+                this.logger.debug('Finished! Saving...');
+                return this.save(`Created execution from ${name}`);
+            })
+            .then(() => this.project.createTag(execName, this.currentHash))
+            .then(() => tgtNode);  // return tgtNode
+    };
+
+    CreateExecution.prototype.getUniqueExecName = function (basename) {
+        var taken = {},
+            name,
+            i = 2;
+
+        basename = basename.replace(/[^\da-zA-Z_]/g, '_');
+        name = basename;
+
+        // Get a unique name wrt the tags and the other executions
+        return this.project.getTags()
+            .then(tags => {
+                Object.keys(tags).forEach(name => taken[name] = true);
+                this.logger.debug(`Existing tags are ${Object.keys(tags).join(',')}`);
+
+                // Get the other executions
+                return this.getExecutionDir();
+            })
+            .then(execDir => {
+                var cIds = this.core.getChildrenPaths(execDir);
+                this.logger.debug(`Current executions are ${cIds.join(', ')}`);
+                return Q.all(cIds.map(id => this.core.loadByPath(this.rootNode, id)));
+            })
+            .then(execs => {
+                var names = execs.map(exec => this.core.getAttribute(exec, 'name'));
+                this.logger.debug(`Existing names are ${names.join(',')}`);
+                names.forEach(name => taken[name] = true);
+
+                while (taken[name]) {
+                    name = basename + '_' + (i++);
+                }
+                this.logger.debug(`Unique name is "${name}"`);
+                return name;
+            });
+    };
+
+    CreateExecution.prototype.copyOperations = function (nodes, dst) {
+        var snapshot = !this.getCurrentConfig().debug;
+
+        if (snapshot) {
+            this.logger.debug('Execution is a snapshot -> severing the inheritance');
+            return Q.all(nodes.map(node => {
+                if (this.isLocalOperation(node) ||
+                    this.isMetaTypeOf(node, this.META.Transporter)) {
+
+                    return [[node, this.core.copyNode(node, dst)]];
+                } else if (this.isMetaTypeOf(node, this.META.Operation)) {
+                    return this.snapshotNode(node, dst);
+                }
+            }))
+            .then(pairs => pairs.filter(pair => !!pair)
+                .reduce((l1, l2) => l1.concat(l2))
+            );
+
+        } else if (nodes.length) {
+            this.logger.debug('Execution is not a snapshot -> doing a simple copy');
+            var copies = this.core.copyNodes(nodes, dst);
+            return nodes.map((node, i) => [node, copies[i]]);
+        }
+        return [];
+    };
+
+    CreateExecution.prototype.snapshotNode = function (op, dst) {
+        // If we are making a snapshot, we should copy the base operation
+        // and set the attributes, add the child nodes, etc
+        var base = this.core.getBase(this.core.getBase(op)),
+            names,
+            values,
+            snapshot = this.core.createNode({
+                base: base,
+                parent: dst
+            });
+
+        // Copy over the attributes
+        names = this.core.getValidAttributeNames(op);
+        values = names.map(name => this.core.getAttribute(op, name));
+        names.forEach((name, i) =>
+            this.core.setAttribute(snapshot, name, values[i]));
+
+        // Copy the pointers
+        names = this.core.getValidPointerNames(op);
+        return Q.all(names
+            .map(name => this.core.getPointerPath(op, name))
+            .map(id => this.core.loadByPath(this.rootNode, id)))
+        .then(values => {
+
+            names.forEach((name, i) =>
+                this.core.setPointer(snapshot, name, values[i]));
+
+            // Copy the data I/O
+            var srcCntrs = this.core.getChildrenPaths(op),
+                dstCntrs = this.core.getChildrenPaths(snapshot);
+
+            return Q.all([srcCntrs, dstCntrs].map(ids =>
+                Q.all(ids.map(id => this.core.loadByPath(this.rootNode, id)))));
+        })
+        .then(cntrs => {
+            var srcCntrs,
+                dstCntrs;
+
+            // Sort all containers by metatype id
+            cntrs.map(l => l.sort((a, b) => {
+                var aId = this.core.getPath(this.core.getMetaType(a)),
+                    bId = this.core.getPath(this.core.getMetaType(b));
+
+                return aId < bId ? -1 : 1;
+            }));
+
+            srcCntrs = cntrs[0];
+            dstCntrs = cntrs[1];
+            return Q.all(srcCntrs.map(ctr => Q.all(this.core.getChildrenPaths(ctr)
+                    .map(id => this.core.loadByPath(this.rootNode, id)))))
+                .then(cntrs =>
+                    cntrs.map((nodes, i) =>
+                        nodes.map(n => [n, this.copyDataNode(n, dstCntrs[i])]))
+                );
+        })
+        .then(nodes => {
+            nodes = nodes.reduce((l1, l2) => l1.concat(l2), []);
+            nodes.push([op, snapshot]);
+            return nodes;
+        });
+    };
+
+    CreateExecution.prototype.copyDataNode = function (original, dst) {
+        // Create new node of the given type
+        var attrNames = this.core.getAttributeNames(original),
+            values,
+            copy = this.core.createNode({
+                base: this.core.getMetaType(original),
+                parent: dst
+            });
+
+        // Set the 'name', 'data' attributes
+        values = attrNames.map(name => this.core.getAttribute(original, name));
+        attrNames.forEach((name, i) =>
+            this.core.setAttribute(copy, name, values[i]));
+
+        return copy;
+    };
+
+    CreateExecution.prototype.addDataToMap = function (srcOp, dstOp, map) {
+        return Q.all(
+            [srcOp, dstOp]
+                .map(op => {
+                    // Get the inputs and outputs for both
+                    return this.core.loadChildren(op)
+                        .then(containers => {
+                            var names = containers.map(c => this.core.getAttribute(c, 'name')),
+                                inputs = containers
+                                    .find((c, i) => names[i] === 'Inputs'),
+                                outputs = containers
+                                    .find((c, i) => names[i] === 'Outputs');
+
+                            return Q.all(
+                                [inputs, outputs].map(c => c ? this.core.loadChildren(c) : [])
+                            );
+                        });
+                })
+            )
+            .then(ios => {
+                var srcIO,
+                    dstIO;
+
+                srcIO = ios[0].map(c => this.sortIOByName(c));
+                dstIO = ios[1].map(c => this.sortIOByName(c));
+
+                // match the nodes by same name!
+                srcIO.forEach((srcContainer, c) => srcContainer.forEach((node, n) => 
+                    map[this.core.getPath(node)] = dstIO[c][n]  // old id -> new node
+                    )
+                );
+                return true;
+            });
+    };
+
+    CreateExecution.prototype.sortIOByName = function (container) {
+        return container.sort((a, b) =>
+            // sort by name
+            this.core.getAttribute(a, 'name') < this.core.getAttribute(b, 'name') ? 1 : -1
+        );
+    };
+
+    // Wrap each Operation with a Job 'box'
+    CreateExecution.prototype.boxOperations = function (operations, container) {
+        operations.forEach(copy => {
+            var name = this.core.getAttribute(copy, 'name'),
+                job;
+
+            // Create job
+            job = this.core.createNode({
+                base: this.META.Job,
+                parent: container
+            });
+            this.core.setAttribute(job, 'name', name);
+
+            // Move the given copy into the Job node
+            this.core.moveNode(copy, job);
+        });
+    };
+
+    CreateExecution.prototype.updateReferences = function (nodes, map) {
+        // For each new node, update the references (other than base)
+        // to the correct nodeId
+        nodes.forEach(copy => {
+            this.core.getPointerNames(copy)
+                .filter(name => name !== 'base')
+                .forEach(name => {
+                    var tgt = this.core.getPointerPath(copy, name);
+                    if (map[tgt]) {
+                        this.logger.info(`Updating ptr ${name}`);
+                        this.core.setPointer(copy, name, map[tgt]);
+                    }
+                });
+        });
+    };
+
+    _.extend(
+        CreateExecution.prototype,
+        LocalExecutor.prototype
+    );
+
+    return CreateExecution;
+});

src/plugins/CreateExecution/metadata.json

@@ -0,0 +1,23 @@
+{
+  "id": "CreateExecution",
+  "name": "Create Execution",
+  "version": "0.1.0",
+  "description": "",
+  "icon": {
+    "class": "glyphicon glyphicon-cog",
+    "src": ""
+  },
+  "disableServerSideExecution": false,
+  "disableBrowserSideExecution": false,
+  "writeAccessRequired": false,
+  "configStructure": [
+      {
+            "name": "snapshot",
+            "displayName": "Snapshot",
+            "description": "Freeze the operation definitions and attributes at current value",
+            "value": true,
+            "valueType": "boolean",
+            "readOnly": false
+      }
+  ]
+}

src/plugins/CreateTorchMeta/CreateTorchMeta.js

@@ -1,21 +1,25 @@
 /*globals define*/
 /*jshint node:true, browser:true*/
 
-/**
- * Generated by PluginGenerator 0.14.0 from webgme on Tue Mar 15 2016 21:19:45 GMT-0500 (CDT).
- */
-
 define([
-    'plugin/PluginConfig',
     'plugin/PluginBase',
-    'deepforge/js-yaml.min',
-    'text!deepforge/layers.yml',
+    'common/util/guid',
+    'deepforge/Constants',
+    'deepforge/utils',
+    'js/RegistryKeys',
+    'js/Panels/MetaEditor/MetaEditorConstants',
+    'underscore',
+    './schemas/index',
     'text!./metadata.json'
 ], function (
-    PluginConfig,
     PluginBase,
-    yaml,
-    DEFAULT_LAYERS,
+    generateGuid,
+    Constants,
+    utils,
+    REGISTRY_KEYS,
+    META_CONSTANTS,
+    _,
+    Schemas,
     metadata
 ) {
     'use strict';
@@ -31,6 +35,8 @@ define([
         // Call base class' constructor.
         PluginBase.call(this);
         this.pluginMetadata = CreateTorchMeta.metadata;
+        this.metaSheets = {};
+        this.sheetCounts = {};
     };
 
     CreateTorchMeta.metadata = JSON.parse(metadata);
@@ -49,129 +55,272 @@ define([
      * @param {function(string, plugin.PluginResult)} callback - the result callback
      */
     CreateTorchMeta.prototype.main = function (callback) {
-        // Use self to access core, project, result, logger etc from PluginBase.
-        // These are all instantiated at this point.
-        var self = this,
-            nodeObject;
-
         if (!this.META.Language) {
-            callback('"Language" container required to run plugin', this.result);
+            return callback('"Language" container required to run plugin', this.result);
         }
 
         // Extra layer names
-        this.getYamlText((err, text) => {
-            if (err) {
-                return callback(err, this.result);
+        // The format is...
+        //      - (Abstract) CategoryLayerTypes
+        //          - LayerName
+        //              - Attributes (if exists)
+        var layers,
+            content = {},
+            categories,
+            config = this.getCurrentConfig(),
+            nodes = {};
+
+        try {
+            layers = this.getJsonLayers();
+        } catch (e) {
+            return callback('JSON parse error: ' + e, this.result);
+        }
+        layers.forEach(layer => {
+            if (!content[layer.type]) {
+                content[layer.type] = [];
             }
-
-            // The format is...
-            //      - (Abstract) CategoryLayerTypes
-            //          - LayerName
-            //              - Attributes (if exists)
-            var content,
-                categories,
-                nodes = {};
-
-            try {
-                content = yaml.load(text);
-            } catch (e) {
-                return callback('YAML parse error: ' + e, this.result);
-            }
-            categories = Object.keys(content);
-            // Create the base class, if needed
-            if (!this.META.Layer) {
-                this.META.Layer = this.createMetaNode('Layer', this.META.FCO);
-            }
-
-            // Create the category nodes
-            categories
-                .forEach(name => nodes[name] = this.createMetaNode(name, this.META.Layer));
-
-            // Make them abstract
-            categories
-                .forEach(name => this.core.setRegistry(nodes[name], 'isAbstract', true));
-            
-
-            // Create the actual nodes
-            categories.forEach(cat => {
-                content[cat]
-                    .forEach(name => {
-                        var attrs = null;
-                        if (typeof name !== 'string') {
-                            attrs = name[Object.keys(name)[0]];
-                            name = Object.keys(name)[0];
-                        }
-                        nodes[name] = this.createMetaNode(name, nodes[cat], attrs);
-                        // Make the node non-abstract
-                        this.core.setRegistry(nodes[name], 'isAbstract', false);
-                    });
-            });
-
-            self.save('CreateTorchMeta updated model.', function (err) {
-                if (err) {
-                    callback(err, self.result);
-                    return;
-                }
-                self.result.setSuccess(true);
-                callback(null, self.result);
-            });
+            content[layer.type].push(layer);
         });
 
+        categories = Object.keys(content);
+        // Create the base class, if needed
+        if (!this.META.Layer) {
+            this.META.Layer = this.createMetaNode('Layer', this.META.FCO);
+        }
 
+        // Create the category nodes
+        categories
+            .forEach(name => {
+                // Create a tab for each
+                this.metaSheets[name] = this.createMetaSheetTab(name);
+                this.sheetCounts[name] = 0;
+                nodes[name] = this.createMetaNode(name, this.META.Layer, name);
+            });
+
+        // Make them abstract
+        categories
+            .forEach(name => this.core.setRegistry(nodes[name], 'isAbstract', true));
+
+        if (config.removeOldLayers) {
+            var isNewLayer = {},
+                newLayers = layers.map(layer => layer.name),
+                oldLayers,
+                oldNames;
+
+            newLayers = newLayers.concat(categories);  // add the category nodes
+            newLayers.forEach(name => isNewLayer[name] = true);
+
+            // Set the newLayer nodes 'base' to 'Layer' so we don't accidentally
+            // delete them
+            newLayers
+                .map(name => this.META[name])
+                .filter(layer => !!layer)
+                .forEach(layer => this.core.setBase(layer, this.META.Layer));
+
+            oldLayers = Object.keys(this.META)
+                    .filter(name => name !== 'Layer')
+                    .map(name => this.META[name])
+                    .filter(node => this.isMetaTypeOf(node, this.META.Layer))
+                    .filter(node => !isNewLayer[this.core.getAttribute(node, 'name')]);
+
+            oldNames = oldLayers.map(l => this.core.getAttribute(l, 'name'));
+            // Get the old layer names
+            this.logger.debug(`Removing layers: ${oldNames.join(', ')}`);
+            oldLayers.forEach(layer => this.core.deleteNode(layer));
+        }
+
+        // Create the actual nodes
+        categories.forEach(cat => {
+            content[cat]
+                .forEach(layer => {
+                    var name = layer.name,
+                        node;
+
+                    node = this.createMetaNode(name, nodes[cat], cat, layer);
+                    // Make the node non-abstract
+                    if (node) {
+                        this.core.setRegistry(node, 'isAbstract', false);
+                        nodes[name] = node;
+                    }
+                });
+        });
+
+        this.save('CreateTorchMeta updated model.')
+            .then(() => {
+                this.result.setSuccess(true);
+                callback(null, this.result);
+            })
+            .fail(err => callback(err, this.result));
     };
 
-    CreateTorchMeta.prototype.getYamlText = function (callback) {
-        var config = this.getCurrentConfig();
+    CreateTorchMeta.prototype.removeFromMeta = function (nodeId) {
+        var sheets = this.core.getRegistry(this.rootNode, REGISTRY_KEYS.META_SHEETS),
+            sheet;
 
-        if (config.layerNameHash) {
-            this.blobClient.getObject(config.layerNameHash, (err, buffer) => {
-                if (err) {
-                    return callback(err, this.result);
-                }
-                var text = String.fromCharCode.apply(null, new Uint8Array(buffer));
-                return callback(null, text);
+        // Remove from meta
+        this.core.delMember(this.rootNode, META_CONSTANTS.META_ASPECT_SET_NAME, nodeId);
+
+        // Remove from the given meta sheet
+        sheet = sheets.find(sheet => {
+            var paths = this.core.getMemberPaths(this.rootNode, sheet.SetID);
+            return paths.indexOf(nodeId) > -1;
+        });
+
+        if (sheet) {
+            this.core.delMember(this.rootNode, sheet.SetID, nodeId);
+        }
+    };
+
+    CreateTorchMeta.prototype.createMetaSheetTab = function (name) {
+        var sheets = this.core.getRegistry(this.rootNode, REGISTRY_KEYS.META_SHEETS),
+            id = META_CONSTANTS.META_ASPECT_SHEET_NAME_PREFIX + generateGuid(),
+            sheet,
+            desc = {
+                SetID: id,
+                order: sheets.length,
+                title: name
+            };
+
+        sheet = sheets.find(sheet => sheet.title === name);
+        if (!sheet) {
+            sheet = desc;
+            this.logger.debug(`creating meta sheet "${name}"`);
+            this.core.createSet(this.rootNode, sheet.SetID);
+            sheets.push(sheet);
+            this.core.setRegistry(this.rootNode, REGISTRY_KEYS.META_SHEETS, sheets);
+        }
+        return sheet.SetID;
+    };
+
+    CreateTorchMeta.prototype.getJsonLayers = function () {
+        var config = this.getCurrentConfig(),
+            schema = config.layerSchema;
+
+        if (schema === 'all') {
+            return Object.keys(Schemas).map(key => JSON.parse(Schemas[key]))
+                .reduce((l1, l2) => l1.concat(l2), []);
+        }
+
+        return JSON.parse(Schemas[schema]);
+    };
+
+    // Some helper methods w/ attribute handling
+    var LUA_TO_GME = {
+        boolean: 'boolean',
+        number: 'float',
+        string: 'string'
+    };
+
+    var isLayerAttribute = type => type && type.substring(0, 3) === 'nn.';
+
+    CreateTorchMeta.prototype.createMetaNode = function (name, base, tabName, layer) {
+        var node = this.META[name],
+            nodeId = node && this.core.getPath(node),
+            tabId = this.metaSheets[tabName],
+            position = this.getPositionFor(name, tabName),
+            setters = {},
+            defaults = {},
+            types = {},
+            type,
+            attrs,
+            desc;
+
+        if (layer) {
+            attrs = layer.params;
+            setters = layer.setters;
+            defaults = layer.defaults;
+            types = layer.types || types;
+        }
+        if (!tabId) {
+            this.logger.error(`No meta sheet for ${tabName}`);
+        }
+
+        if (!node) {
+            // Create a node
+            node = this.core.createNode({
+                parent: this.META.Language,
+                base: base
             });
+            this.core.setAttribute(node, 'name', name);
+
+            nodeId = this.core.getPath(node);
         } else {
-            return callback(null, DEFAULT_LAYERS);
+            // Remove from meta
+            this.removeFromMeta(nodeId);
+            this.core.setBase(node, base);
         }
-    };
-
-    CreateTorchMeta.prototype.createMetaNode = function (name, base, attrs) {
-        var node;
-
-        if (this.META[name]) {
-            this.logger.warn('"' + name + '" already exists. skipping...');
-            return this.META[name];
-        }
-
-        // Create a node
-        node = this.core.createNode({
-            parent: this.META.Language,
-            base: base
-        });
-        this.core.setAttribute(node, 'name', name);
 
         // Add it to the meta sheet
-        this.core.addMember(this.rootNode, 'MetaAspectSet', node);
+        this.core.addMember(this.rootNode, META_CONSTANTS.META_ASPECT_SET_NAME, node);
+        this.core.addMember(this.rootNode, tabId, node);
 
-        // Add it to a tab of the meta sheet
-        var set = this.core.getSetNames(this.rootNode)
-            .find(name => name !== 'MetaAspectSet');
-
-        this.core.addMember(this.rootNode, set, node);
-        // TODO: Position the nodes on the META
-        // TODO: Put each group of nodes on their own META sheet
+        this.core.setMemberRegistry(
+            this.rootNode,
+            META_CONSTANTS.META_ASPECT_SET_NAME,
+            nodeId,
+            REGISTRY_KEYS.POSITION,
+            position
+        );
+        this.core.setMemberRegistry(
+            this.rootNode,
+            tabId,
+            nodeId,
+            REGISTRY_KEYS.POSITION,
+            position
+        );
 
         if (attrs) {  // Add the attributes
-            attrs.forEach((name, index) => {
-                var desc = null;
-                if (typeof name !== 'string') {
-                    desc = name[Object.keys(name)[0]];
-                    name = Object.keys(name)[0];
+            // Remove attributes not in the given list
+            var currentAttrs = this.core.getValidAttributeNames(node),
+                defVal,
+                rmAttrs,
+                simpleAttrs,
+                rmPtrs;
+
+            simpleAttrs = attrs.filter(name => !isLayerAttribute(types[name]));
+            rmAttrs = _.difference(currentAttrs, simpleAttrs)  // old attribute names
+                .filter(attr => attr !== 'name')
+                .filter(attr => !setters[attr]);
+
+            rmAttrs.forEach(attr => {
+                this.core.delAttributeMeta(node, attr);
+                if (this.core.getOwnAttribute(node, attr) !== undefined) {
+                    this.core.delAttribute(node, attr);
                 }
-                desc = desc || {};
-                desc.argindex = index;
-                this.addAttribute(name, node, desc);
+            });
+
+            // Remove all old pointers
+            rmPtrs = _.difference(this.core.getPointerNames(node), currentAttrs)
+                .filter(ptr => ptr !== 'base');
+
+            if (rmPtrs.length + rmAttrs.length) {
+                this.logger.debug(`Removing ${rmPtrs.concat(rmAttrs).join(', ')} from ${name}`);
+            }
+            rmPtrs.forEach(ptr => this.core.delPointerMeta(node, ptr));
+
+            attrs.forEach(name => {
+                desc = {};
+                defVal = defaults.hasOwnProperty(name) ? defaults[name] : '';
+                type = LUA_TO_GME[types[name]];
+                if (type) {
+                    desc.type = type;
+                }
+                if (isLayerAttribute(types[name])) {  // Check if it is an nn layer type
+                    // If so, create a pointer rather than attribute
+                    this.addLayerAttribute(name, node);
+                    this.logger.debug(`${name} is a layer type attribute`);
+                } else {
+                    this.addAttribute(name, node, desc, defVal);
+                }
+            });
+            this.core.setAttribute(node, Constants.CTOR_ARGS_ATTR, attrs.join(','));
+
+            // Add the setters to the meta
+            Object.keys(setters).forEach(name => {
+                desc = utils.getSetterSchema(name, setters, defaults);
+                defVal = desc.default;
+                delete desc.default;
+                this.addAttribute(name, node, desc, defVal);
             });
         }
         this.logger.debug(`added ${name} to the meta`);
@@ -179,42 +328,60 @@ define([
         return node;
     };
 
-    CreateTorchMeta.prototype.addAttribute = function (name, node, def) {
-        var initial,
-            schema = {};
+    CreateTorchMeta.prototype.getPositionFor = function(name, tabName) {
+        var index = this.sheetCounts[tabName],
+            dx = 140,
+            dy = 100,
+            MAX_WIDTH = 1200,
+            x;
 
-        schema.type = def.type || 'integer';
+        if (tabName === 'Convolution') {
+            dx *= 1.3;
+            dy *= 1.5;
+        }
+
+        this.sheetCounts[tabName]++;
+        if (index === 0) {
+            return {
+                x: MAX_WIDTH/2,
+                y: 50
+            };
+        }
+
+        x = dx*index;
+        return {
+            x: x%MAX_WIDTH,
+            y: Math.floor(x/MAX_WIDTH+1)*dy + 50
+        };
+    };
+
+    CreateTorchMeta.prototype.addLayerAttribute = function (name, node) {
+        // No default value support for now...
+        // Create a pointer of the given type on the node
+        this.core.setPointerMetaTarget(node, name, this.META.Architecture, 1, 1);
+        this.core.setPointerMetaLimits(node, name, 1, 1);
+    };
+
+    CreateTorchMeta.prototype.addAttribute = function (name, node, schema, defVal) {
+        schema.type = schema.type || 'string';
         if (schema.type === 'list') {  // FIXME: add support for lists
             schema.type = 'string';
         }
 
-        if (def.min !== undefined) {
-            schema.min = +def.min;
+        if (schema.min !== undefined) {
+            schema.min = +schema.min;
         }
 
-        if (def.max !== undefined) {
+        if (schema.max !== undefined) {
             // Set the min, max
-            schema.max = +def.max;
+            schema.max = +schema.max;
         }
 
-        // Add the infer flag
-        if (def.infer) {
-            schema.infer = def.infer;
-        }
-
-        // Add the argindex flag
-        schema.argindex = def.argindex;
-
         // Create the attribute and set the schema
         this.core.setAttributeMeta(node, name, schema);
 
-        // Determine a default value
-        initial = def.hasOwnProperty('default') ? def.default : def.min || null;
-        if (schema.type === 'boolean') {
-            initial = initial !== null ? initial : false;
-        }
-        if (initial !== null) {  // optional attribute - set default value
-            this.core.setAttribute(node, name, initial);
+        if (defVal) {
+            this.core.setAttribute(node, name, defVal);
         }
     };
 

src/plugins/CreateTorchMeta/metadata.json

@@ -1,22 +1,35 @@
 {
   "id": "CreateTorchMeta",
-  "name": "Create Torch Meta",
+  "name": "Update nn meta",
   "version": "0.1.0",
   "description": "Create metamodel from Torch yaml",
   "icon": {
     "src": "",
     "class": "glyphicon glyphicon-ok-circle"
   },
-  "disableServerSideExecution": false,
+  "disableServerSideExecution": true,
   "disableBrowserSideExecution": false,
   "configStructure": [
-      {
-          "name": "layerNameHash",
-          "displayName": "Torch Layers",
-          "description": "Yaml file of torch layer descriptors (optional)",
-          "value": "",
-          "valueType": "asset",
-          "readOnly": false
-      }
+    {
+      "name": "layerSchema",
+      "displayName": "Torch Libraries",
+      "description": "Torch nn libraries to create layers from",
+      "value": "all",
+      "valueItems": [
+        "nn",
+        "rnn",
+        "all"
+      ],
+      "valueType": "string",
+      "readOnly": false
+    },
+    {
+      "name": "removeOldLayers",
+      "displayName": "Delete old layers",
+      "description": "Delete all layers not in the current description",
+      "value": true,
+      "valueType": "boolean",
+      "readOnly": false
+    }
   ]
-}
+}

src/plugins/CreateTorchMeta/schemas/index.js

@@ -0,0 +1,13 @@
+/*globals define*/
+define([
+    'text!./nn.json',
+    'text!./rnn.json'
+], function(
+    nn,
+    rnn
+) {
+    return {
+        nn: nn,
+        rnn: rnn
+    };
+});

src/plugins/CreateTorchMeta/schemas/rnn.json

@@ -0,0 +1,178 @@
+[
+  {
+    "name": "CopyGrad",
+    "baseType": "Identity",
+    "setters": {},
+    "defaults": {},
+    "type": "RNN"
+  },
+  {
+    "name": "FastLSTM",
+    "baseType": "LSTM",
+    "params": [
+      "inputSize",
+      "outputSize",
+      "rho",
+      "eps",
+      "momentum",
+      "affine"
+    ],
+    "setters": {},
+    "types": {
+      "eps": "number",
+      "momentum": "number"
+    },
+    "defaults": {
+      "momentum": 0.1,
+      "eps": 0.1
+    },
+    "type": "RNN"
+  },
+  {
+    "name": "LSTM",
+    "baseType": "AbstractRecurrent",
+    "params": [
+      "inputSize",
+      "outputSize",
+      "rho",
+      "cell2gate"
+    ],
+    "setters": {},
+    "types": {
+      "rho": "number"
+    },
+    "defaults": {
+      "rho": 9999
+    },
+    "type": "RNN"
+  },
+  {
+    "name": "LinearNoBias",
+    "baseType": "Linear",
+    "params": [
+      "inputSize",
+      "outputSize"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Simple"
+  },
+  {
+    "name": "LookupTableMaskZero",
+    "baseType": "LookupTable",
+    "params": [
+      "nIndex",
+      "nOutput"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "RNN"
+  },
+  {
+    "name": "NormStabilizer",
+    "baseType": "AbstractRecurrent",
+    "params": [
+      "beta"
+    ],
+    "setters": {},
+    "defaults": {},
+    "type": "RNN"
+  },
+  {
+    "name": "Recurrent",
+    "baseType": "AbstractRecurrent",
+    "params": [
+      "start",
+      "input",
+      "feedback",
+      "transfer",
+      "rho",
+      "merge"
+    ],
+    "setters": {},
+    "types": {
+      "start": "nn.Module",
+      "transfer": "nn.Module",
+      "feedback": "nn.Module",
+      "input": "nn.Module"
+    },
+    "defaults": {},
+    "type": "RNN"
+  },
+  {
+    "name": "SAdd",
+    "baseType": "Module",
+    "params": [
+      "addend",
+      "negate"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "RNN"
+  },
+  {
+    "name": "SeqBRNN",
+    "baseType": "Container",
+    "params": [
+      "inputDim",
+      "hiddenDim",
+      "batchFirst"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "RNN"
+  },
+  {
+    "name": "SeqGRU",
+    "baseType": "Module",
+    "params": [
+      "inputSize",
+      "outputSize"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "RNN"
+  },
+  {
+    "name": "SeqLSTM",
+    "baseType": "Module",
+    "params": [
+      "inputsize",
+      "hiddensize",
+      "outputsize"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "RNN"
+  },
+  {
+    "name": "SeqLSTMP",
+    "baseType": "SeqLSTM",
+    "params": [
+      "inputsize",
+      "hiddensize",
+      "outputsize"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "RNN"
+  },
+  {
+    "name": "SeqReverseSequence",
+    "baseType": "Module",
+    "params": [
+      "dim"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "RNN"
+  }
+]

src/plugins/CreateTorchMeta/update-schemas.js

@@ -0,0 +1,37 @@
+/* eslint-disable no-console */
+// Update the metadata and schemas/index based on the new schemas in schemas/
+
+// Update metadata
+var fs = require('fs'),
+    path = require('path'),
+    schemas,
+    metadata = require('./metadata.json'),
+    schemaList;
+
+schemas = fs.readdirSync(__dirname + '/schemas/')
+    .filter(name => path.extname(name) === '.json')
+    .map(name => name.replace(/\.json$/, ''));
+
+console.log('Discovered schemas: ' + schemas.join(', '));
+
+schemaList = metadata.configStructure.find(struct => struct.name === 'layerSchema');
+schemaList.valueItems = schemas.concat('all');
+
+console.log('Updating metadata...');
+fs.writeFileSync(__dirname + '/metadata.json', JSON.stringify(metadata, null, 2));
+
+// Update index.js
+var index =
+`/*globals define*/
+define([
+    ${schemas.map(s => `'text!./${s}.json'`).join(',\n    ')}
+], function(
+    ${schemas.map(s => s).join(',\n    ')}
+) {
+    return {
+        ${schemas.map(s => s + ': ' + s).join(',\n        ')}
+    };
+});`;
+
+console.log('Updating index.js...');
+fs.writeFileSync(__dirname + '/schemas/index.js', index);

src/plugins/ExecuteJob/ExecuteJob.Files.js

@@ -0,0 +1,338 @@
+/*globals define*/
+define([
+    './templates/index',
+    'q',
+    'underscore',
+    'deepforge/Constants'
+], function(
+    Templates,
+    Q,
+    _,
+    CONSTANTS
+) {
+
+    var ExecuteJob = function() {
+    };
+
+    ExecuteJob.prototype.createOperationFiles = function (node) {
+        var files = {};
+        // For each operation, generate the output files:
+        //   inputs/<arg-name>/init.lua  (respective data deserializer)
+        //   pointers/<name>/init.lua  (result of running the main plugin on pointer target - may need a rename)
+        //   outputs/<name>/  (make dirs for each of the outputs)
+        //   outputs/init.lua  (serializers for data outputs)
+        //
+        //   attributes.lua (returns lua table of operation attributes)
+        //   init.lua (main file -> calls main and serializes outputs)
+        //   <name>.lua (entry point -> calls main operation code)
+
+        // add the given files
+        this.logger.info('About to create dist execution files');
+        files['start.js'] = _.template(Templates.START)(CONSTANTS);
+        return this.createEntryFile(node, files)
+            .then(() => this.createClasses(node, files))
+            .then(() => this.createCustomLayers(node, files))
+            .then(() => this.createInputs(node, files))
+            .then(() => this.createOutputs(node, files))
+            .then(() => this.createMainFile(node, files))
+            .then(() => {
+                this.createAttributeFile(node, files);
+                return Q.ninvoke(this, 'createPointers', node, files);
+            });
+    };
+
+    ExecuteJob.prototype.createEntryFile = function (node, files) {
+        this.logger.info('Creating entry files...');
+        return this.getOutputs(node)
+            .then(outputs => {
+                var name = this.getAttribute(node, 'name'),
+                    content = {};
+
+                // inputs and outputs
+                content.name = name;
+                content.outputs = outputs;
+
+                files['init.lua'] = _.template(Templates.ENTRY)(content);
+
+                // Create the deepforge file
+                files['deepforge.lua'] = _.template(Templates.DEEPFORGE)(CONSTANTS);
+            });
+    };
+
+    ExecuteJob.prototype.createClasses = function (node, files) {
+        var metaDict = this.core.getAllMetaNodes(this.rootNode),
+            isClass,
+            metanodes,
+            classNodes,
+            inheritanceLvl = {},
+            code;
+
+        this.logger.info('Creating custom layer file...');
+        metanodes = Object.keys(metaDict).map(id => metaDict[id]);
+        isClass = this.getTypeDictFor('Complex', metanodes);
+
+        classNodes = metanodes.filter(node => {
+            var base = this.core.getBase(node),
+                baseId = this.core.getPath(base),
+                count = 1;
+
+            // Count the sets back to a class node
+            while (base) {
+                if (isClass[baseId]) {
+                    inheritanceLvl[this.core.getPath(node)] = count;
+                    return true;
+                }
+                base = this.core.getBase(base);
+                baseId = this.core.getPath(base);
+                count++;
+            }
+
+            return false;
+        });
+
+        // Get the code definitions for each
+        // Sort by levels of inheritance...
+        code = classNodes.sort((a, b) => {
+            var aId = this.core.getPath(a),
+                bId = this.core.getPath(b);
+
+            return inheritanceLvl[aId] > inheritanceLvl[bId];
+        }).map(node =>
+            `require './${this.getAttribute(node, 'name')}.lua'`
+        ).join('\n');
+
+        // Create the class files
+        classNodes.forEach(node => {
+            var name = this.getAttribute(node, 'name');
+            files[`classes/${name}.lua`] = this.getAttribute(node, 'code');
+        });
+
+        // Create the custom layers file
+        files['classes/init.lua'] = code;
+    };
+
+    ExecuteJob.prototype.getTypeDictFor = function (name, metanodes) {
+        var isType = {};
+        // Get all the custom layers
+        for (var i = metanodes.length; i--;) {
+            if (this.getAttribute(metanodes[i], 'name') === name) {
+                isType[this.core.getPath(metanodes[i])] = true;
+            }
+        }
+        return isType;
+    };
+
+    ExecuteJob.prototype.createCustomLayers = function (node, files) {
+        var metaDict = this.core.getAllMetaNodes(this.rootNode),
+            isCustomLayer,
+            metanodes,
+            customLayers,
+            code;
+
+        this.logger.info('Creating custom layer file...');
+        metanodes = Object.keys(metaDict).map(id => metaDict[id]);
+        isCustomLayer = this.getTypeDictFor('CustomLayer', metanodes);
+
+        customLayers = metanodes.filter(node =>
+            this.core.getMixinPaths(node).some(id => isCustomLayer[id]));
+
+        // Get the code definitions for each
+        code = 'require \'nn\'\n\n' + customLayers
+            .map(node => this.getAttribute(node, 'code')).join('\n');
+
+        // Create the custom layers file
+        files['custom-layers.lua'] = code;
+    };
+
+    ExecuteJob.prototype.createInputs = function (node, files) {
+        var tplContents,
+            inputs;
+
+        this.logger.info('Retrieving inputs and deserialize fns...');
+        return this.getInputs(node)
+            .then(allInputs => {
+                // For each input, match the connection with the input name
+                //   [ name, type ] => [ name, type, node ]
+                //
+                // For each input,
+                //  - create the deserializer
+                //  - put it in inputs/<name>/init.lua
+                //  - copy the data asset to /inputs/<name>/init.lua
+                inputs = allInputs
+                    .filter(pair => !!this.getAttribute(pair[2], 'data'));  // remove empty inputs
+
+                files.inputAssets = {};  // data assets
+                return Q.all(inputs.map(pair => {
+                    var name = pair[0],
+                        node = pair[2],
+                        nodeId = this.core.getPath(node),
+                        fromNodeId;
+
+                    // Get the deserialize function. First, try to get it from
+                    // the source method (this guarantees that the correct
+                    // deserialize method is used despite any auto-upcasting
+                    fromNodeId = this.inputPortsFor[nodeId][0] || nodeId;
+
+                    return this.core.loadByPath(this.rootNode, fromNodeId)
+                        .then(fromNode => {
+                            var deserFn,
+                                base,
+                                className;
+
+                            deserFn = this.getAttribute(fromNode, 'deserialize');
+
+                            if (this.isMetaTypeOf(node, this.META.Complex)) {
+                                // Complex objects are expected to define their own
+                                // (static) deserialize factory method
+                                base = this.core.getMetaType(node);
+                                className = this.getAttribute(base, 'name');
+                                deserFn = `return ${className}.deserialize(path)`;
+                            }
+
+                            return {
+                                name: name,
+                                code: deserFn
+                            };
+                        });
+                }));
+            })
+            .then(_tplContents => {
+                tplContents = _tplContents;
+                var hashes = inputs.map(pair => {
+                    var hash = this.getAttribute(pair[2], 'data');
+                    files.inputAssets[pair[0]] = hash;
+                    return {
+                        hash: hash,
+                        name: pair[0]
+                    };
+                });
+
+                return Q.all(hashes.map(pair => 
+                    this.blobClient.getMetadata(pair.hash)
+                        .fail(err => this.onBlobRetrievalFail(node, pair.name, err))));
+            })
+            .then(metadatas => {
+                // Create the deserializer
+                tplContents.forEach((ctnt, i) => {
+                    // Get the name of the given asset
+                    ctnt.filename = metadatas[i].name;
+                    files['inputs/' + ctnt.name + '/init.lua'] = _.template(Templates.DESERIALIZE)(ctnt);
+                });
+                return files;
+            });
+    };
+
+    ExecuteJob.prototype.createOutputs = function (node, files) {
+        // For each of the output types, grab their serialization functions and
+        // create the `outputs/init.lua` file
+        this.logger.info('Creating outputs/init.lua...');
+        return this.getOutputs(node)
+            .then(outputs => {
+                var outputTypes = outputs
+                // Get the serialize functions for each
+                    .map(tuple => {
+                        var node = tuple[2],
+                            serFn = this.getAttribute(node, 'serialize');
+
+                        if (this.isMetaTypeOf(node, this.META.Complex)) {
+                            // Complex objects are expected to define their own
+                            // serialize methods
+                            serFn = 'if data ~= nil then data:serialize(path) end';
+                        }
+
+                        return [tuple[1], serFn];
+                    });
+
+                files['outputs/init.lua'] = _.template(Templates.SERIALIZE)({types: outputTypes});
+            });
+    };
+
+    ExecuteJob.prototype.createMainFile = function (node, files) {
+        this.logger.info('Creating main file...');
+        return this.getInputs(node)
+            .then(inputs => {
+                var name = this.getAttribute(node, 'name'),
+                    code = this.getAttribute(node, 'code'),
+                    pointers = this.core.getPointerNames(node).filter(ptr => ptr !== 'base'),
+                    content = {
+                        name: name
+                    };
+
+                // Get input data arguments
+                content.inputs = inputs
+                    .map(pair => [pair[0], !this.getAttribute(pair[2], 'data')]);  // remove empty inputs
+
+                // Defined variables for each pointers
+                content.pointers = pointers
+                    .map(id => [id, this.core.getPointerPath(node, id) === null]);
+
+                // Add remaining code
+                content.code = code;
+
+                files['main.lua'] = _.template(Templates.MAIN)(content);
+
+                // Set the line offset
+                var lineOffset = this.getLineOffset(files['main.lua'], code);
+                this.setAttribute(node, CONSTANTS.LINE_OFFSET, lineOffset);
+            });
+    };
+
+    ExecuteJob.prototype.getLineOffset = function (main, snippet) {
+        var i = main.indexOf(snippet),
+            lines = main.substring(0, i).match(/\n/g);
+
+        return lines ? lines.length : 0;
+    };
+
+    ExecuteJob.prototype.createAttributeFile = function (node, files) {
+        var skip = ['code', 'stdout', 'execFiles', 'jobId', 'secret'],
+            numOrBool = /^(-?\d+\.?\d*((e|e-)\d+)?|(true|false))$/,
+            table;
+
+        this.logger.info('Creating attributes file...');
+        table = '{\n\t' + this.core.getAttributeNames(node)
+            .filter(attr => skip.indexOf(attr) === -1)
+            .map(name => {
+                var value = this.getAttribute(node, name);
+                if (!numOrBool.test(value)) {
+                    value = `"${value}"`;
+                }
+                return [name, value];
+            })
+            .map(pair => pair.join(' = '))
+            .join(',\n\t') + '\n}';
+
+        files['attributes.lua'] = `-- attributes of ${this.getAttribute(node, 'name')}\nreturn ${table}`;
+    };
+
+    ExecuteJob.prototype.createPointers = function (node, files, cb) {
+        var pointers,
+            nIds;
+
+        this.logger.info('Creating pointers file...');
+        pointers = this.core.getPointerNames(node)
+            .filter(name => name !== 'base')
+            .filter(id => this.core.getPointerPath(node, id) !== null);
+
+        nIds = pointers.map(p => this.core.getPointerPath(node, p));
+        files.ptrAssets = {};
+        Q.all(
+            nIds.map(nId => this.getPtrCodeHash(nId))
+        )
+        .then(resultHashes => {
+            var name = this.getAttribute(node, 'name');
+            this.logger.info(`Pointer generation for ${name} FINISHED!`);
+            resultHashes.forEach((hash, index) => {
+                files.ptrAssets[`pointers/${pointers[index]}/init.lua`] = hash;
+            });
+            return cb(null, files);
+        })
+        .fail(e => {
+            this.logger.error(`Could not generate pointer files for ${this.getAttribute(node, 'name')}: ${e.toString()}`);
+            return cb(e);
+        });
+    };
+
+    return ExecuteJob;
+});
+

src/plugins/ExecuteJob/ExecuteJob.Metadata.js

@@ -0,0 +1,122 @@
+/*globals define*/
+define([
+    'deepforge/Constants'
+], function(
+    CONSTANTS
+) {
+
+    var ExecuteJob = function() {
+    };
+
+    ExecuteJob.prototype[CONSTANTS.GRAPH_CREATE] = function (job, id) {
+        var graph,
+            name = Array.prototype.slice.call(arguments, 2).join(' '),
+            jobId = this.core.getPath(job);
+
+        id = jobId + '/' + id;
+        this.logger.info(`Creating graph ${id} named ${name}`);
+
+        // Check if the graph already exists
+        graph = this._getExistingMetadata(jobId, 'Graph', name);
+        if (!graph) {
+            graph = this.createNode('Graph', job);
+
+            if (name) {
+                this.setAttribute(graph, 'name', name);
+            }
+            this.createIdToMetadataId[graph] = id;
+        }
+
+        this._metadata[id] = graph;
+    };
+
+    ExecuteJob.prototype[CONSTANTS.GRAPH_PLOT] = function (job, id, x, y) {
+        var jobId = this.core.getPath(job),
+            nonNum = /[^\d-\.]*/g,
+            line,
+            points;
+            
+
+        id = jobId + '/' + id;
+        this.logger.info(`Adding point ${x}, ${y} to ${id}`);
+        line = this._metadata[id];
+        if (!line) {
+            this.logger.warn(`Can't add point to non-existent line: ${id}`);
+            return;
+        }
+
+        // Clean the points by removing and special characters
+        x = x.replace(nonNum, '');
+        y = y.replace(nonNum, '');
+        points = this.getAttribute(line, 'points');
+        points += `${x},${y};`;
+        this.setAttribute(line, 'points', points);
+    };
+
+    ExecuteJob.prototype[CONSTANTS.GRAPH_CREATE_LINE] = function (job, graphId, id) {
+        var jobId = this.core.getPath(job),
+            graph = this._metadata[jobId + '/' + graphId],
+            name = Array.prototype.slice.call(arguments, 3).join(' '),
+            line;
+
+        // Create a 'line' node in the given Graph metadata node
+        name = name.replace(/\s+$/, '');
+        line = this.createNode('Line', graph);
+        this.setAttribute(line, 'name', name);
+        this._metadata[jobId + '/' + id] = line;
+        this.createIdToMetadataId[line] = jobId + '/' + id;
+    };
+
+    ExecuteJob.prototype[CONSTANTS.IMAGE.BASIC] =
+    ExecuteJob.prototype[CONSTANTS.IMAGE.UPDATE] =
+    ExecuteJob.prototype[CONSTANTS.IMAGE.CREATE] = function (job, hash, imgId) {
+        var name = Array.prototype.slice.call(arguments, 3).join(' '),
+            imageNode = this._getImageNode(job, imgId, name);
+
+        this.setAttribute(imageNode, 'data', hash);
+    };
+
+    ExecuteJob.prototype[CONSTANTS.IMAGE.NAME] = function (job, imgId) {
+        var name = Array.prototype.slice.call(arguments, 2).join(' '),
+            imageNode = this._getImageNode(job, imgId, name);
+
+        this.setAttribute(imageNode, 'name', name);
+    };
+
+    ExecuteJob.prototype._getImageNode = function (job, imgId, name) {
+        var jobId = this.core.getPath(job),
+            id = jobId + '/IMAGE/' + imgId,
+            imageNode = this._metadata[id];  // Look for the metadata imageNode
+
+        if (!imageNode) {
+
+            // Check if the imageNode already exists
+            imageNode = this._getExistingMetadata(jobId, 'Image', name);
+            if (!imageNode) {
+                this.logger.info(`Creating image ${id} named ${name}`);
+                imageNode = this.createNode('Image', job);
+                this.setAttribute(imageNode, 'name', name);
+                this.createIdToMetadataId[imageNode] = id;
+            }
+            this._metadata[id] = imageNode;
+        }
+        return imageNode;
+    };
+
+    ExecuteJob.prototype._getExistingMetadata = function (jobId, type, name) {
+        var oldMetadata = this._oldMetadataByName[jobId] && this._oldMetadataByName[jobId][type],
+            node,
+            id;
+
+        if (oldMetadata && oldMetadata[name]) {
+            id = oldMetadata[name];
+            node = this._markForDeletion[jobId][id];
+            delete this._markForDeletion[jobId][id];
+            this.createdMetadataIds[jobId].push(id);  // used for resuming jobs
+        }
+
+        return node || null;
+    };
+
+    return ExecuteJob;
+});

src/plugins/ExecuteJob/ExecuteJob.SafeSave.js

@@ -0,0 +1,377 @@
+/*globals define*/
+define([
+    'plugin/PluginBase',
+    'common/storage/constants',
+    'q',
+    'common/util/assert'
+], function(
+    PluginBase,
+    STORAGE_CONSTANTS,
+    Q,
+    assert
+) {
+
+    var CREATE_PREFIX = 'created_node_',
+        INDEX = 1;
+
+    var ExecuteJob = function() {
+        this.forkNameBase = null;
+        this.runningJobHashes = [];
+        this._currentSave = Q();
+    };
+
+    ExecuteJob.prototype.getCreateId = function () {
+        return CREATE_PREFIX + (++INDEX);
+    };
+
+    ExecuteJob.prototype.isCreateId = function (id) {
+        return (typeof id === 'string') && (id.indexOf(CREATE_PREFIX) === 0);
+    };
+
+    ExecuteJob.prototype.createNode = function (baseType, parent) {
+        var id = this.getCreateId(),
+            parentId = this.isCreateId(parent) ? parent : this.core.getPath(parent);
+
+        this.logger.info(`Creating ${id} of type ${baseType} in ${parentId}`);
+        assert(this.META[baseType], `Cannot create node w/ unrecognized type: ${baseType}`);
+        this.creations[id] = {
+            base: baseType,
+            parent: parentId
+        };
+        return id;
+    };
+
+    ExecuteJob.prototype.deleteNode = function (nodeId) {
+        this.deletions.push(nodeId);
+    };
+
+    ExecuteJob.prototype.delAttribute = function (node, attr) {
+        return this.setAttribute(node, attr, null);
+    };
+
+    ExecuteJob.prototype.setAttribute = function (node, attr, value) {
+        var nodeId;
+
+        if (this.isCreateId(node)) {
+            nodeId = node;
+        } else {
+            nodeId = this.core.getPath(node);
+            assert(typeof nodeId === 'string', `Cannot set attribute of ${nodeId}`);
+        }
+
+        if (value !== null) {
+            this.logger.info(`Setting ${attr} of ${nodeId} to ${value}`);
+        } else {
+            this.logger.info(`Deleting ${attr} of ${nodeId}`);
+        }
+
+        if (!this.changes[nodeId]) {
+            this.changes[nodeId] = {};
+        }
+        this.changes[nodeId][attr] = value;
+    };
+
+    ExecuteJob.prototype.getAttribute = function (node, attr) {
+        var nodeId;
+
+        assert(this.deletions.indexOf(nodeId) === -1,
+            `Cannot get ${attr} from deleted node ${nodeId}`);
+
+        // Check if it was newly created
+        if (this.isCreateId(node)) {
+            nodeId = node;
+            assert(this.creations[nodeId], `Creation node not updated: ${nodeId}`);
+            node = this.META[this.creations[nodeId].base];
+        } else {
+            nodeId = this.core.getPath(node);
+        }
+
+        // Check the most recent changes, then the currentChanges, then the model
+        var value = this._getValueFrom(nodeId, attr, node, this.changes) ||
+            this._getValueFrom(nodeId, attr, node, this.currentChanges);
+
+        if (value) {
+            return value;
+        }
+
+        return this.core.getAttribute(node, attr);
+    };
+
+    ExecuteJob.prototype._getValueFrom = function (nodeId, attr, node, changes) {
+        var base;
+        if (changes[nodeId] && changes[nodeId][attr] !== undefined) {
+            // If deleted the attribute, get the default (inherited) value
+            if (changes[nodeId][attr] === null) {
+                base = this.isCreateId(nodeId) ? node : this.core.getBase(node);
+                return this.getAttribute(base, attr);
+            }
+            return changes[nodeId][attr];
+        }
+    };
+
+    ExecuteJob.prototype._applyNodeChanges = function (node, changes) {
+        var attr,
+            value;
+
+        this.logger.info(`About to apply changes for ${this.core.getPath(node)}`);
+        for (var i = changes.length; i--;) {
+            attr = changes[i][0];
+            value = changes[i][1];
+            if (value !== null) {
+                this.logger.info(`Setting ${attr} to ${value} (${this.core.getPath(node)})`);
+                this.core.setAttribute(node, attr, value);
+            } else {
+                this.core.delAttribute(node, attr);
+            }
+        }
+        return node;
+    };
+
+    ExecuteJob.prototype.applyModelChanges = function () {
+        return this.applyCreations()
+            .then(() => this.applyChanges())
+            .then(() => this.applyDeletions());
+    };
+
+    ExecuteJob.prototype.applyChanges = function () {
+        var nodeIds = Object.keys(this.changes),
+            attrs,
+            value,
+            changes,
+            promises = [],
+            changesFor = {},
+            id,
+            promise;
+
+        this.logger.info('Collecting changes to apply in commit');
+
+        for (var i = nodeIds.length; i--;) {
+            changes = [];
+            attrs = Object.keys(this.changes[nodeIds[i]]);
+            for (var a = attrs.length; a--;) {
+                value = this.changes[nodeIds[i]][attrs[a]];
+                changes.push([attrs[a], value]);
+            }
+            changesFor[nodeIds[i]] = changes;
+
+            assert(changes, `changes are invalid for ${nodeIds[i]}: ${changes}`);
+            assert(!this.isCreateId(nodeIds[i]),
+                `Creation id not resolved to actual id: ${nodeIds[i]}`);
+            promise = this.core.loadByPath(this.rootNode, nodeIds[i]);
+            promises.push(promise);
+        }
+
+        this.currentChanges = this.changes;
+        this.changes = {};
+        // Need to differentiate between read/write changes.
+        this.logger.info(`About to apply changes for ${promises.length} nodes`);
+        return Q.all(promises)
+            .then(nodes => {
+                for (var i = nodes.length; i--;) {
+                    id = this.core.getPath(nodes[i]);
+                    assert(nodes[i], `node is ${nodes[i]} (${nodeIds[i]})`);
+                    this._applyNodeChanges(nodes[i], changesFor[id]);
+                }
+
+                // Local model is now up-to-date. No longer need currentChanges
+                this.currentChanges = {};
+            });
+    };
+
+    ExecuteJob.prototype.applyCreations = function () {
+        var nodeIds = Object.keys(this.creations),
+            tiers = this.createCreationTiers(nodeIds),
+            creations = this.creations,
+            newIds = {},
+            promise = Q(),
+            tier;
+
+        this.logger.info('Applying node creations');
+        for (var i = 0; i < tiers.length; i++) {
+            tier = tiers[i];
+            // Chain the promises, loading each tier sequentially
+            promise = promise.then(this.applyCreationTier.bind(this, creations, newIds, tier));
+        }
+
+        this.creations = {};
+        return promise;
+    };
+
+    ExecuteJob.prototype.applyCreationTier = function (creations, newIds, tier) {
+        var promises = [],
+            parentId,
+            node;
+
+        for (var j = tier.length; j--;) {
+            node = creations[tier[j]];
+            assert(node, `Could not find create info for ${tier[j]}`);
+            parentId = newIds[node.parent] || node.parent;
+            promises.push(this.applyCreation(tier[j], node.base, parentId));
+        }
+        return Q.all(promises).then(nodes => {
+            // Record the newIds so they can be used to resolve creation ids
+            // in subsequent tiers
+            for (var i = tier.length; i--;) {
+                newIds[tier[i]] = this.core.getPath(nodes[i]);
+            }
+        });
+    };
+
+    // Figure out the dependencies between nodes to create.
+    // eg, if newId1 is to be created in newId2, then newId2 will
+    // be in an earlier tier than newId1. Essentially a topo-sort
+    // on a tree structure
+    ExecuteJob.prototype.createCreationTiers = function (nodeIds) {
+        var tiers = [],
+            prevTier = {},
+            tier = {},
+            id,
+            prevLen,
+            i;
+
+        // Create first tier (created inside existing nodes)
+        for (i = nodeIds.length; i--;) {
+            id = nodeIds[i];
+            if (!this.isCreateId(this.creations[id].parent)) {
+                tier[id] = true;
+                nodeIds.splice(i, 1);
+            }
+        }
+        prevTier = tier;
+        tiers.push(Object.keys(tier));
+
+        // Now, each tier consists of the nodes to be created inside a
+        // node from the previous tier
+        while (nodeIds.length) {
+            prevLen = nodeIds.length;
+            tier = {};
+            for (i = nodeIds.length; i--;) {
+                id = nodeIds[i];
+                if (prevTier[this.creations[id].parent]) {
+                    tier[id] = true;
+                    nodeIds.splice(i, 1);
+                }
+            }
+            prevTier = tier;
+            tiers.push(Object.keys(tier));
+            // Every iteration should find at least one node
+            assert(prevLen > nodeIds.length,
+                `Created empty create tier! Remaining: ${nodeIds.join(', ')}`);
+        }
+
+        return tiers;
+    };
+
+    ExecuteJob.prototype.applyCreation = function (tmpId, baseType, parentId) {
+        var base = this.META[baseType],
+            nodeId,
+            id;
+
+        this.logger.info(`Applying creation of ${tmpId} (${baseType}) in ${parentId}`);
+
+        assert(!this.isCreateId(parentId),
+            `Did not resolve parent id: ${parentId} for ${tmpId}`);
+        assert(base, `Invalid base type: ${baseType}`);
+        return this.core.loadByPath(this.rootNode, parentId)
+            .then(parent => this.core.createNode({base, parent}))
+            .then(node => {  // Update the _metadata records
+                id = this.createIdToMetadataId[tmpId];
+                delete this.createIdToMetadataId[tmpId];
+                this._metadata[id] = node;
+
+                // Update creations
+                nodeId = this.core.getPath(node);
+                if (this.changes[tmpId]) {
+                    assert(!this.changes[nodeId],
+                        `Newly created node cannot already have changes! (${nodeId})`);
+                    this.changes[nodeId] = this.changes[tmpId];
+                    delete this.changes[tmpId];
+                }
+                return node;
+            });
+    };
+
+    ExecuteJob.prototype.applyDeletions = function () {
+        var deletions = this.deletions;
+
+        this.deletions = [];
+        return Q.all(deletions.map(id => this.core.loadByPath(this.rootNode, id)))
+            .then(nodes => {
+                for (var i = nodes.length; i--;) {
+                    this.core.deleteNode(nodes[i]);
+                }
+            });
+    };
+
+    // Override 'save' to notify the user on fork
+    ExecuteJob.prototype.save = function (msg) {
+        this._currentSave = this._currentSave
+            .then(() => this.updateForkName(this.forkNameBase))
+            .then(() => this.applyModelChanges())
+            .then(() => PluginBase.prototype.save.call(this, msg))
+            .then(result => {
+                this.logger.info(`Save finished w/ status: ${result.status}`);
+                if (result.status === STORAGE_CONSTANTS.FORKED) {
+                    return this.onSaveForked(result.forkName);
+                } else if (result.status === STORAGE_CONSTANTS.MERGED) {
+                    this.logger.debug('Merged changes. About to update plugin nodes');
+                    return this.updateNodes();
+                }
+            });
+
+        return this._currentSave;
+    };
+
+    ExecuteJob.prototype.onSaveForked = function (forkName) {
+        var name = this.getAttribute(this.activeNode, 'name'),
+            msg = `"${name}" execution has forked to "${forkName}"`;
+        this.currentForkName = forkName;
+
+        this.logManager.fork(forkName);
+        this.runningJobHashes.forEach(jobId => this.originManager.fork(jobId, forkName));
+        this.sendNotification(msg);
+    };
+
+    ExecuteJob.prototype.updateNodes = function (hash) {
+        var activeId = this.core.getPath(this.activeNode);
+
+        hash = hash || this.currentHash;
+        return Q.ninvoke(this.project, 'loadObject', hash)
+            .then(commitObject => {
+                return this.core.loadRoot(commitObject.root);
+            })
+            .then(rootObject => {
+                this.rootNode = rootObject;
+                return this.core.loadByPath(rootObject,activeId);
+            })
+            .then(activeObject => this.activeNode = activeObject)
+            .then(() => {
+                var metaNames = Object.keys(this.META);
+
+                return Q.all(metaNames.map(name => this.updateMetaNode(name)));
+            })
+            .then(() => {
+                var mdNodes,
+                    mdIds;
+
+                mdIds = Object.keys(this._metadata)
+                    .filter(id => !this.isCreateId(this._metadata[id]));
+
+                mdNodes = mdIds.map(id => this.core.getPath(this._metadata[id]))
+                    .map(nodeId => this.core.loadByPath(this.rootNode, nodeId));
+
+                return Q.all(mdNodes).then(nodes => {
+                    for (var i = nodes.length; i--;) {
+                        this._metadata[mdIds[i]] = nodes[i];
+                    }
+                });
+            });
+    };
+
+    ExecuteJob.prototype.updateMetaNode = function (name) {
+        var id = this.core.getPath(this.META[name]);
+        return this.core.loadByPath(this.rootNode, id).then(node => this.META[name] = node);
+    };
+
+    return ExecuteJob;
+});

src/plugins/ExecuteJob/ExecuteJob.js

@@ -0,0 +1,979 @@
+/*globals define*/
+/*jshint node:true, browser:true*/
+
+define([
+    'common/util/assert',
+    'text!./metadata.json',
+    'executor/ExecutorClient',
+    'plugin/PluginBase',
+    'deepforge/plugin/LocalExecutor',
+    'deepforge/plugin/PtrCodeGen',
+    'deepforge/api/JobLogsClient',
+    'deepforge/api/JobOriginClient',
+    'deepforge/api/ExecPulseClient',
+    './ExecuteJob.Files',
+    './ExecuteJob.Metadata',
+    './ExecuteJob.SafeSave',
+    'deepforge/Constants',
+    'deepforge/utils',
+    'q',
+    'superagent',
+    'underscore'
+], function (
+    assert,
+    pluginMetadata,
+    ExecutorClient,
+    PluginBase,
+    LocalExecutor,  // DeepForge operation primitives
+    PtrCodeGen,
+    JobLogsClient,
+    JobOriginClient,
+    ExecPulseClient,
+
+    ExecuteJobFiles,
+    ExecuteJobMetadata,
+    ExecuteJobSafeSave,
+
+    CONSTANTS,
+    utils,
+    Q,
+    superagent,
+    _
+) {
+    'use strict';
+
+    pluginMetadata = JSON.parse(pluginMetadata);
+
+    var OUTPUT_INTERVAL = 1500,
+        STDOUT_FILE = 'job_stdout.txt';
+
+    /**
+     * Initializes a new instance of ExecuteJob.
+     * @class
+     * @augments {PluginBase}
+     * @classdesc This class represents the plugin ExecuteJob.
+     * @constructor
+     */
+    var ExecuteJob = function () {
+        // Call base class' constructor.
+        PluginBase.call(this);
+        ExecuteJobSafeSave.call(this);
+        this.pluginMetadata = pluginMetadata;
+        this._metadata = {};
+        this._beating = null;
+
+        // Metadata updating
+        this._markForDeletion = {};  // id -> node
+        this._oldMetadataByName = {};  // name -> id
+        this.lastAppliedCmd = {};
+        this.createdMetadataIds = {};
+        this.canceled = false;
+
+        this.changes = {};
+        this.currentChanges = {};  // read-only changes being applied
+        this.creations = {};
+        this.deletions = [];
+        this.createIdToMetadataId = {};
+        this.logManager = null;
+    };
+
+    /**
+     * Metadata associated with the plugin. Contains id, name, version, description, icon, configStructue etc.
+     * This is also available at the instance at this.pluginMetadata.
+     * @type {object}
+     */
+    ExecuteJob.metadata = pluginMetadata;
+    ExecuteJob.UPDATE_INTERVAL = 1500;
+    ExecuteJob.HEARTBEAT_INTERVAL = 2500;
+
+    // Prototypical inheritance from PluginBase.
+    ExecuteJob.prototype = Object.create(PluginBase.prototype);
+    ExecuteJob.prototype.constructor = ExecuteJob;
+
+    ExecuteJob.prototype.configure = function () {
+        var result = PluginBase.prototype.configure.apply(this, arguments),
+            params = {
+                logger: this.logger,
+                port: this.gmeConfig.server.port,
+                branchName: this.branchName,
+                projectId: this.projectId
+            },
+            isHttps = typeof window === 'undefined' ? false :
+                window.location.protocol !== 'http:';
+
+        this.logManager = new JobLogsClient(params);
+        this.originManager = new JobOriginClient(params);
+        this.pulseClient = new ExecPulseClient(params);
+
+        this.executor = new ExecutorClient({
+            logger: this.logger,
+            serverPort: this.gmeConfig.server.port,
+            httpsecure: isHttps
+        });
+        return result;
+    };
+
+    /**
+     * Main function for the plugin to execute. This will perform the execution.
+     * Notes:
+     * - Always log with the provided logger.[error,warning,info,debug].
+     * - Do NOT put any user interaction logic UI, etc. inside this method.
+     * - callback always has to be called even if error happened.
+     *
+     * @param {function(string, plugin.PluginResult)} callback - the result callback
+     */
+    ExecuteJob.prototype.main = function (callback) {
+        // Check the activeNode to make sure it is a valid node
+        var type = this.core.getMetaType(this.activeNode),
+            typeName = type && this.getAttribute(type, 'name'),
+            execNode,
+            status;
+
+        if (typeName !== 'Job') {
+            return callback(`Cannot execute ${typeName} (expected Job)`, this.result);
+        }
+
+        // Set the parent execution to 'running'
+        execNode = this.core.getParent(this.activeNode);
+        status = this.getAttribute(execNode, 'status');
+        if (status !== 'running') {
+            this.setAttribute(execNode, 'status', 'running');
+        }
+
+        this._callback = callback;
+        this.currentForkName = null;
+        this.forkNameBase = this.getAttribute(this.activeNode, 'name');
+        this.isResuming(this.activeNode)
+            .then(resuming => {
+                this._resumed = resuming;
+                return this.prepare(resuming);
+            })
+            .then(() => {
+                if (this._resumed) {
+                    this.currentRunId = this.getAttribute(this.activeNode, 'jobId');
+                    this.startExecHeartBeat();
+                    if (this.canResumeJob(this.activeNode)) {
+                        return this.resumeJob(this.activeNode);
+                    } else {
+                        var name = this.getAttribute(this.activeNode, 'name'),
+                            id = this.core.getPath(this.activeNode),
+                            msg = `Cannot resume ${name} (${id}). Missing jobId.`;
+
+                        this.logger.error(msg);
+                        return callback(msg);
+                    }
+                } else {
+                    this.currentRunId = null;  // will be set after exec files created
+                    return this.executeJob(this.activeNode);
+                }
+            })
+            .catch(err => this._callback(err));
+    };
+
+    ExecuteJob.prototype.isResuming = function (job) {
+        job = job || this.activeNode;
+        var deferred = Q.defer(),
+            status = this.getAttribute(job, 'status'),
+            jobId;
+
+        if (status === 'running') {
+            jobId = this.getAttribute(job, 'jobId');
+            // Check if on the origin branch
+            this.originManager.getOrigin(jobId)
+                .then(origin => {
+                    if (this.branchName === origin.branch) {
+                        // Check if plugin is no longer running
+                        return this.pulseClient.check(jobId)
+                            .then(alive => {
+                                deferred.resolve(alive !== CONSTANTS.PULSE.ALIVE);
+                            });
+                    } else {
+                        deferred.resolve(false);
+                    }
+                });
+        } else {
+            deferred.resolve(false);
+        }
+        return deferred.promise;
+    };
+
+    ExecuteJob.prototype.canResumeJob = function (job) {
+        return !!this.getAttribute(job, 'jobId');
+    };
+
+    ExecuteJob.prototype.resumeJob = function (job) {
+        var hash = this.getAttribute(job, 'jobId'),
+            name = this.getAttribute(job, 'name'),
+            id = this.core.getPath(job),
+            msg;
+
+        this.logger.info(`Resuming job ${name} (${id})`);
+
+        return this.logManager.getMetadata(id)
+            .then(metadata => {
+                var count = metadata.lineCount;
+
+                if (count === -1) {
+                    this.logger.warn(`No line count found for ${id}. Setting count to 0`);
+                    count = 0;
+                    return this.logManager.deleteLog(id)
+                        .then(() => count);
+                }
+                return count;
+            })
+            .then(count => {  // update line count (to inform logClient appendTo)
+                this.outputLineCount[id] = count;
+                return this.executor.getOutput(hash, 0, count);
+            })
+            .then(output => {  // parse the stdout to update the job metadata
+                var stdout = output.map(o => o.output).join(''),
+                    result = this.processStdout(job, stdout),
+                    name = this.getAttribute(job, 'name'),
+                    promise = Q();
+
+
+                if (result.hasMetadata) {
+                    msg = `Updated graph/image output for ${name}`;
+                    promise = this.save(msg);
+                }
+                return promise.then(() => this.getOperation(job));
+            })
+            .then(opNode => this.watchOperation(hash, opNode, job));
+    };
+
+    ExecuteJob.prototype.updateForkName = function (basename) {
+        basename = basename + '_fork';
+        basename = basename.replace(/[- ]/g, '_');
+        return this.project.getBranches().then(branches => {
+            var names = Object.keys(branches),
+                name = basename,
+                i = 2;
+
+            while (names.indexOf(name) !== -1) {
+                name = basename + '_' + i;
+                i++;
+            }
+
+            this.forkName = name;
+        });
+    };
+
+    ExecuteJob.prototype.getConnections = function (nodes) {
+        var conns = [];
+        for (var i = nodes.length; i--;) {
+            if (this.core.getPointerPath(nodes[i], 'src') &&
+                this.core.getPointerPath(nodes[i], 'dst')) {
+
+                conns.push(nodes[i]);
+            }
+        }
+        return conns;
+    };
+
+    ExecuteJob.prototype.prepare = function (isResuming) {
+        var dstPortId,
+            srcPortId,
+            conns,
+            executionNode = this.core.getParent(this.activeNode);
+
+        this.pipelineName = this.getAttribute(executionNode, 'name');
+        return this.core.loadSubTree(executionNode)
+            .then(nodes => {
+                this.inputPortsFor = {};
+                this.outputLineCount = {};
+
+                conns = this.getConnections(nodes);
+
+                // Create inputPortsFor for the given input ports
+                for (var i = conns.length; i--;) {
+                    dstPortId = this.core.getPointerPath(conns[i], 'dst');
+                    srcPortId = this.core.getPointerPath(conns[i], 'src');
+
+                    if (!this.inputPortsFor[dstPortId]) {
+                        this.inputPortsFor[dstPortId] = [srcPortId];
+                    } else {
+                        this.inputPortsFor[dstPortId].push(srcPortId);
+                    }
+                }
+            })
+            .then(() => this.recordOldMetadata(this.activeNode, isResuming));
+    };
+
+    ExecuteJob.prototype.recordOldMetadata = function (job) {
+        var nodeId = this.core.getPath(job),
+            name,
+            id,
+            idsToDelete = [],
+            type,
+            base,
+            child,
+            i;
+
+        // If we are resuming the pipeline, we will not be deleting any metadata
+        this.lastAppliedCmd[nodeId] = 0;
+        this.createdMetadataIds[nodeId] = [];
+        this._oldMetadataByName[nodeId] = {};
+        this._markForDeletion[nodeId] = {};
+        return this.core.loadChildren(job)
+            .then(jobChildren => {
+                // Remove any metadata nodes
+                for (i = jobChildren.length; i--;) {
+                    child = jobChildren[i];
+                    if (this.isMetaTypeOf(child, this.META.Metadata)) {
+                        id = this.core.getPath(child);
+                        name = this.getAttribute(child, 'name');
+                        base = this.core.getBase(child);
+                        type = this.getAttribute(base, 'name');
+
+                        this._markForDeletion[nodeId][id] = child;
+                        // namespace by metadata type
+                        if (!this._oldMetadataByName[nodeId][type]) {
+                            this._oldMetadataByName[nodeId][type] = {};
+                        }
+
+                        this._oldMetadataByName[nodeId][type][name] = id;
+
+                        // children of metadata nodes get deleted
+                        idsToDelete = idsToDelete
+                            .concat(this.core.getChildrenPaths(child));
+                    }
+                }
+
+                // make the deletion ids relative to the job node
+                this.logger.debug(`About to delete ${idsToDelete.length}: ${idsToDelete.join(', ')}`);
+                for (i = idsToDelete.length; i--;) {
+                    this.deleteNode(idsToDelete[i]);
+                }
+            });
+    };
+
+    ExecuteJob.prototype.clearOldMetadata = function (job) {
+        var nodeId = this.core.getPath(job),
+            nodeIds,
+            node;
+
+        // Remove created nodes left over from resumed job
+        this.createdMetadataIds[nodeId].forEach(id => delete this._markForDeletion[nodeId][id]);
+        nodeIds = Object.keys(this._markForDeletion[nodeId]);
+        this.logger.debug(`About to delete ${nodeIds.length}: ${nodeIds.join(', ')}`);
+        for (var i = nodeIds.length; i--;) {
+            node = this._markForDeletion[nodeId][nodeIds[i]];
+            this.deleteNode(this.core.getPath(node));
+        }
+        delete this.lastAppliedCmd[nodeId];
+        delete this.createdMetadataIds[nodeId];
+        delete this._markForDeletion[nodeId];
+
+        this.delAttribute(job, 'jobId');
+        this.delAttribute(job, 'secret');
+    };
+
+    ExecuteJob.prototype.resultMsg = function(msg) {
+        this.sendNotification(msg);
+        this.createMessage(null, msg);
+    };
+
+    ExecuteJob.prototype.onOperationCanceled = function(op) {
+        var job = this.core.getParent(op),
+            name = this.getAttribute(op, 'name'),
+            msg = `"${name}" canceled!`;
+
+        this.setAttribute(job, 'status', 'canceled');
+        this.resultMsg(msg);
+        this.onComplete(op, null);
+    };
+
+    ExecuteJob.prototype.onOperationFail =
+    ExecuteJob.prototype.onOperationComplete =
+    ExecuteJob.prototype.onComplete = function (opNode, err) {
+        var job = this.core.getParent(opNode),
+            exec = this.core.getParent(job),
+            name = this.getAttribute(job, 'name'),
+            jobId = this.core.getPath(job),
+            status = err ? 'fail' : (this.canceled ? 'canceled' : 'success'),
+            msg = err ? `${name} execution failed!` :
+                `${name} executed successfully!`,
+            promise = Q();
+
+        this.setAttribute(job, 'status', status);
+        this.logger.info(`Setting ${name} (${jobId}) status to ${status}`);
+        this.clearOldMetadata(job);
+
+        if (this.currentForkName) {
+            // notify client that the job has completed
+            this.sendNotification(`"${name}" execution completed on branch "${this.currentForkName}"`);
+        }
+        if (err) {
+            this.logger.warn(`${name} failed: ${err}`);
+            this.setAttribute(exec, 'status', 'failed');
+        } else if (this.canceled) {
+            // Should I set this to 'canceled'?
+            this.setAttribute(exec, 'status', 'canceled');
+        } else {
+            // Check if all the other jobs are successful. If so, set the
+            // execution status to 'success'
+            promise = this.core.loadChildren(exec)
+                .then(nodes => {
+                    var execSuccess = true,
+                        type,
+                        typeName;
+
+                    for (var i = nodes.length; i--;) {
+                        type = this.core.getMetaType(nodes[i]);
+                        typeName = this.getAttribute(type, 'name');
+
+                        if (typeName === 'Job' &&
+                            this.getAttribute(nodes[i], 'status') !== 'success') {
+                            execSuccess = false;
+                        }
+                    }
+
+                    if (execSuccess) {
+                        this.setAttribute(exec, 'status', 'success');
+                    }
+                });
+        }
+
+        this.createMessage(null, msg);
+        promise
+            .then(() => this.save(msg))
+            .then(() => {
+                this.result.setSuccess(!err);
+                this.stopExecHeartBeat();
+                this._callback(err, this.result);
+            })
+            .catch(err => {
+                // Result success is false at invocation.
+                this._callback(err, this.result);
+            });
+    };
+
+    ExecuteJob.prototype.getOperation = function (job) {
+        return this.core.loadChildren(job).then(children =>
+            children.find(child => this.isMetaTypeOf(child, this.META.Operation)));
+    };
+
+    ExecuteJob.prototype.onBlobRetrievalFail = function (node, input, err) {
+        var job = this.core.getParent(node),
+            e = `Failed to retrieve "${input}" (${err})`,
+            consoleErr = `Failed to execute operation: ${e}`;
+
+        consoleErr += [
+            '\n\nA couple things to check out:\n',
+            '- Has the location of DeepForge\'s blob changed?',
+            '    (Configurable using "blob.dir" in the deepforge config' +
+            ' or setting the DEEPFORGE_BLOB_DIR environment variable)\n',
+
+            '- Was this project created using a different blob location?'
+        ].join('\n    ');
+
+        this.setAttribute(job, 'stdout', consoleErr);
+        this.onOperationFail(node, `Blob retrieval failed for "${name}": ${e}`);
+    };
+
+    ExecuteJob.prototype.executeJob = function (job) {
+        return this.getOperation(job).then(node => {
+            var jobId = this.core.getPath(job),
+                name = this.getAttribute(node, 'name'),
+                localTypeId = this.getLocalOperationType(node),
+                artifact,
+                artifactName,
+                files,
+                data = {},
+                inputs;
+
+            // Execute any special operation types here - not on an executor
+            this.logger.debug(`Executing operation "${name}"`);
+            if (localTypeId !== null) {
+                return this.executeLocalOperation(localTypeId, node);
+            } else {
+                // Generate all execution files
+                return this.createOperationFiles(node).then(results => {
+                    this.logger.info('Created operation files!');
+                    files = results;
+                    artifactName = `${name}_${jobId.replace(/\//g, '_')}-execution-files`;
+                    artifact = this.blobClient.createArtifact(artifactName);
+
+                    // Add the input assets
+                    //   - get the metadata (name)
+                    //   - add the given inputs
+                    inputs = Object.keys(files.inputAssets);
+
+                    return Q.all(
+                        inputs.map(input => {  // Get the metadata for each input
+                            var hash = files.inputAssets[input];
+
+                            // data asset for "input"
+                            return this.blobClient.getMetadata(hash)
+                                .fail(err => this.onBlobRetrievalFail(job, input, err));
+                        })
+                    );
+                })
+                .then(mds => {
+                    // Record the large files
+                    var inputData = {},
+                        runsh = '# Bash script to download data files and run job\n' +
+                        'if [ -z "$DEEPFORGE_URL" ]; then\n  echo "Please set DEEPFORGE_URL and' +
+                        ' re-run:"\n  echo ""  \n  echo "  DEEPFORGE_URL=http://my.' +
+                        'deepforge.server.com:8080 bash run.sh"\n  echo ""\n exit 1\nfi\n';
+
+                    mds.forEach((metadata, i) => {
+                        // add the hashes for each input
+                        var input = inputs[i], 
+                            hash = files.inputAssets[input],
+                            dataPath = 'inputs/' + input + '/data',
+                            url = this.blobClient.getRelativeDownloadURL(hash);
+
+                        inputData[dataPath] = {
+                            req: hash,
+                            cache: metadata.content
+                        };
+
+                        // Add to the run.sh file
+                        runsh += `wget $DEEPFORGE_URL${url} -O ${dataPath}\n`;
+                    });
+
+                    delete files.inputAssets;
+                    files['input-data.json'] = JSON.stringify(inputData, null, 2);
+                    runsh += 'th init.lua';
+                    files['run.sh'] = runsh;
+
+                    // Add pointer assets
+                    Object.keys(files.ptrAssets)
+                        .forEach(path => data[path] = files.ptrAssets[path]);
+
+                    // Add the executor config
+                    return this.getOutputs(node);
+                })
+                .then(outputArgs => {
+                    var config,
+                        outputs,
+                        fileList,
+                        ptrFiles = Object.keys(files.ptrAssets),
+                        file;
+
+                    delete files.ptrAssets;
+                    fileList = Object.keys(files).concat(ptrFiles);
+
+                    outputs = outputArgs.map(pair => pair[0])
+                        .map(name => {
+                            return {
+                                name: name,
+                                resultPatterns: [`outputs/${name}`]
+                            };
+                        });
+
+                    outputs.push(
+                        {
+                            name: 'stdout',
+                            resultPatterns: [STDOUT_FILE]
+                        },
+                        {
+                            name: name + '-all-files',
+                            resultPatterns: fileList
+                        }
+                    );
+
+                    config = {
+                        cmd: 'node',
+                        args: ['start.js'],
+                        outputInterval: OUTPUT_INTERVAL,
+                        resultArtifacts: outputs
+                    };
+                    files['executor_config.json'] = JSON.stringify(config, null, 4);
+
+                    // Save the artifact
+                    // Remove empty hashes
+                    for (file in data) {
+                        if (!data[file]) {
+                            this.logger.warn(`Empty data hash has been found for file "${file}". Removing it...`);
+                            delete data[file];
+                        }
+                    }
+                    return artifact.addObjectHashes(data);
+                })
+                .then(() => {
+                    this.logger.info(`Added ptr/input data hashes for "${artifactName}"`);
+                    return artifact.addFiles(files);
+                })
+                .then(() => {
+                    this.logger.info(`Added execution files for "${artifactName}"`);
+                    return artifact.save();
+                })
+                .then(hash => {
+                    this.logger.info(`Saved execution files "${artifactName}"`);
+                    this.result.addArtifact(hash);  // Probably only need this for debugging...
+                    this.executeDistOperation(job, node, hash);
+                })
+                .fail(e => {
+                    this.onOperationFail(node, `Distributed operation "${name}" failed ${e}`);
+                });
+            }
+        });
+    };
+
+    ExecuteJob.prototype.executeDistOperation = function (job, opNode, hash) {
+        var name = this.getAttribute(opNode, 'name'),
+            jobId = this.core.getPath(job);
+
+        this.logger.info(`Executing operation "${name}"`);
+
+        this.outputLineCount[jobId] = 0;
+        // Set the job status to 'running'
+        this.setAttribute(job, 'status', 'queued');
+        this.delAttribute(job, 'stdout');
+        this.logManager.deleteLog(jobId);
+        this.logger.info(`Setting ${jobId} status to "queued" (${this.currentHash})`);
+        this.logger.debug(`Making a commit from ${this.currentHash}`);
+        this.save(`Queued "${name}" operation in ${this.pipelineName}`)
+            .then(() => this.executor.createJob({hash}))
+            .then(info => {
+                this.setAttribute(job, 'jobId', info.hash);
+                if (info.secret) {  // o.w. it is a cached job!
+                    this.setAttribute(job, 'secret', info.secret);
+                }
+                if (!this.currentRunId) {
+                    this.currentRunId = info.hash;
+                    if (this._beating === null) {
+                        this.startExecHeartBeat();
+                    }
+                }
+                return this.recordJobOrigin(hash, job);
+            })
+            .then(() => this.watchOperation(hash, opNode, job))
+            .catch(err => this.logger.error(`Could not execute "${name}": ${err}`));
+
+    };
+
+    ExecuteJob.prototype.recordJobOrigin = function (hash, job) {
+        var execNode = this.core.getParent(job),
+            info;
+
+        info = {
+            hash: hash,
+            nodeId: this.core.getPath(job),
+            job: this.getAttribute(job, 'name'),
+            execution: this.getAttribute(execNode, 'name')
+        };
+        this.runningJobHashes.push(hash);
+        return this.originManager.record(hash, info);
+    };
+
+
+    ExecuteJob.prototype.notifyStdoutUpdate = function (nodeId) {
+        this.sendNotification({
+            message: `${CONSTANTS.STDOUT_UPDATE}/${nodeId}`,
+            toBranch: true
+        });
+    };
+
+    ExecuteJob.prototype.isExecutionCanceled = function () {
+        var execNode = this.core.getParent(this.activeNode);
+        return this.getAttribute(execNode, 'status') === 'canceled';
+    };
+
+    ExecuteJob.prototype.startExecHeartBeat = function () {
+        this._beating = true;
+        this.updateExecHeartBeat();
+    };
+
+    ExecuteJob.prototype.stopExecHeartBeat = function () {
+        this._beating = false;
+    };
+
+    ExecuteJob.prototype.updateExecHeartBeat = function () {
+        var time = Date.now(),
+            next = () => {
+                if (this._beating) {
+                    setTimeout(this.updateExecHeartBeat.bind(this),
+                        ExecuteJob.HEARTBEAT_INTERVAL - (Date.now() - time));
+                }
+            };
+
+        this.pulseClient.update(this.currentRunId)
+            .then(() => next())
+            .catch(err => {
+                if (err) {
+                    this.logger.error(`heartbeat failed: ${err}`);
+                    next();
+                }
+            });
+    };
+
+    ExecuteJob.prototype.watchOperation = function (hash, op, job) {
+        var jobId = this.core.getPath(job),
+            opId = this.core.getPath(op),
+            info,
+            secret,
+            name = this.getAttribute(job, 'name');
+
+        // If canceled, stop the operation
+        if (this.canceled || this.isExecutionCanceled()) {
+            secret = this.getAttribute(job, 'secret');
+            if (secret) {
+                this.executor.cancelJob(hash, secret);
+                this.delAttribute(job, 'secret');
+                this.canceled = true;
+                return this.onOperationCanceled(op);
+            }
+        }
+
+        return this.executor.getInfo(hash)
+            .then(_info => {  // Update the job's stdout
+                var actualLine,  // on executing job
+                    currentLine = this.outputLineCount[jobId],
+                    prep = Q();
+
+                info = _info;
+                actualLine = info.outputNumber;
+                if (actualLine !== null && actualLine >= currentLine) {
+                    this.outputLineCount[jobId] = actualLine + 1;
+                    return prep
+                        .then(() => this.executor.getOutput(hash, currentLine, actualLine+1))
+                        .then(outputLines => {
+                            var stdout = this.getAttribute(job, 'stdout'),
+                                output = outputLines.map(o => o.output).join(''),
+                                last = stdout.lastIndexOf('\n'),
+                                result,
+                                lastLine,
+                                next = Q(),
+                                msg;
+
+                            // parse deepforge commands
+                            if (last !== -1) {
+                                stdout = stdout.substring(0, last+1);
+                                lastLine = stdout.substring(last+1);
+                                output = lastLine + output;
+                            }
+                            result = this.processStdout(job, output, true);
+                            output = result.stdout;
+
+                            if (output) {
+                                // Send notification to all clients watching the branch
+                                var metadata = {
+                                    lineCount: this.outputLineCount[jobId]
+                                };
+                                next = next
+                                    .then(() => this.logManager.appendTo(jobId, output, metadata))
+                                    .then(() => this.notifyStdoutUpdate(jobId));
+                            }
+                            if (result.hasMetadata) {
+                                msg = `Updated graph/image output for ${name}`;
+                                next = next.then(() => this.save(msg));
+                            }
+                            return next;
+                        });
+                }
+            })
+            .then(() => {
+                if (info.status === 'CREATED' || info.status === 'RUNNING') {
+                    var time = Date.now(),
+                        next = Q();
+
+                    if (info.status === 'RUNNING' &&
+                        this.getAttribute(job, 'status') !== 'running') {
+
+                        this.setAttribute(job, 'status', 'running');
+                        next = this.save(`Started "${name}" operation in ${this.pipelineName}`);
+                    }
+
+                    return next.then(() => {
+                        var delta = Date.now() - time;
+                            
+                        if (delta > ExecuteJob.UPDATE_INTERVAL) {
+                            return this.watchOperation(hash, op, job);
+                        }
+
+                        setTimeout(
+                            this.watchOperation.bind(this, hash, op, job),
+                            ExecuteJob.UPDATE_INTERVAL - delta
+                        );
+                    });
+                }
+
+                // Record that the job hash is no longer running
+                this.logger.info(`Job "${name}" has finished (${info.status})`);
+                var i = this.runningJobHashes.indexOf(hash);
+                if (i !== -1) {
+                    this.runningJobHashes.splice(i, 1);
+                } else {
+                    this.logger.warn(`Could not find running job hash ${hash}`);
+                }
+
+                if (info.status === 'CANCELED') {
+                    // If it was cancelled, the pipeline has been stopped
+                    this.logger.debug(`"${name}" has been CANCELED!`);
+                    this.canceled = true;
+                    return this.logManager.getLog(jobId)
+                        .then(stdout => {
+                            this.setAttribute(job, 'stdout', stdout);
+                            return this.onOperationCanceled(op);
+                        });
+                }
+
+                if (info.status === 'SUCCESS' || info.status === 'FAILED_TO_EXECUTE') {
+                    this.setAttribute(job, 'execFiles', info.resultHashes[name + '-all-files']);
+                    return this.blobClient.getArtifact(info.resultHashes.stdout)
+                        .then(artifact => {
+                            var stdoutHash = artifact.descriptor.content[STDOUT_FILE].content;
+                            return this.blobClient.getObjectAsString(stdoutHash);
+                        })
+                        .then(stdout => {
+                            // Parse the remaining code
+                            var result = this.processStdout(job, stdout);
+                            this.setAttribute(job, 'stdout', result.stdout);
+                            this.logManager.deleteLog(jobId);
+                            if (info.status !== 'SUCCESS') {
+                                // Download all files
+                                this.result.addArtifact(info.resultHashes[name + '-all-files']);
+                                // Set the job to failed! Store the error
+                                this.onOperationFail(op, `Operation "${opId}" failed! ${JSON.stringify(info)}`); 
+                            } else {
+                                this.onDistOperationComplete(op, info);
+                            }
+                        });
+                } else {  // something bad happened...
+                    var err = `Failed to execute operation "${opId}": ${info.status}`,
+                        consoleErr = `Failed to execute operation: ${info.status}`;
+                    this.setAttribute(job, 'stdout', consoleErr);
+                    this.logger.error(err);
+                    this.onOperationFail(op, err);
+                }
+            })
+            .catch(err => this.logger.error(`Could not get op info for ${opId}: ${err}`));
+    };
+
+    ExecuteJob.prototype.onDistOperationComplete = function (node, result) {
+        var nodeId = this.core.getPath(node),
+            outputMap = {},
+            outputs;
+
+        // Match the output names to the actual nodes
+        // Create an array of [name, node]
+        // For now, just match by type. Later we may use ports for input/outputs
+        // Store the results in the outgoing ports
+        return this.getOutputs(node)
+            .then(outputPorts => {
+                outputs = outputPorts.map(tuple => [tuple[0], tuple[2]]);
+                outputs.forEach(output => outputMap[output[0]] = output[1]);
+
+                // this should not be in directories -> flatten the data!
+                return Q.all(outputs.map(tuple =>  // [ name, node ]
+                    this.blobClient.getArtifact(result.resultHashes[tuple[0]])
+                ));
+            })
+            .then(artifacts => {
+                this.logger.info(`preparing outputs -> retrieved ${artifacts.length} objects`);
+                // Create new metadata for each
+                artifacts.forEach((artifact, i) => {
+                    var name = outputs[i][0],
+                        outputData = artifact.descriptor.content[`outputs/${name}`],
+                        hash = outputData && outputData.content;
+
+                    if (hash) {
+                        this.setAttribute(outputMap[name], 'data', hash);
+                        this.logger.info(`Setting ${nodeId} data to ${hash}`);
+                    }
+                });
+
+                return this.onOperationComplete(node);
+            })
+            .fail(e => this.onOperationFail(node, `Operation ${nodeId} failed: ${e}`));
+    };
+
+    ExecuteJob.prototype.getOutputs = function (node) {
+        return this.getOperationData(node, this.META.Outputs);
+    };
+
+    ExecuteJob.prototype.getInputs = function (node) {
+        return this.getOperationData(node, this.META.Inputs);
+    };
+
+    ExecuteJob.prototype.getOperationData = function (node, metaType) {
+        // Load the children and the output's children
+        return this.core.loadChildren(node)
+            .then(containers => {
+                var outputs = containers.find(c => this.core.isTypeOf(c, metaType));
+                return outputs ? this.core.loadChildren(outputs) : [];
+            })
+            .then(outputs => {
+                var bases = outputs.map(node => this.core.getMetaType(node));
+                // return [[arg1, Type1, node1], [arg2, Type2, node2]]
+                return outputs.map((node, i) => [
+                    this.getAttribute(node, 'name'),
+                    this.getAttribute(bases[i], 'name'),
+                    node
+                ]);
+            });
+    };
+
+    //////////////////////////// Special Operations ////////////////////////////
+    ExecuteJob.prototype.executeLocalOperation = function (type, node) {
+        // Retrieve the given LOCAL_OP type
+        if (!this[type]) {
+            this.logger.error(`No local operation handler for ${type}`);
+        }
+        this.logger.info(`Running local operation ${type}`);
+
+        return this[type](node);
+    };
+
+    _.extend(
+        ExecuteJob.prototype,
+        ExecuteJobFiles.prototype,
+        ExecuteJobMetadata.prototype,
+        ExecuteJobSafeSave.prototype,
+        PtrCodeGen.prototype,
+        LocalExecutor.prototype
+    );
+
+    ExecuteJob.prototype.processStdout = function (job, text, continued) {
+        var lines = text.replace(/\u0000/g, '').split('\n'),
+            result = this.parseForMetadataCmds(job, lines, !continued);
+
+        result.stdout = utils.resolveCarriageReturns(result.stdout).join('\n');
+        return result;
+    };
+
+    //////////////////////////// Metadata ////////////////////////////
+    ExecuteJob.prototype.parseForMetadataCmds = function (job, lines, skip) {
+        var jobId = this.core.getPath(job),
+            args,
+            result = [],
+            cmdCnt = 0,
+            ansiRegex = /\[\d+(;\d+)?m/g,
+            hasMetadata = false,
+            trimStartRegex = new RegExp(CONSTANTS.START_CMD + '.*'),
+            matches,
+            cmd;
+
+        for (var i = 0; i < lines.length; i++) {
+            // Check for a deepforge command
+            if (lines[i].indexOf(CONSTANTS.START_CMD) !== -1) {
+                matches = lines[i].replace(ansiRegex, '').match(trimStartRegex);
+                for (var m = 0; m < matches.length; m++) {
+                    cmdCnt++;
+                    args = matches[m].split(/\s+/);
+                    args.shift();
+                    cmd = args[0];
+                    args[0] = job;
+                    if (this[cmd] && (!skip || cmdCnt >= this.lastAppliedCmd[jobId])) {
+                        this[cmd].apply(this, args);
+                        this.lastAppliedCmd[jobId]++;
+                        hasMetadata = true;
+                    } else if (!this[cmd]) {
+                        this.logger.error(`Invoked unimplemented metadata method "${cmd}"`);
+                    }
+                }
+            } else {
+                result.push(lines[i]);
+            }
+        }
+        return {
+            stdout: result.join('\n'),
+            hasMetadata: hasMetadata
+        };
+    };
+
+    return ExecuteJob;
+});

src/plugins/ExecuteJob/metadata.json

@@ -0,0 +1,14 @@
+{
+  "id": "ExecuteJob",
+  "name": "ExecuteJob",
+  "version": "0.1.0",
+  "description": "",
+  "icon": {
+    "class": "glyphicon glyphicon-cog",
+    "src": ""
+  },
+  "disableServerSideExecution": false,
+  "disableBrowserSideExecution": false,
+  "writeAccessRequired": false,
+  "configStructure": []
+}

src/plugins/ExecuteJob/templates/deepforge.ejs

@@ -0,0 +1,97 @@
+-- Instantiate the deepforge object
+deepforge = {}
+
+function deepforge.initialize()
+    require 'nn'
+    require 'rnn'
+    require './classes/init'
+    require './custom-layers'
+end
+
+function deepforge.id()
+    if __deepforge_id == nil then
+        __deepforge_id = 0
+    end
+    __deepforge_id = __deepforge_id + 1
+    return __deepforge_id
+end
+
+function deepforge._cmd(...)
+    local cmd = '<%= START_CMD %>'
+    local arg = {...}
+    local n = #arg
+    for i=1,n do
+        cmd = cmd .. ' ' .. tostring(arg[i])
+    end
+    print(cmd .. ' ')  -- guarantee ends w/ space
+end
+
+-- Graph support
+Graph = torch.class('deepforge.Graph')
+
+function Graph:__init(name)
+    self.id = deepforge.id()
+    self.name = name
+    deepforge._cmd('<%= GRAPH_CREATE %>', self.id, name)
+end
+
+_Line = torch.class('deepforge._Line')
+
+function _Line:__init(graphId, name, opts)
+    self.id = deepforge.id()
+    self.name = name
+    deepforge._cmd('<%= GRAPH_CREATE_LINE %>', graphId, self.id, name)
+end
+
+function _Line:add(x, y)
+    assert(type(x) == "number" and type(y) == "number", "adding point (" .. tostring(x) .. ", " .. tostring(y) .. ") to " .. self.name .. " failed: expected (number, number)")
+    deepforge._cmd('<%= GRAPH_PLOT %>', self.id, x, y)
+end
+
+function Graph:line(name, opts)
+    return deepforge._Line(self.id, name, opts)
+end
+
+-- Image support
+local function saveImage(name, tensor)
+    require 'image'
+    require 'paths'
+
+    -- save it in the tmp directory
+    local filename = name .. '.png'
+    local path = paths.concat('metadata', filename)
+
+    if paths.dir('metadata') == nil then
+       paths.mkdir('metadata')
+    end
+
+    image.save(path, tensor)
+end
+
+function deepforge.image(name, tensor)
+    saveImage(name, tensor)
+    deepforge._cmd("<%= IMAGE.BASIC %>", deepforge.id(), name)
+end
+
+Image = torch.class('deepforge.Image')
+function Image:__init(name, tensor)
+    self.id = deepforge.id()
+    self.name = name
+
+    if tensor ~= nil then
+        saveImage(name, tensor)
+        deepforge._cmd('<%= IMAGE.CREATE %>', self.id, self.name)
+    end
+end
+
+function Image:update(tensor)
+    saveImage(self.name, tensor)
+    deepforge._cmd('<%= IMAGE.UPDATE %>', self.id, self.name)
+end
+
+function Image:title(name)
+    self.name = name
+    deepforge._cmd('<%= IMAGE.NAME %>', self.id, self.name)
+end
+
+return deepforge

src/plugins/ExecuteJob/templates/deserialize.ejs

@@ -0,0 +1,6 @@
+require 'paths'
+
+local path = 'inputs/<%= name %>/data'
+local abs_path = paths.concat('inputs', '<%= name %>', 'data')
+
+<%= code %>

src/plugins/ExecuteJob/templates/entry.ejs

@@ -0,0 +1,16 @@
+-- Instantiate the deepforge object
+require './deepforge'
+
+-- run the <%= name %> and serialize the results
+print('\n############### Running "<%= name.replace(/'/g, '\\\'') %>" Operation ############### ')
+results = require './main'
+print('############### "<%= name.replace(/'/g, '\\\'') %>" Operation Complete! ###############')
+
+-- serialize by type
+outputs = require './outputs'
+<% outputs.forEach(pair => { 
+    var name = pair[0],
+        type = pair[1];
+%>
+outputs.<%= type %>('<%= name %>', results.<%= name %>)
+<% }); %>

src/plugins/ExecuteJob/templates/index.js

@@ -1,19 +1,26 @@
+/*globals define*/
 define([
+    'text!./start.ejs',
     'text!./entry.ejs',
     'text!./main.ejs',
+    'text!./deepforge.ejs',
     'text!./serialize.ejs',
     'text!./deserialize.ejs'
 ], function(
+    START,
     ENTRY,
     MAIN,
+    DEEPFORGE,
     SERIALIZE,
     DESERIALIZE
 ) {
 
     return {
+        START,
         ENTRY,
         MAIN,
         SERIALIZE,
+        DEEPFORGE,
         DESERIALIZE
     };
 });