v1.0.0

* Added pipelineName, pipelineInputNames fields. Fixes #975

* WIP #975 Added logs, removed unnecessary code

* WIP #975 Updated export pipeline to cli

README.md

@@ -1,8 +1,8 @@
 [![Release State](https://img.shields.io/badge/state-beta-yellow.svg)](https://img.shields.io/badge/state-beta-yellow.svg)
 [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](./LICENSE)
-[![Build Status](https://travis-ci.org/dfst/deepforge.svg?branch=master)](https://travis-ci.org/dfst/deepforge)
-[![Join the chat at https://gitter.im/dfst/deepforge](https://badges.gitter.im/dfst/deepforge.svg)](https://gitter.im/dfst/deepforge?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
-[![Stories in Ready](https://badge.waffle.io/dfst/deepforge.png?label=ready&title=Ready)](https://waffle.io/dfst/deepforge)
+[![Build Status](https://travis-ci.org/deepforge-dev/deepforge.svg?branch=master)](https://travis-ci.org/deepforge-dev/deepforge)
+[![Join the chat at https://gitter.im/deepforge-dev/deepforge](https://badges.gitter.im/deepforge-dev/deepforge.svg)](https://gitter.im/deepforge-dev/deepforge?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+[![Stories in Ready](https://badge.waffle.io/deepforge-dev/deepforge.png?label=ready&title=Ready)](https://waffle.io/deepforge-dev/deepforge)
 
 **Notice**: DeepForge is still a work in progress and in beta! That being said, any contributions and/or feedback is greatly appreciated. If you have any questions, check out the [wiki](https://github.com/dfst/deepforge/wiki/) or drop me a line on the gitter!
 
@@ -40,6 +40,14 @@ Finally, start deepforge with `deepforge start`and navigate to [http://localhost
 
 Also, be sure to check out the other available features of the `deepforge` cli; it can be used to update, manage your torch installation, uninstall deepforge and run individual components!
 
+## Additional Resources
+- [Intro to DeepForge Slides](https://docs.google.com/presentation/d/10_y5O3gHXSATfjHVLJg7dOdrz-tAXNWjlxhJ5SlA0ic/edit?usp=sharing)
+- [wiki](https://github.com/deepforge-dev/deepforge/wiki) containing overview, installation, configuration and developer information
+- [Starter Kit](https://github.com/deepforge-dev/examples/tree/master/starterkit) containing example pipelines demonstrating various deepforge features
+- [Examples](https://github.com/deepforge-dev/examples)
+
+- [Datamodel Developer Slides](https://docs.google.com/presentation/d/1hd3IyUlzW_TIPnzCnE-1pdz00Pw8WaIxYiOW_Hyog-M/edit#slide=id.p)
+
 ## Interested in contributing?
 Contributions are welcome! Either fork the project and submit some PR's or shoot me an email about getting more involved!
 

_config.yml

@@ -0,0 +1 @@
+theme: jekyll-theme-cayman

bin/deepforge

@@ -8,7 +8,8 @@ var Command = require('commander').Command,
     execSync = childProcess.execSync,
     path = require('path'),
     fs = require('fs'),
-    version = require('../package.json').version,
+    pkgJson = require('../package.json'),
+    version = pkgJson.version,
     exists = require('exists-file'),
     DEFAULT_CONFIG = require('./config.json'),
     merge = require('lodash.merge'),
@@ -334,7 +335,7 @@ program
         if (!args.torch || args.server) {
 
             if (args.git) {
-                pkg = 'dfst/deepforge';
+                pkg = pkgJson.repository.url;
             } else {
                 // Check the version
                 try {
@@ -447,6 +448,10 @@ program
         }
     });
 
+// extensions
+program
+    .command('extensions <command>', 'Manage deepforge extensions');
+
 module.exports = function(cmd) {
     var cmds = cmd.split(/\s+/).filter(w => !!w);
     cmds.unshift('./bin/deepforge');

bin/deepforge-extensions

@@ -0,0 +1,67 @@
+#!/usr/bin/env node
+
+var Command = require('commander').Command,
+    program = new Command(),
+    extender = require('../utils/extender');
+
+// Supported commands
+//  - add
+//  - remove
+//  - list
+//  - update
+program
+    .command('add <project>')
+    .description('Add an extension to deepforge')
+    .option('-n, --name <name>', 'Project name (if different from <project>)')
+    .action(project => {
+        console.log('loading extension from: ' + project);
+        extender.install(project)
+            .then(extConfig => 
+                console.log(`The ${extConfig.name} extension has been added to deepforge.`))
+            .fail(err => {
+                console.error('Could not install extension:\n');
+                console.error(err);
+                process.exit(1);
+            });
+    });
+
+program
+    .command('remove <name>').alias('rm')
+    .description('Remove an extension from deepforge')
+    .action(name => {
+        try {
+            extender.uninstall(name);
+            console.log(`${name} has been successfully removed!`);
+        } catch (e) {
+            console.error('Could not remove extension:');
+            console.error(e);
+            process.exit(1);
+        }
+    });
+
+program
+    .command('list').alias('ls')
+    .description('List installed deepforge extensions')
+    .action(() => {
+        var allExtConfigs = extender.getExtensionsConfig(),
+            types = Object.keys(allExtConfigs),
+            hasContents = false,
+            names;
+
+        for (var i = types.length; i--;) {
+            names = Object.keys(allExtConfigs[types[i]]);
+            if (names.length) {
+                hasContents = true;
+                console.log(types[i]);
+                for (var j = names.length; j--;) {
+                    console.log(`  ${names[j]}`);
+                }
+            }
+        }
+
+        if (!hasContents) {
+            console.log('No installed extensions');
+        }
+    });
+
+program.parse(process.argv);

bin/start-local.js

@@ -13,8 +13,8 @@ if (gmeConfig.blob.type === 'FS') {
 }
 
 process.env.NODE_ENV = 'local';
-execJob = spawn('npm', [
-    'start'
+execJob = spawn('node', [
+    path.join(__dirname, '..', 'app.js')
 ], env);
 execJob.stdout.pipe(process.stdout);
 execJob.stderr.pipe(process.stderr);

bin/start-worker.js

@@ -6,8 +6,7 @@ var path = require('path'),
     spawn = childProcess.spawn,
     rm_rf = require('rimraf'),
     projectConfig = require(__dirname + '/../config'),
-    executorSrc = path.join(__dirname, '..', 'node_modules', 'webgme', 'src',
-        'server', 'middleware', 'executor', 'worker'),
+    executorSrc = path.join(__dirname, '..', 'node_modules', '.bin', 'webgme-executor-worker'),
     id = Date.now(),
     workerRootPath = process.env.DEEPFORGE_WORKER_DIR || path.join(__dirname, '..', 'src', 'worker'),
     workerPath = path.join(workerRootPath, `worker_${id}`),
@@ -60,7 +59,7 @@ var startExecutor = function() {
 
     // Start the executor
     var execJob = spawn('node', [
-        'node_worker.js',
+        executorSrc,
         workerConfigPath,
         workerTmp
     ]);
@@ -80,14 +79,4 @@ var createConfigJson = function() {
     fs.writeFile(workerConfigPath, JSON.stringify(config), startExecutor);
 };
 
-process.chdir(executorSrc);
-
-fs.mkdir(workerTmp, function() {
-    // npm install in this directory
-    var npmInstall = spawn('npm', ['install']);
-    npmInstall.stdout.pipe(process.stdout);
-    npmInstall.stderr.pipe(process.stderr);
-    npmInstall.on('close', function() {
-        createConfigJson();
-    });
-});
+fs.mkdir(workerTmp, createConfigJson);

config/components.json

@@ -48,10 +48,6 @@
             "ImportTorch": {
                 "icon": "import_export",
                 "priority": -1
-            },
-            "GenerateExecFile": {
-                "icon": "play_for_work",
-                "priority": -1
             }
         }
     },

package.json

@@ -1,28 +1,35 @@
 {
   "name": "deepforge",
+  "repository": {
+      "type": "git",
+      "url": "https://github.com/deepforge-dev/deepforge.git"
+  },
   "bin": {
     "deepforge": "./bin/deepforge"
   },
   "scripts": {
-    "start": "node app.js",
+    "start": "./bin/deepforge start",
+    "postinstall": "node utils/reinstall-extensions.js",
     "start-dev": "NODE_ENV=dev node app.js",
     "local": "node ./bin/start-local.js",
     "worker": "node ./bin/start-worker.js",
-    "test": "mkdir ./test-tmp; node ./node_modules/mocha/bin/mocha --recursive test",
-    "watch-test": "./node_modules/nodemon/bin/nodemon.js --exec 'node ./node_modules/mocha/bin/mocha --recursive test'",
+    "test": "mkdir ./test-tmp; mocha --recursive test",
+    "watch-test": "nodemon --exec 'mocha --recursive test'",
     "build-nn": "node ./utils/nn-parser.js"
   },
-  "version": "0.21.0",
+  "version": "1.0.0",
   "dependencies": {
     "commander": "^2.9.0",
     "dotenv": "^2.0.0",
     "exists-file": "^2.1.0",
     "express": "^4.14.0",
-    "lodash.difference": "^4.1.2",
     "graceful-fs": "^4.1.10",
+    "lodash.difference": "^4.1.2",
     "lodash.merge": "^4.5.1",
+    "lodash.template": "^4.4.0",
     "mongodb": "^2.2.10",
     "nodemon": "^1.9.2",
+    "npm": "^4.0.5",
     "q": "1.4.1",
     "rimraf": "^2.4.0",
     "webgme": "^2.7.1",
@@ -30,6 +37,7 @@
     "webgme-breadcrumbheader": "^2.1.1",
     "webgme-chflayout": "^2.0.0",
     "webgme-easydag": "dfst/webgme-easydag",
+    "webgme-executor-worker": "^1.0.1",
     "webgme-fab": "dfst/webgme-fab",
     "webgme-simple-nodes": "^2.1.0"
   },

src/common/globals.js

@@ -83,7 +83,9 @@ define([
             exists = {},
             i = 2;
 
-        children.forEach(child => exists[child.getAttribute('name')] = true);
+        children
+            .filter(child => child !== null)
+            .forEach(child => exists[child.getAttribute('name')] = true);
 
         while (exists[name]) {
             name = basename + '_' + i;
@@ -269,10 +271,8 @@ define([
             .filter(n => !n.getRegistry('isAbstract'))
             .map(node => node.getAttribute('name'));
 
-        //this.logger.info(`Found ${dataTypes.length} data types`);
-
-        // Add the target type to the pluginMetadata... hacky :/
-        var metadata = WebGMEGlobal.allPluginsMetadata[UPLOAD_PLUGIN], 
+        // Add the target type to the pluginMetadata...
+        var metadata = WebGMEGlobal.allPluginsMetadata[UPLOAD_PLUGIN],
             config = metadata.configStructure
                 .find(opt => opt.name === DATA_TYPE_CONFIG.name);
 

src/decorators/ArtifactOpDecorator/EasyDAG/ArtifactOpDecorator.EasyDAGWidget.js

@@ -40,10 +40,40 @@ define([
     ArtifactOpDecorator.prototype.DECORATOR_ID = DECORATOR_ID;
 
     ArtifactOpDecorator.prototype.getTargetFilterFnFor = function() {
+        var currentNode,
+            pId,
+            peerIds,
+            currentDataId,
+            targetTypeIds = [];
+
+        // Get all connections from this node's data
+        if (this._node.outputs[0]) {
+            currentNode = this.client.getNode(this._node.id);
+            pId = currentNode.getParentId();
+            peerIds = this.client.getNode(pId).getChildrenIds();
+            currentDataId = this._node.outputs[0].id;
+
+            targetTypeIds = peerIds.map(id => this.client.getNode(id))
+                .filter(node => {
+                    var ptr = node.getPointer('src');
+                    return ptr.to === currentDataId;
+                })
+
+                // Get the target data types
+                .map(node => this.client.getNode(node.getPointer('dst').to).getMetaTypeId());
+        }
+
         return id => {
             var node = this.client.getNode(id),
-                isMetaTgt = node.getId() === node.getMetaTypeId();
-            return isMetaTgt === this.castOpts.metaTgt;
+                isMetaTgt = node.getId() === node.getMetaTypeId(),
+                isValidType = true;
+
+            // make sure it is a type of the target types
+            isValidType = targetTypeIds.reduce((passing, typeId) => {
+                return passing && node.isTypeOf(typeId);
+            }, true);
+
+            return isValidType && isMetaTgt === this.castOpts.metaTgt;
         };
     };
 

src/plugins/ExecuteJob/ExecuteJob.Files.js

@@ -73,17 +73,17 @@ define([
 
         classNodes = metanodes.filter(node => {
             var base = this.core.getBase(node),
-                baseId = this.core.getPath(base),
+                baseId,
                 count = 1;
 
             // Count the sets back to a class node
             while (base) {
+                baseId = this.core.getPath(base);
                 if (isClass[baseId]) {
                     inheritanceLvl[this.core.getPath(node)] = count;
                     return true;
                 }
                 base = this.core.getBase(base);
-                baseId = this.core.getPath(base);
                 count++;
             }
 
@@ -285,7 +285,7 @@ define([
     };
 
     ExecuteJob.prototype.createAttributeFile = function (node, files) {
-        var skip = ['code', 'stdout', 'execFiles', 'jobId', 'secret'],
+        var skip = ['code', 'stdout', 'execFiles', 'jobId', 'secret', CONSTANTS.LINE_OFFSET],
             numOrBool = /^(-?\d+\.?\d*((e|e-)\d+)?|(true|false))$/,
             table;
 

src/plugins/ExecuteJob/templates/start.ejs

@@ -74,7 +74,7 @@ requirejs([
     var checkFinished = () => {
         if (exitCode !== null && remainingImageCount === 0) {
             log('finished!');
-            process.exit(exitCode);
+            cleanup();
         }
     };
 
@@ -201,14 +201,45 @@ requirejs([
 
     // Download the large files
     var inputData = JSON.parse(fs.readFileSync('./input-data.json')),
-        inputPaths = Object.keys(inputData);
+        inputPaths = Object.keys(inputData),
+        pid,
+        job,
+        cleanup;
+
+    // Make sure to kill the spawned process group on exit
+    cleanup = function() {
+        if (job) {
+            pid = job.pid;
+            job = null;
+            log(`killing process group: ${pid}`);
+            process.kill(-pid, 'SIGTERM');
+            if (exitCode !== null) {
+                log(`exiting w/ code ${exitCode}`);
+                process.exit(exitCode);
+            }
+        }
+    };
+
+    process.on('exit', () => {
+        log('received "exit" event')
+        cleanup();
+    });
+    process.on('SIGINT', function() {
+        log('received "SIGINT" event')
+        cleanup();
+        process.exit(130);
+    });
+    process.on('uncaughtException', () => {
+        log('received "uncaughtException" event')
+        cleanup();
+    });
 
     // Request the data from the blob
     prepareCache()
         .then(() => Q.all(inputPaths.map(ipath => getData(ipath, inputData[ipath]))))
         .then(() => {
             // Run 'th init.lua' and merge the stdout, stderr
-            var job = spawn('th', ['init.lua']);
+            job = spawn('th', ['init.lua'], {detached: true});
             job.stdout.on('data', onStdout);
             job.stderr.on('data', onStderr);
             job.on('close', code => {

src/plugins/Export/Export.js

@@ -4,19 +4,21 @@
 define([
     'text!./metadata.json',
     'text!./deepforge.ejs',
-    'text!./toboolean.lua',
+    './format',
     'plugin/PluginBase',
     'deepforge/plugin/PtrCodeGen',
     'deepforge/Constants',
+    'blob/BlobConfig',
     'underscore',
     'q'
 ], function (
     pluginMetadata,
-    deepForgeTxt,
-    TOBOOLEAN,
+    DeepForgeBaseCode,
+    FORMATS,
     PluginBase,
     PtrCodeGen,
     CONSTANTS,
+    BlobConfig,
     _,
     Q
 ) {
@@ -29,21 +31,16 @@ define([
             code: true
         },
         RESERVED = /^(and|break|do|else|elseifend|false|for|function|if|in|local|nil|not|orrepeat|return|then|true|until|while|print)$/,
-        INDENT = '   ',
-        INIT_CLASSES_FN = '__initClasses',
-        INIT_LAYERS_FN = '__initLayers',
-        DEEPFORGE_CODE = _.template(deepForgeTxt)({
-            initCode: `${INIT_CLASSES_FN}()\n${INDENT}${INIT_LAYERS_FN}()`
-        });
+        DeepForgeTpl = _.template(DeepForgeBaseCode);
 
     /**
-     * Initializes a new instance of GenerateExecFile.
+     * Initializes a new instance of Export.
      * @class
      * @augments {PluginBase}
-     * @classdesc This class represents the plugin GenerateExecFile.
+     * @classdesc This class represents the plugin Export.
      * @constructor
      */
-    var GenerateExecFile = function () {
+    var Export = function () {
         // Call base class' constructor.
         PluginBase.call(this);
         this.initRecords();
@@ -54,13 +51,13 @@ define([
      * This is also available at the instance at this.pluginMetadata.
      * @type {object}
      */
-    GenerateExecFile.metadata = pluginMetadata;
+    Export.metadata = pluginMetadata;
 
     // Prototypical inheritance from PluginBase.
-    GenerateExecFile.prototype = Object.create(PluginBase.prototype);
-    GenerateExecFile.prototype.constructor = GenerateExecFile;
+    Export.prototype = Object.create(PluginBase.prototype);
+    Export.prototype.constructor = Export;
 
-    GenerateExecFile.prototype.initRecords = function() {
+    Export.prototype.initRecords = function() {
         this.pluginMetadata = pluginMetadata;
 
         this._srcIdFor = {};  // input path -> output data node path
@@ -98,9 +95,7 @@ define([
      *
      * @param {function(string, plugin.PluginResult)} callback - the result callback
      */
-    GenerateExecFile.prototype.main = function (callback) {
-        var name = this.core.getAttribute(this.activeNode, 'name');
-
+    Export.prototype.main = function (callback) {
         this.initRecords();
 
         // Get all the children and call generate exec file
@@ -112,8 +107,8 @@ define([
         }
 
         return this.core.loadChildren(this.activeNode)
-            .then(nodes => this.createExecFile(nodes))
-            .then(code => this.blobClient.putFile(`${name}.lua`, code))
+            .then(nodes => this.generateOutputFiles(nodes))
+            .catch(err => callback(err))
             .then(hash => {
                 this.result.addArtifact(hash);
                 this.result.setSuccess(true);
@@ -122,62 +117,84 @@ define([
             .fail(err => callback(err));
     };
 
-    GenerateExecFile.prototype.createExecFile = function (children) {
+    Export.prototype.getCurrentConfig = function () {
+        var config = PluginBase.prototype.getCurrentConfig.call(this);
+        config.staticInputs = config.staticInputs || [];
+        return config;
+    };
+
+    Export.prototype.getExporterFor = function (name) {
+        var Exporter = function() {},
+            format = FORMATS[name],
+            exporter;
+
+        Exporter.prototype = this;
+        exporter = new Exporter();
+
+        if (typeof format === 'function') {
+            exporter.main = format;
+        } else {
+            _.extend(exporter, format);
+        }
+        return exporter;
+    };
+
+    Export.prototype.generateOutputFiles = function (children) {
+        var name = this.core.getAttribute(this.activeNode, 'name');
+
         return this.createCodeSections(children)
             .then(sections => {
-                var classes,
-                    initClassFn,
-                    initLayerFn,
-                    code = [];
+                // Get the selected format
+                var config = this.getCurrentConfig(),
+                    format = config.format || 'Basic CLI',
+                    exporter,
+                    staticInputs,
+                    files;
 
-                // concat all the sections into a single file
+                this.logger.info(`About to retrieve ${config.format} exporter`);
+                exporter = this.getExporterFor(format);
 
-                // wrap the class/layer initialization in a fn
-                // Add the classes ordered wrt their deps
-                classes = Object.keys(sections.classes)
-                    .sort((a, b) => {
-                        // if a depends on b, switch them (return 1)
-                        if (sections.classDependencies[a].includes(b)) {
-                            return 1;
+                staticInputs = config.staticInputs.map(id => {
+                    var opId = id.split('/').splice(0, this.activeNodeDepth).join('/'),
+                        port = this._portCache[id];
+
+                    return {
+                        portId: id,
+                        id: opId,
+                        hash: this.core.getAttribute(port, 'data'),
+                        name: this._nameFor[opId]
+                    };
+                });
+
+                this.logger.info('Invoking exporter "main" function...');
+                try {
+                    files = exporter.main(sections, staticInputs, config.extensionConfig);
+                } catch (e) {
+                    this.logger.error(`Exporter failed: ${e.toString()}`);
+                    throw e;
+                }
+                // If it returns a string, just put a single file
+                if (typeof files === 'string') {
+                    return this.blobClient.putFile(`${name}.lua`, files);
+                } else {  // filename -> content
+                    var artifact = this.blobClient.createArtifact(name),
+                        objects = {};
+
+                    Object.keys(files).forEach(key => {
+                        if (BlobConfig.hashRegex.test(files[key])) {
+                            objects[key] = files[key];
+                            delete files[key];
                         }
-                        return -1;
-                    })
-                    // Create fns from the classes
-                    .map(name => [
-                        `local function init${name}()`,
-                        indent(sections.classes[name]),
-                        'end',
-                        `init${name}()`
-                    ].join('\n'));
+                    });
 
-                initClassFn = [
-                    `local function ${INIT_CLASSES_FN}()`,
-                    indent(classes.join('\n\n')),
-                    'end'
-                ].join('\n');
-
-                code = code.concat(initClassFn);
-
-                // wrap the layers in a function
-                initLayerFn = [
-                    `local function ${INIT_LAYERS_FN}()`,
-                    indent(_.values(sections.layers).join('\n\n')),
-                    'end'
-                ].join('\n');
-                code = code.concat(initLayerFn);
-                code = code.concat(_.values(sections.operations));
-
-                code = code.concat(_.values(sections.pipelines));
-
-                code.push(DEEPFORGE_CODE);
-                code.push('deepforge.initialize()');
-                code.push(sections.main);
-
-                return code.join('\n\n');
+                    return artifact.addFiles(files)
+                        .then(() => artifact.addObjectHashes(objects))
+                        .then(() => artifact.save());
+                }
             });
     };
 
-    GenerateExecFile.prototype.createCodeSections = function (children) {
+    Export.prototype.createCodeSections = function (children) {
         // Convert opNodes' jobs to the nested operations
         var opNodes,
             nodes;
@@ -221,7 +238,7 @@ define([
             .fail(err => this.logger.error(err));
     };
 
-    GenerateExecFile.prototype.unpackJobs = function (nodes) {
+    Export.prototype.unpackJobs = function (nodes) {
         return Q.all(
             nodes.map(node => {
                 if (!this.isMetaTypeOf(node, this.META.Job)) {
@@ -235,7 +252,7 @@ define([
         );
     };
 
-    GenerateExecFile.prototype.sortOperations = function (operationDict, opIds) {
+    Export.prototype.sortOperations = function (operationDict, opIds) {
         var nextIds = [],
             sorted = opIds,
             dstIds,
@@ -263,7 +280,7 @@ define([
             .concat(this.sortOperations(operationDict, nextIds));
     };
 
-    GenerateExecFile.prototype.generateCodeSections = function(sortedOps) {
+    Export.prototype.generateCodeSections = function(sortedOps) {
         // Create the code sections:
         //  - operation definitions
         //  - pipeline definition
@@ -293,8 +310,12 @@ define([
         // Define the pipeline function
         code.pipelines = this.definePipelineFn(mainOps, outputOps);
 
-        // Define the main body
-        this.addCodeMain(code);
+        // Define the serializers/deserializers
+        this.addCodeSerializers(code);
+
+        // Define the main input names
+        code.pipelineName = Object.keys(code.pipelines)[0];
+        code.pipelineInputNames = Object.keys(this.isInputOp).map(id => this._nameFor[id]);
 
         // Add custom class definitions
         this.addCustomClasses(code);
@@ -305,11 +326,13 @@ define([
         return code;
     };
 
-    var indent = function(text) {
-        return text.replace(/^/mg, INDENT);
+    // expose this utility function to format extensions
+    var indent = Export.prototype.indent = function(text, spaces) {
+        spaces = spaces || 3;
+        return text.replace(/^/mg, new Array(spaces+1).join(' '));
     };
 
-    GenerateExecFile.prototype.defineOperationFn = function(operation) {
+    Export.prototype.defineOperationFn = function(operation) {
         var lines = [],
             args = operation.inputNames || [];
 
@@ -327,7 +350,7 @@ define([
         return lines.join('\n');
     };
 
-    GenerateExecFile.prototype.definePipelineFn = function(sortedOps, outputOps) {
+    Export.prototype.definePipelineFn = function(sortedOps, outputOps) {
         var inputArgs = Object.keys(this.isInputOp).map(id => this._nameFor[id]),
             name = this.core.getAttribute(this.activeNode, 'name'),
             safename = getUniqueName(name, this._opBaseNames),
@@ -353,7 +376,7 @@ define([
         return result;
     };
 
-    GenerateExecFile.prototype.getOutputPair = function(operation) {
+    Export.prototype.getOutputPair = function(operation) {
         var input = operation.inputValues[0].slice(),
             value;
 
@@ -363,39 +386,36 @@ define([
         return [this._nameFor[operation.id], value];
     };
 
-    GenerateExecFile.prototype.addCodeMain = function(sections) {
-        var pipelineName = Object.keys(sections.pipelines)[0],
-            hasBool = false,
-            code = [],
-            loadNodes = {},
-            args;
+    Export.prototype.addCodeSerializers = function(sections) {
+        var loadNodes = {},
+            saveNodes = {};
 
-        args = Object.keys(this.isInputOp).map((id, index) => {
-            var node = this.inputNode[id],
-                base = this.core.getBase(node),
-                type = this.core.getAttribute(base, 'name'),
-                arg = `arg[${index+1}]`;
+        // Add the serializer fn names for each input
+        sections.serializerFor = {};
+        sections.deserializerFor = {};
 
-            if (type === 'boolean') {
-                hasBool = true;
-                return `toboolean(${arg})`;
-            } else if (type === 'number') {
-                return `tonumber(${arg})`;
-            } else if (type === 'string') {
-                return arg;
-            } else {
-                loadNodes[id] = node;
-                return `load['${this._nameFor[id]}'](${arg})`;
-            }
+        Object.keys(this.isOutputOp).map(id => {
+            var name = this._nameFor[id];
+            sections.serializerFor[name] = `__save['${name}']`;
         });
 
-        // Handle the arg types
-        if (hasBool) {
-            // add toboolean def
-            code.push(TOBOOLEAN);
-        }
-        // Define the 'saveOutputs' method
-        var saveNodes = {};
+        // Add the serializer definitions
+        Object.keys(this.isInputOp).forEach(id => {
+            var node = this.inputNode[id],
+                name = this._nameFor[id];
+
+            loadNodes[id] = node;
+            sections.deserializerFor[name] = `__load['${this._nameFor[id]}']`;
+        });
+
+        sections.deserializers = this.createTorchFnDict(
+            '__load',
+            loadNodes,
+            'deserialize',
+            'path'
+        );
+
+        // Add the deserializer definitions
         Object.keys(this.outputDataToOpId).forEach(dataId => {
             var opId = this.outputDataToOpId[dataId];
             // The key is used for the output name resolution. The
@@ -405,32 +425,31 @@ define([
             saveNodes[opId] = this._portCache[this._srcIdFor[dataId]];
         });
 
-        // Add dictionary of serializers/deserializers
-        code.push(
-            this.createTorchFnDict('load', loadNodes, 'deserialize', 'path'),
-            this.createTorchFnDict('save', saveNodes, 'serialize', 'path, data')
+        sections.serializers = this.createTorchFnDict(
+            '__save',
+            saveNodes,
+            'serialize',
+            'path, data'
         );
 
         // Add a saveOutputs method for convenience
-        code.push([
-            'local function saveOutputs(data)',
+        sections.serializeOutputsDef = [
+            'local function __saveOutputs(data)',
             indent(Object.keys(this.isOutputOp).map(id => {
                 var name = this._nameFor[id];
-                return `print('saving ${name}...')\nsave['${name}']('${name}', data['${name}'])`;
+                return [
+                    `print('saving ${name}...')`,
+                    `${sections.serializerFor[name]}('${name}', data['${name}'])`
+                ].join('\n');
+
             }).join('\n')),
             'end'
-        ].join('\n'));
+        ].join('\n');
 
-        code.push(
-            `local outputs = ${pipelineName}(${args.join(', ')})\n` +
-            'saveOutputs(outputs)',
-            'return outputs'
-        );
-
-        sections.main = code.join('\n\n');
+        sections.serializeOutputs = '__saveOutputs(outputs)';
     };
 
-    GenerateExecFile.prototype.createTorchFnDict = function(name, nodeDict, attr, args) {
+    Export.prototype.createTorchFnDict = function(name, nodeDict, attr, args) {
         return [
             `local ${name} = {}`,
             Object.keys(nodeDict).map(id => {
@@ -444,7 +463,7 @@ define([
         ].join('\n');
     };
 
-    GenerateExecFile.prototype.addCustomClasses = function(sections) {
+    Export.prototype.addCustomClasses = function(sections) {
         var metaDict = this.core.getAllMetaNodes(this.rootNode),
             isClass,
             metanodes,
@@ -460,7 +479,7 @@ define([
 
         classNodes = metanodes.filter(node => {
             var base = this.core.getBase(node),
-                baseId = this.core.getPath(base),
+                baseId,
                 deps = [],
                 name,
                 count = 1;
@@ -468,6 +487,7 @@ define([
             // Count the sets back to a class node
             while (base) {
                 deps.push(this.core.getAttribute(base, 'name'));
+                baseId = this.core.getPath(base);
                 if (isClass[baseId]) {
                     inheritanceLvl[this.core.getPath(node)] = count;
                     name = this.core.getAttribute(node, 'name');
@@ -475,7 +495,6 @@ define([
                     return true;
                 }
                 base = this.core.getBase(base);
-                baseId = this.core.getPath(base);
                 count++;
             }
 
@@ -497,9 +516,19 @@ define([
 
                 sections.classes[name] = code;
             });
+
+        // order classes by dependency
+        sections.orderedClasses = Object.keys(sections.classes)
+            .sort((a, b) => {
+                // if a depends on b, switch them (return 1)
+                if (sections.classDependencies[a].includes(b)) {
+                    return 1;
+                }
+                return -1;
+            });
     };
 
-    GenerateExecFile.prototype.addCustomLayers = function(sections) {
+    Export.prototype.addCustomLayers = function(sections) {
         var metaDict = this.core.getAllMetaNodes(this.rootNode),
             isCustomLayer,
             metanodes,
@@ -523,7 +552,7 @@ define([
     };
 
 
-    GenerateExecFile.prototype.getTypeDictFor = function (name, metanodes) {
+    Export.prototype.getTypeDictFor = function (name, metanodes) {
         var isType = {};
         // Get all the custom layers
         for (var i = metanodes.length; i--;) {
@@ -541,7 +570,7 @@ define([
         return `"${attr}"`;
     };
 
-    GenerateExecFile.prototype.getOpInvocation = function(op) {
+    Export.prototype.getOpInvocation = function(op) {
         var lines = [],
             attrs,
             refInits = [],
@@ -574,13 +603,13 @@ define([
         return lines.join('\n');
     };
 
-    GenerateExecFile.prototype.getOutputName = function(node) {
+    Export.prototype.getOutputName = function(node) {
         var basename = this.core.getAttribute(node, 'saveName');
 
         return getUniqueName(basename, this._outputNames, true);
     };
 
-    GenerateExecFile.prototype.getVariableName = function (/*node*/) {
+    Export.prototype.getVariableName = function (/*node*/) {
         var c = Object.keys(this.isInputOp).length;
 
         if (c !== 1) {
@@ -590,7 +619,7 @@ define([
         return 'input';
     };
 
-    GenerateExecFile.prototype.registerNode = function (node) {
+    Export.prototype.registerNode = function (node) {
         if (this.isMetaTypeOf(node, this.META.Operation)) {
             return this.registerOperation(node);
         } else if (this.isMetaTypeOf(node, this.META.Transporter)) {
@@ -619,7 +648,7 @@ define([
         return name;
     };
 
-    GenerateExecFile.prototype.registerOperation = function (node) {
+    Export.prototype.registerOperation = function (node) {
         var name = this.core.getAttribute(node, 'name'),
             id = this.core.getPath(node),
             base = this.core.getBase(node),
@@ -681,7 +710,7 @@ define([
             });
     };
 
-    GenerateExecFile.prototype.registerTransporter = function (node) {
+    Export.prototype.registerTransporter = function (node) {
         var outputData = this.core.getPointerPath(node, 'src'),
             inputData = this.core.getPointerPath(node, 'dst'),
             srcOpId = this.getOpIdFor(outputData),
@@ -700,7 +729,7 @@ define([
         this._incomingCnts[dstOpId]++;
     };
 
-    GenerateExecFile.prototype.getOpIdFor = function (dataId) {
+    Export.prototype.getOpIdFor = function (dataId) {
         var ids = dataId.split('/'),
             depth = ids.length;
 
@@ -715,7 +744,7 @@ define([
     //   - add the references
     //     - generate the code
     //     - replace the `return <thing>` w/ `<ref-name> = <thing>`
-    GenerateExecFile.prototype.createOperation = function (node) {
+    Export.prototype.createOperation = function (node) {
         var id = this.core.getPath(node),
             baseId = this.core.getPath(this.core.getBase(node)),
             attrNames = this.core.getValidAttributeNames(node),
@@ -790,12 +819,12 @@ define([
             });
     };
 
-    GenerateExecFile.prototype.genPtrSnippet = function (ptrName, pId) {
+    Export.prototype.genPtrSnippet = function (ptrName, pId) {
         return this.getPtrCodeHash(pId)
             .then(hash => this.blobClient.getObjectAsString(hash));
     };
 
-    GenerateExecFile.prototype.createHeader = function (title, length) {
+    Export.prototype.createHeader = function (title, length) {
         var len;
         title = ` ${title} `;
         length = length || HEADER_LENGTH;
@@ -813,7 +842,7 @@ define([
 
     };
 
-    GenerateExecFile.prototype.genOperationCode = function (operation) {
+    Export.prototype.genOperationCode = function (operation) {
         var header = this.createHeader(`"${operation.name}" Operation`),
             codeParts = [],
             body = [];
@@ -840,15 +869,57 @@ define([
         return operation;
     };
 
-    GenerateExecFile.prototype.assignResultToVar = function (code, name) {
-        var i = code.lastIndexOf('return');
+    _.extend(Export.prototype, PtrCodeGen.prototype);
 
-        return code.substring(0, i) +
-            code.substring(i)
-                .replace('return', `${name} = `);
+    // Extra utilities for export types
+    Export.prototype.INIT_CLASSES_FN = '__init_classes';
+    Export.prototype.INIT_LAYERS_FN = '__init_layers';
+    Export.prototype.getAllDefinitions = function (sections) {
+        var code = [],
+            classes,
+            initClassFn,
+            initLayerFn;
+
+        classes = sections.orderedClasses
+            // Create fns from the classes
+            .map(name => this.indent(sections.classes[name])).join('\n');
+
+        initClassFn = [
+            `local function ${this.INIT_CLASSES_FN}()`,
+            this.indent(classes),
+            'end'
+        ].join('\n');
+
+        code = code.concat(initClassFn);
+
+        // wrap the layers in a function
+        initLayerFn = [
+            `local function ${this.INIT_LAYERS_FN}()`,
+            this.indent(_.values(sections.layers).join('\n\n')),
+            'end'
+        ].join('\n');
+        code = code.concat(initLayerFn);
+
+        // Add operation fn definitions
+        code = code.concat(_.values(sections.operations));
+        code = code.concat(_.values(sections.pipelines));
+
+        // define deserializers, serializers
+        code.push(sections.deserializers);
+        code.push(sections.serializers);
+
+        code.push(this.getDeepforgeObject());
+        code.push('deepforge.initialize()');
+
+        code.push(sections.serializeOutputsDef);
+        return code.join('\n\n');
     };
 
-    _.extend(GenerateExecFile.prototype, PtrCodeGen.prototype);
+    Export.prototype.getDeepforgeObject = function (content) {
+        content = content || {};
+        content.initCode = content.initCode || `${this.INIT_CLASSES_FN}()\n${'   '}${this.INIT_LAYERS_FN}()`;
+        return DeepForgeTpl(content);
+    };
 
-    return GenerateExecFile;
+    return Export;
 });

src/plugins/Export/format.js

@@ -0,0 +1,13 @@
+
+/* globals define*/
+// The supported export formats and metadata
+define([
+    './formats/cli/cli'
+], function(
+    Format0
+) {
+
+    return {
+        'Basic CLI': Format0
+    };
+});

src/plugins/Export/format.js.ejs

@@ -0,0 +1,21 @@
+<% // Add default format
+formats.unshift({ name: 'cli', main: 'cli.js', displayName: 'Basic CLI' })
+%>
+/* globals define*/
+// The supported export formats and metadata
+define([
+<%= formats.map(function(format) {
+    return '    \'./formats/' + format.name + '/' +
+        path.basename(format.main.replace(/\.js$/, '')) + '\''
+    })
+    .join(',\n') %>
+], function(
+<%= formats.map(function(f, index) { return '    Format' + index; }).join(',\n') %>
+) {
+
+    return {
+<%= formats.map(function(f, index) {
+        return '        \'' + f.displayName + '\': Format' + index;
+    }).join(',\n') %>
+    };
+});

src/plugins/Export/formats/cli/cli.js

@@ -0,0 +1,103 @@
+/*globals define*/
+// Simple torch cli for the given pipeline
+define([
+], function(
+) {
+
+    var TOBOOLEAN = 
+`local function toboolean(str)
+    if str == 'true' then
+        return true
+    elseif str == 'false' then
+        return false
+    end
+end`;
+
+    var CliExporter = {};
+
+    CliExporter.deserializersFromString = function(sections) {
+        var hasBool = false;
+
+        // Add serializers given cli string input
+        Object.keys(this.isInputOp).forEach(id => {
+            var node = this.inputNode[id],
+                base = this.core.getBase(node),
+                type = this.core.getAttribute(base, 'name'),
+                name = this._nameFor[id];
+
+            if (type === 'boolean') {
+                hasBool = true;
+                sections.deserializerFor[name] = 'toboolean';
+            } else if (type === 'number') {
+                sections.deserializerFor[name] = 'tonumber';
+            } else if (type === 'string') {
+                sections.deserializerFor[name] = 'tostring';
+            }
+        });
+
+        if (hasBool) {
+            sections.deserializers += '\n' + TOBOOLEAN;
+        }
+
+        return sections;
+    };
+
+    CliExporter.main = function (sections, staticInputs) {
+        var code = [];
+
+        // Update deserializers for cli input
+        this.deserializersFromString(sections);
+
+        // Define all the operations, pipelines, etc
+        // 'getAllDefinitions' is provided as part of the public api
+        code.push(this.getAllDefinitions(sections));
+
+        // Command line specific stuff
+        var files = {},
+            main,
+            args,
+            staticNames = staticInputs.map(input => input.name),
+            varDefs,
+            index = 1;
+
+        // Create some names for the inputs
+        args = sections.pipelineInputNames.map(name => `${sections.deserializerFor[name]}(${name})`);
+
+        main = `local outputs = ${sections.pipelineName}(${args.join(', ')})`;
+
+        // Grab the args from the cli
+        code.push(sections.pipelineInputNames.map((name, index) => {
+            return `local ${name} = arg[${index + 1}]`;
+        }).join('\n'));
+
+        // Add the hash for each of the static inputs and reference them
+        staticInputs.forEach(input => {
+            files[`res/${input.name}`] = input.hash;
+        });
+
+        varDefs = staticNames.map(name => {
+            return `local ${name} = './res/${name}'`;
+        });
+
+        // Grab the remaining args from the cli
+        varDefs = varDefs.concat(sections.pipelineInputNames.map(name => {
+            if (!staticNames.includes(name)) {
+                return `local ${name} = arg[${index++}]`;
+            }
+        }));
+
+        // Add the main fn
+        code.push(varDefs.join('\n'));
+        code.push(main);
+
+        // Save outputs to disk
+        code.push(sections.serializeOutputs);
+
+        files['init.lua'] = code.join('\n\n');
+
+        // if no extra assets, just return the main file
+        return staticInputs.length ? files : files['init.lua'];
+    };
+
+    return CliExporter;
+});

src/plugins/Export/metadata.json

@@ -1,7 +1,7 @@
 {
-  "id": "GenerateExecFile",
-  "name": "Generate Execution File",
-  "version": "0.1.0",
+  "id": "Export",
+  "name": "Export",
+  "version": "1.0.0",
   "description": "",
   "icon": {
     "class": "glyphicon glyphicon-cog",

src/plugins/GenerateArchitecture/GenerateArchitecture.js

@@ -49,6 +49,11 @@ define([
         this.LayerDict = createLayerDict(this.core, this.META);
         this.uniqueId = 2;
         this.varnames = {net: true};
+        this.definitions = [
+            'require \'nn\'',
+            'require \'rnn\''
+        ];
+
         return PluginBase.prototype.main.apply(this, arguments);
     };
 
@@ -71,11 +76,6 @@ define([
             result = {},
             code = '';
 
-        this.definitions = [
-            'require \'nn\'',
-            'require \'rnn\''
-        ];
-
         // Add an index to each layer
         layers.forEach((l, index) => l[INDEX] = index);
 

src/plugins/ValidateArchitecture/ValidateArchitecture.js

@@ -0,0 +1,194 @@
+/*globals define*/
+/*jshint node:true, browser:true*/
+
+define([
+    'plugin/GenerateArchitecture/GenerateArchitecture/GenerateArchitecture',
+    'SimpleNodes/Constants',
+    'text!./metadata.json',
+    'q',
+    'fs',
+    'path',
+    'child_process',
+    'rimraf'
+], function (
+    PluginBase,
+    SimpleNodeConstants,
+    pluginMetadata,
+    Q,
+    fs,
+    path,
+    childProcess,
+    rm_rf
+) {
+    'use strict';
+
+    pluginMetadata = JSON.parse(pluginMetadata);
+
+    /**
+     * Initializes a new instance of ValidateArchitecture.
+     * @class
+     * @augments {PluginBase}
+     * @classdesc This class represents the plugin ValidateArchitecture.
+     * @constructor
+     */
+    var TMP_DIR = '/tmp',
+        spawn = childProcess.spawn,
+        GET_ARG_INDEX = /argument #([0-9]+) to/,
+        TORCH_INSTALLED = true;
+
+    var ValidateArchitecture = function () {
+        // Call base class' constructor.
+        PluginBase.call(this);
+        this.pluginMetadata = pluginMetadata;
+    };
+
+    /**
+     * Metadata associated with the plugin. Contains id, name, version, description, icon, configStructue etc.
+     * This is also available at the instance at this.pluginMetadata.
+     * @type {object}
+     */
+    ValidateArchitecture.metadata = pluginMetadata;
+
+    // Prototypical inheritance from PluginBase.
+    ValidateArchitecture.prototype = Object.create(PluginBase.prototype);
+    ValidateArchitecture.prototype.constructor = ValidateArchitecture;
+
+    ValidateArchitecture.prototype.main = function (callback) {
+        var name = this.core.getAttribute(this.activeNode, 'name');
+
+        this._callback = callback;
+        // make the tmp dir
+        this._tmpFileId = path.join(TMP_DIR, `${name}_${Date.now()}`);
+        fs.mkdir(this._tmpFileId, err => {
+            if (err) throw err;
+            return PluginBase.prototype.main.call(this, callback);
+        });
+    };
+
+    ValidateArchitecture.prototype.createOutputFiles = function (tree) {
+        var layers = tree[SimpleNodeConstants.CHILDREN],
+            tests = [],
+            id;
+
+        if (!TORCH_INSTALLED) {
+            return this.validationFinished();
+        }
+
+        // Generate code for each layer
+        this.layerName = {};
+        for (var i = layers.length; i--;) {
+            id = layers[i][SimpleNodeConstants.NODE_PATH];
+            this.layerName[id] = layers[i].name;
+            tests.push([id, this.createLayerTestCode(layers[i])]);
+        }
+
+        // Run each code snippet
+        this.validateLayers(tests)
+            .then(errors => this.validationFinished(errors))
+            .fail(err => this.logger.error(`validation failed: ${err}`));
+    };
+
+    ValidateArchitecture.prototype.validationFinished = function (errors) {
+        if (!TORCH_INSTALLED) {
+            this.logger.warn('Torch is not installed. Architecture validation is not supported.');
+        } else {
+            this.logger.info(`found ${errors.length} validation errors`);
+        }
+
+        this.createMessage(null, {
+            errors: TORCH_INSTALLED ? errors : null
+        });
+        this.result.setSuccess(true);
+        this._callback(null, this.result);
+    };
+
+    ValidateArchitecture.prototype.createLayerTestCode = function (layer) {
+        var customLayerDefs = this.genLayerDefinitions([layer]);
+
+        return this.definitions.concat([
+            customLayerDefs,
+            this.createLayer(layer)
+        ]).join('\n');
+    };
+
+    ValidateArchitecture.prototype.validateLayers = function (layerTests) {
+        return Q.all(layerTests.map(layer => this.validateLayer(layer[0], layer[1])))
+            .then(results => Q.nfcall(rm_rf, this._tmpFileId)
+                .then(() => results.filter(result => !!result))
+            );
+    };
+
+    ValidateArchitecture.prototype.validateLayer = function (id, code) {
+        var deferred = Q.defer(),
+            tmpPath = path.join(this._tmpFileId, id.replace(/[^a-zA-Z\d]+/g, '_'));
+
+        if (!TORCH_INSTALLED) {
+            deferred.resolve(null);
+        } else {
+            // Write to a temp file
+            fs.writeFile(tmpPath, code, err => {
+                var job,
+                    stderr = '',
+                    stdout = '';
+
+                if (err) {
+                    return deferred.reject(`Could not create tmp file at ${tmpPath}: ${err}`);
+                }
+                // Run the file
+                job = spawn('th', [tmpPath]);
+                job.stderr.on('data', data => stderr += data.toString());
+                job.stdout.on('data', data => stdout += data.toString());
+                job.on('error', err => {
+                    if (err.code === 'ENOENT') {
+                        TORCH_INSTALLED = false;
+                    }
+                });
+                job.on('close', code => {
+                    if (code === 0) {
+                        deferred.resolve(null);
+                    } else {
+                        // If it errored, clean the error and return it
+                        deferred.resolve(this.parseError(id, stderr));
+                    }
+                });
+            });
+        }
+
+        return deferred.promise;
+    };
+
+    ValidateArchitecture.prototype.parseError = function (id, stderr) {
+        var msg = stderr
+            .split('\n').shift()  // first line
+            .replace(/^[^:]*: /, '')  // remove the file path
+            .replace(/ at [^ ]*\)/, ')')  // remove last line number
+            .replace(/ to '\?'/, '');  // remove unknown symbol
+
+        // convert 'bad argument #[num]' to the argument name
+        if (msg.indexOf('bad argument') === 0) {
+            var layerName = this.layerName[id],
+                args = this.LayerDict[layerName].args,
+                argIndex = +(stderr.match(GET_ARG_INDEX)[1]),
+                argName = args[argIndex-1].name;
+
+            // FIXME: This is not the correct index...
+            // This is the index for the incorrect argument passed to the
+            // tensor...
+            msg = msg.replace(`#${argIndex}`, `"${argName}"`);
+        }
+
+        return {
+            id: id,
+            msg: msg
+        };
+    };
+
+    ValidateArchitecture.prototype._saveOutput = function () {};
+
+    // for testing
+    ValidateArchitecture.prototype.setTorchInstalled = function (value) {
+        TORCH_INSTALLED = !!value;
+    };
+
+    return ValidateArchitecture;
+});

src/plugins/ValidateArchitecture/metadata.json

@@ -0,0 +1,14 @@
+{
+  "id": "ValidateArchitecture",
+  "name": "ValidateArchitecture",
+  "version": "0.1.0",
+  "description": "",
+  "icon": {
+    "class": "glyphicon glyphicon-cog",
+    "src": ""
+  },
+  "disableServerSideExecution": false,
+  "disableBrowserSideExecution": true,
+  "writeAccessRequired": false,
+  "configStructure": []
+}

src/visualizers/panels/ArchEditor/ArchEditorControl.js

@@ -7,14 +7,16 @@ define([
     'deepforge/viz/panels/ThumbnailControl',
     'js/NodePropertyNames',
     'js/Utils/ComponentSettings',
-    'underscore'
+    'underscore',
+    'q'
 ], function (
     Constants,
     DeepForge,
     ThumbnailControl,
     nodePropertyNames,
     ComponentSettings,
-    _
+    _,
+    Q
 ) {
 
     'use strict';
@@ -36,6 +38,7 @@ define([
         ThumbnailControl.call(this, options);
         this._config = DEFAULT_CONFIG;
         ComponentSettings.resolveWithWebGMEGlobal(this._config, this.getComponentId());
+        this.validateLayers = _.debounce(() => this.validateArchitecture(), 500);
     };
 
     _.extend(ArchEditorControl.prototype, ThumbnailControl.prototype);
@@ -248,5 +251,28 @@ define([
         this._client.completeTransaction();
     };
 
+    ArchEditorControl.prototype._eventCallback = function() {
+        ThumbnailControl.prototype._eventCallback.apply(this, arguments);
+        this.validateLayers();
+    };
+
+    ArchEditorControl.prototype.validateArchitecture = function() {
+        var pluginId = 'ValidateArchitecture',
+            context = this._client.getCurrentPluginContext(pluginId);
+
+        this._logger.info('about to validate arch');
+        // Run the plugin in the browser (set namespace)
+        context.managerConfig.namespace = 'nn';
+        context.pluginConfig = {};
+        Q.ninvoke(this._client, 'runServerPlugin', pluginId, context)
+            .then(res => {
+                var results = res.messages[0].message;
+                if (results.errors !== null) {
+                    this._widget.displayErrors(results.errors);
+                }
+            })
+            .fail(err => this._logger.warn(`Validation failed: ${err}`));
+    };
+
     return ArchEditorControl;
 });

src/visualizers/panels/ArchIndex/ArchIndexControl.js

@@ -0,0 +1,32 @@
+/* globals define */
+define([
+    'panels/PipelineIndex/PipelineIndexControl'
+], function(
+    PipelineIndexControl
+) {
+    var ArchIndexControl = function() {
+        PipelineIndexControl.apply(this, arguments);
+    };
+
+    ArchIndexControl.prototype = Object.create(PipelineIndexControl.prototype);
+    ArchIndexControl.prototype._getObjectDescriptor = function (nodeId) {
+        var node = this._client.getNode(nodeId),
+            base,
+            desc;
+
+        if (node) {
+            base = this._client.getNode(node.getBaseId());
+            desc = {
+                id: node.getId(),
+                name: node.getAttribute('name'),
+                parentId: node.getParentId(),
+                thumbnail: node.getAttribute('thumbnail'),
+                type: base.getAttribute('name')
+            };
+        }
+
+        return desc;
+    };
+
+    return ArchIndexControl;
+});

src/visualizers/panels/ArchIndex/ArchIndexPanel.js

@@ -5,7 +5,7 @@ define([
     'js/PanelBase/PanelBaseWithHeader',
     'js/PanelManager/IActivePanel',
     'widgets/ArchIndex/ArchIndexWidget',
-    'panels/PipelineIndex/PipelineIndexControl'
+    'panels/ArchIndex/ArchIndexControl'
 ], function (
     PanelBaseWithHeader,
     IActivePanel,

src/visualizers/panels/ArtifactIndex/ArtifactIndexControl.js

@@ -80,16 +80,16 @@ define([
     // This next function retrieves the relevant node information for the widget
     ArtifactIndexControl.prototype._getObjectDescriptor = function (nodeId) {
         var node = this._client.getNode(nodeId),
-            base,
+            type,
             hash,
             objDescriptor;
 
         if (node) {
-            base = this._client.getNode(node.getBaseId());
+            type = this._client.getNode(node.getMetaTypeId());
             hash = node.getAttribute('data');
             objDescriptor = {
                 id: node.getId(),
-                type: base ? base.getAttribute('name') : 'n/a',
+                type: type ? type.getAttribute('name') : 'n/a',
                 name: node.getAttribute('name'),
                 createdAt: node.getAttribute('createdAt'),
                 dataURL: this.blobClient.getDownloadURL(hash),

src/visualizers/panels/ForgeActionButton/Actions.js

@@ -5,7 +5,8 @@ define([
     'panel/FloatingActionButton/styles/Materialize',
     'q',
     'js/RegistryKeys',
-    'deepforge/globals'
+    'deepforge/globals',
+    'deepforge/Constants'
 ], function(
     Materialize,
     Q,
@@ -266,29 +267,23 @@ define([
                 }
             },
             {
-                name: 'Export for local execution',
-                icon: 'play_for_work',
+                name: 'Export Pipeline',
+                icon: 'launch',
                 priority: -1,
                 action: function() {
-                    var pluginId = 'GenerateExecFile',
-                        context = this.client.getCurrentPluginContext(pluginId);
-
-                    // Run the plugin in the browser (set namespace)
-                    context.managerConfig.namespace = 'pipeline';
-                    context.pluginConfig = {};
-                    Q.ninvoke(this.client, 'runBrowserPlugin', pluginId, context)
-                        .then(res => {
-                            var id = this._currentNodeId,
-                                node = this.client.getNode(id),
-                                base = this.client.getNode(node.getBaseId()),
-                                type = base.getAttribute('name'),
-                                name = node.getAttribute('name');
-
-                            // Get the file and download it
-                            this.downloadFromBlob(res.artifacts[0]);
-                            Materialize.toast(`Exported ${name} ${type}!`, 2000);
+                    this.exportPipeline()
+                        .then(result => {
+                            Materialize.toast('Export successful!', 2000);
+                            // Download the result!
+                            this.downloadFromBlob(result.artifacts[0]);
+                            result.__unread = true;
+                            this.results.push(result);
+                            this._updatePluginBtns();
                         })
-                        .fail(err => Materialize.toast(`Export failed: ${err}`, 2000));
+                        .fail(err => {
+                            this.logger.warn('Pipeline export failed:', err);
+                            Materialize.toast(`Export failed: ${err}`, 4000);
+                        });
                 }
             }
         ],

src/visualizers/panels/ForgeActionButton/ConfigDialog.css

@@ -0,0 +1,5 @@
+.config-section-header {
+  margin-top: 0;
+  color: #888;
+  font-style: italic;
+}

src/visualizers/panels/ForgeActionButton/ConfigDialog.js

@@ -0,0 +1,222 @@
+/* globals define, $*/
+define([
+    'js/Dialogs/PluginConfig/PluginConfigDialog',
+    'text!js/Dialogs/PluginConfig/templates/PluginConfigDialog.html',
+    'plugin/Export/Export/format',
+    'css!./ConfigDialog.css'
+], function(
+    PluginConfigDialog,
+    pluginConfigDialogTemplate,
+    ExportFormats
+) {
+    var SECTION_DATA_KEY = 'section',
+        ATTRIBUTE_DATA_KEY = 'attribute',
+    //jscs:disable maximumLineLength
+        PLUGIN_CONFIG_SECTION_BASE = $('<div><fieldset><form class="form-horizontal" role="form"></form><fieldset></div>'),
+        ENTRY_BASE = $('<div class="form-group"><div class="row"><label class="col-sm-4 control-label">NAME</label><div class="col-sm-8 controls"></div></div><div class="row description"><div class="col-sm-4"></div></div></div>'),
+    //jscs:enable maximumLineLength
+        DESCRIPTION_BASE = $('<div class="desc muted col-sm-8"></div>'),
+        SECTION_HEADER = $('<h6 class="config-section-header">');
+
+    var ConfigDialog = function(client, nodeId) {
+        PluginConfigDialog.call(this, {client: client});
+        this._widgets = {};
+        this._node = this._client.getNode(nodeId);
+    };
+
+    ConfigDialog.prototype = Object.create(PluginConfigDialog.prototype);
+
+    ConfigDialog.prototype.getFormatOptions = function() {
+        this._exportFormats = Object.keys(ExportFormats);
+
+        this._formatOptions = [];
+        if (this._exportFormats.length > 1) {
+            this._formatOptions.push({  // format options
+                name: 'exportFormat',
+                displayName: 'Export Format',
+                valueType: 'string',
+                value: this._exportFormats[0],
+                valueItems: this._exportFormats,
+                readOnly: false
+            });
+        }
+    };
+
+    ConfigDialog.prototype.show = function(pluginMetadata, callback) {
+        this._pluginMetadata = pluginMetadata;
+
+        this.getFormatOptions();
+        this._initDialog();
+
+        this._dialog.on('shown', () => {
+            this._dialog.find('input').first().focus();
+        });
+
+        this._btnSave.on('click', event => {
+            this.submit(callback);
+            event.stopPropagation();
+            event.preventDefault();
+        });
+
+        //save&run on CTRL + Enter
+        this._dialog.on('keydown.PluginConfigDialog', event => {
+            if (event.keyCode === 13 && (event.ctrlKey || event.metaKey)) {
+                event.stopPropagation();
+                event.preventDefault();
+                this.submit(callback);
+            }
+        });
+        this._dialog.modal('show');
+    };
+
+    ConfigDialog.prototype._initDialog = function() {
+        this._dialog = $(pluginConfigDialogTemplate);
+
+        this._btnSave = this._dialog.find('.btn-save');
+        this._divContainer = this._dialog.find('.modal-body');
+        this._saveConfigurationCb = this._dialog.find('.save-configuration');
+        this._modalHeader = this._dialog.find('.modal-header');
+
+        // Create the header
+        var iconEl = $('<i/>', {
+            class: this._pluginMetadata.icon.class || 'glyphicon glyphicon-cog'
+        });
+        iconEl.addClass('plugin-icon pull-left');
+        this._modalHeader.prepend(iconEl);
+        this._title = this._modalHeader.find('.modal-title');
+        this._title.text(this._pluginMetadata.id + ' ' + 'v' + this._pluginMetadata.version);
+
+        // Generate the config options
+        var sectionHeader = SECTION_HEADER.clone();
+
+        sectionHeader.text('Static Artifacts');
+        this._divContainer.append(sectionHeader);
+        this.generateConfigSection(this._pluginMetadata);
+
+        if (this._exportFormats.length > 1) {
+            this._divContainer.append($('<hr class="extension-config-divider">'));
+            sectionHeader = SECTION_HEADER.clone();
+            sectionHeader.text('Export Options');
+            this._divContainer.append(sectionHeader);
+
+            this.generateConfigSection({
+                id: 'FormatOptions',
+                configStructure: this._formatOptions
+            });
+            this._widgets.FormatOptions.exportFormat.el.find('select').on('change', event => {
+                var format = event.target.value;
+                // Update the ext config
+                this.updateExtConfig(format);
+            });
+        }
+
+        this.updateExtConfig(this._exportFormats[0]);
+    };
+
+    ConfigDialog.prototype.submit = function (callback) {
+        var config = this._getAllConfigValues();
+        this._dialog.modal('hide');
+
+        if (this._exportFormats.length === 1) {
+            config.FormatOptions = {
+                exportFormat: this._exportFormats[0]
+            };
+        }
+        return callback(config);
+    };
+
+    ConfigDialog.prototype._getAllConfigValues = function () {
+        var settings = {};
+
+        Object.keys(this._widgets).forEach(namespace => {
+            settings[namespace] = {};
+
+            Object.keys(this._widgets[namespace]).forEach(name => {
+                settings[namespace][name] = this._widgets[namespace][name].getValue();
+            });
+        });
+
+        return settings;
+    };
+
+    ConfigDialog.prototype.updateExtConfig = function (format) {
+        var extConfig = {
+            id: 'extensionConfig',
+            class: 'extension-config',
+            configStructure: ExportFormats[format].getConfigStructure ?
+                ExportFormats[format].getConfigStructure(this._node, this._client) : []
+        };
+        this._divContainer.find('.extension-config').remove();
+
+        if (extConfig.configStructure.length) {
+            this.generateConfigSection(extConfig);
+        }
+    };
+
+    ConfigDialog.prototype.generateConfigSection = function (metadata) {
+        var len = metadata.configStructure.length,
+            i,
+            el,
+            pluginConfigEntry,
+            widget,
+            descEl,
+            containerEl,
+            pluginSectionEl = PLUGIN_CONFIG_SECTION_BASE.clone();
+
+        pluginSectionEl.data(SECTION_DATA_KEY, metadata.id);
+        this._divContainer.append(pluginSectionEl);
+        containerEl = pluginSectionEl.find('.form-horizontal');
+
+        if (metadata.class) {
+            pluginSectionEl.addClass(metadata.class);
+        }
+
+        this._widgets[metadata.id] = {};
+        for (i = 0; i < len; i += 1) {
+            pluginConfigEntry = metadata.configStructure[i];
+            descEl = undefined;
+
+            // Make sure not modify the global metadata.
+            pluginConfigEntry = JSON.parse(JSON.stringify(pluginConfigEntry));
+            if (this._client.getProjectAccess().write === false && pluginConfigEntry.writeAccessRequired === true) {
+                pluginConfigEntry.readOnly = true;
+            }
+
+            widget = this._propertyGridWidgetManager.getWidgetForProperty(pluginConfigEntry);
+            this._widgets[metadata.id][pluginConfigEntry.name] = widget;
+
+            el = ENTRY_BASE.clone();
+            el.data(ATTRIBUTE_DATA_KEY, pluginConfigEntry.name);
+
+            el.find('label.control-label').text(pluginConfigEntry.displayName);
+
+            if (pluginConfigEntry.description && pluginConfigEntry.description !== '') {
+                descEl = descEl || DESCRIPTION_BASE.clone();
+                descEl.text(pluginConfigEntry.description);
+            }
+
+            if (pluginConfigEntry.minValue !== undefined &&
+                pluginConfigEntry.minValue !== null &&
+                pluginConfigEntry.minValue !== '') {
+                descEl = descEl || DESCRIPTION_BASE.clone();
+                descEl.append(' The minimum value is: ' + pluginConfigEntry.minValue + '.');
+            }
+
+            if (pluginConfigEntry.maxValue !== undefined &&
+                pluginConfigEntry.maxValue !== null &&
+                pluginConfigEntry.maxValue !== '') {
+                descEl = descEl || DESCRIPTION_BASE.clone();
+                descEl.append(' The maximum value is: ' + pluginConfigEntry.maxValue + '.');
+            }
+
+            el.find('.controls').append(widget.el);
+            if (descEl) {
+                el.find('.description').append(descEl);
+            }
+
+            containerEl.append(el);
+        }
+    };
+
+    return ConfigDialog;
+});

src/visualizers/panels/ForgeActionButton/ForgeActionButton.js

@@ -1,9 +1,10 @@
-/*globals $, window, define, _ */
+/*globals $, window, define, _, WebGMEGlobal */
 /*jshint browser: true*/
 
 define([
     'blob/BlobClient',
     'js/Utils/SaveToDisk',
+    './ConfigDialog',
     'js/Constants',
     'panel/FloatingActionButton/FloatingActionButton',
     'deepforge/viz/PipelineControl',
@@ -14,11 +15,14 @@ define([
     'js/RegistryKeys',
     'js/Panels/MetaEditor/MetaEditorConstants',
     'q',
-    'deepforge/globals'
+    'deepforge/globals',
+    'deepforge/Constants',
+    'plugin/Export/Export/format'
 ], function (
     BlobClient,
     SaveToDisk,
-    CONSTANTS,
+    ConfigDialog,
+    GME_CONSTANTS,
     PluginButton,
     PipelineControl,
     NodePrompter,
@@ -28,7 +32,9 @@ define([
     REGISTRY_KEYS,
     META_CONSTANTS,
     Q,
-    DeepForge
+    DeepForge,
+    Constants,
+    ExportFormatDict
 ) {
     'use strict';
 
@@ -142,19 +148,19 @@ define([
 
     // Helper functions REMOVE! FIXME
     ForgeActionButton.prototype.addToMetaSheet = function(nodeId, metasheetName) {
-        var root = this.client.getNode(CONSTANTS.PROJECT_ROOT_ID),
+        var root = this.client.getNode(GME_CONSTANTS.PROJECT_ROOT_ID),
             metatabs = root.getRegistry(REGISTRY_KEYS.META_SHEETS),
             metatab = metatabs.find(tab => tab.title === metasheetName) || metatabs[0],
             metatabId = metatab.SetID;
 
         // Add to the general meta
         this.client.addMember(
-            CONSTANTS.PROJECT_ROOT_ID,
+            GME_CONSTANTS.PROJECT_ROOT_ID,
             nodeId,
             META_CONSTANTS.META_ASPECT_SET_NAME
         );
         this.client.setMemberRegistry(
-            CONSTANTS.PROJECT_ROOT_ID,
+            GME_CONSTANTS.PROJECT_ROOT_ID,
             nodeId,
             META_CONSTANTS.META_ASPECT_SET_NAME,
             REGISTRY_KEYS.POSITION,
@@ -165,9 +171,9 @@ define([
         );
 
         // Add to the specific sheet
-        this.client.addMember(CONSTANTS.PROJECT_ROOT_ID, nodeId, metatabId);
+        this.client.addMember(GME_CONSTANTS.PROJECT_ROOT_ID, nodeId, metatabId);
         this.client.setMemberRegistry(
-            CONSTANTS.PROJECT_ROOT_ID,
+            GME_CONSTANTS.PROJECT_ROOT_ID,
             nodeId,
             metatabId,
             REGISTRY_KEYS.POSITION,
@@ -363,18 +369,131 @@ define([
     };
 
     ForgeActionButton.prototype.downloadFromBlob = function(hash) {
-        var name;
-
         this._blobClient.getMetadata(hash)
             .then(metadata => {
-                name = metadata.name;
-                return this._blobClient.getObjectAsString(hash);
-            })
-            .then(text => {
-                SaveToDisk.downloadTextAsFile(name, text);
+                var url = this._blobClient.getDownloadURL(hash),
+                    name = metadata.name,
+                    save = document.createElement('a');
+
+                save.href = url;
+                save.target = '_self';
+                save.download = name;
+
+                save.click();
+                (window.URL || window.webkitURL).revokeObjectURL(save.href);
             })
             .fail(err => this.logger.error(`Blob download failed: ${err}`));
     };
 
+    /// Export Pipeline Support
+    ForgeActionButton.prototype.exportPipeline = function() {
+        var deferred = Q.defer(),
+            pluginId = 'Export',
+            metadata = WebGMEGlobal.allPluginsMetadata[pluginId],
+            id = this._currentNodeId,
+            node = this.client.getNode(id),
+            inputData,
+            inputNames;
+
+        inputData = node.getChildrenIds()
+            .map(id => this.client.getNode(id))
+            .filter(node => {
+                var typeId = node.getMetaTypeId(),
+                    type = this.client.getNode(typeId).getAttribute('name');
+
+                return type === Constants.OP.INPUT;
+            })
+            .map(input => {
+                var outputCntr,
+                    outputIds;
+
+                outputCntr = input.getChildrenIds()
+                    .map(id => this.client.getNode(id))
+                    .find(node => {
+                        var typeId = node.getMetaTypeId(),
+                            type = this.client.getNode(typeId).getAttribute('name');
+                        return type === 'Outputs';
+                    });
+
+                // input operations only have a single output
+                outputIds = outputCntr.getChildrenIds();
+
+                if (outputIds.length === 1) {
+                    return outputIds[0];
+                } else if (outputIds.length > 1) {
+                    this.logger.warn(`Found multiple ids for input op: ${outputIds.join(', ')}`);
+                    return;
+                }
+            })
+            .filter(outputId => !!outputId)
+            .map(id => this.client.getNode(id))
+            .filter(output => output.getAttribute('data'));
+
+        // get the name of node referenced from the input op
+        inputNames = inputData
+            .map(node => {
+                var cntrId = node.getParentId(),
+                    opId = this._client.getNode(cntrId).getParentId(),
+                    inputOp = this._client.getNode(opId),
+                    targetNodeId = inputOp.getPointer('artifact').to;
+
+                return this._client.getNode(targetNodeId).getAttribute('name');
+            })
+            .sort();
+
+        // create config options from inputs
+        var inputOpts = inputNames.map((input, index) => {
+            return {
+                name: inputData[index].getId(),
+                displayName: input,
+                description: `Export ${input} as static (non-input) content`,
+                value: false,
+                valueType: 'boolean',
+                readOnly: false
+            };
+        });
+
+        var exportFormats = Object.keys(ExportFormatDict),
+            configDialog = new ConfigDialog(this.client, this._currentNodeId),
+            inputConfig = _.extend({}, metadata);
+
+        inputConfig.configStructure = inputOpts;
+
+        // Try to get the extension options
+        if (inputOpts.length || exportFormats.length > 1) {
+            configDialog.show(inputConfig, (allConfigs) => {
+                var context = this.client.getCurrentPluginContext(pluginId),
+                    exportFormat = allConfigs.FormatOptions.exportFormat,
+                    staticInputs = Object.keys(allConfigs[pluginId]).filter(input => allConfigs[pluginId][input]);
+
+                this.logger.debug('Exporting pipeline to format', exportFormat);
+                this.logger.debug('static inputs:', staticInputs);
+
+                context.managerConfig.namespace = 'pipeline';
+                context.pluginConfig = {
+                    format: exportFormat,
+                    staticInputs: staticInputs,
+                    extensionConfig: allConfigs.extensionConfig
+                };
+                return Q.ninvoke(this.client, 'runBrowserPlugin', pluginId, context)
+                    .then(deferred.resolve)
+                    .fail(deferred.reject);
+            });
+        } else {  // no options - just run the plugin!
+            var context = this.client.getCurrentPluginContext(pluginId);
+
+            this.logger.debug('Exporting pipeline to format', exportFormats[0]);
+
+            context.managerConfig.namespace = 'pipeline';
+            context.pluginConfig = {
+                format: exportFormats[0],
+                staticInputs: []
+            };
+            return Q.ninvoke(this.client, 'runBrowserPlugin', pluginId, context);
+        }
+
+        return deferred.promise;
+    };
+
     return ForgeActionButton;
 });

src/visualizers/panels/PipelineIndex/PipelineIndexControl.js

@@ -103,11 +103,11 @@ define([
     PipelineIndexControl.prototype._getObjectDescriptor = function (nodeId) {
         var node = this._client.getNode(nodeId),
             base,
-            objDescriptor;
+            desc;
 
         if (node) {
             base = this._client.getNode(node.getBaseId());
-            objDescriptor = {
+            desc = {
                 id: node.getId(),
                 name: node.getAttribute(nodePropertyNames.Attributes.name),
                 parentId: node.getParentId(),
@@ -115,10 +115,9 @@ define([
                 type: base.getAttribute('name'),
                 executionCount: node.getMemberIds('executions').length
             };
-
         }
 
-        return objDescriptor;
+        return desc;
     };
 
     /* * * * * * * * Node Event Handling * * * * * * * */

src/visualizers/widgets/ArchEditor/ArchEditorWidget.js

@@ -42,6 +42,7 @@ define([
         }
         ThumbnailWidget.apply(this, arguments);
         this._emptyMsg = 'Click to add a new layer';
+        this.hasError = {};
     };
 
     _.extend(ArchEditorWidget.prototype, ThumbnailWidget.prototype);
@@ -223,5 +224,28 @@ define([
             .then(selected => this.insertLayer(selected.node.id, item.id));
     };
 
+    ArchEditorWidget.prototype.displayErrors = function(errors) {
+        // For each of the errors, highlight the given nodes
+        var oldErrored = Object.keys(this.hasError),
+            currentErrored = errors.map(err => err.id),
+            newErrored = _.difference(currentErrored, oldErrored),
+            fixedLayers = _.difference(oldErrored, currentErrored);
+
+        this._logger.info('updating displayed errors to', currentErrored);
+        this.hasError = {};
+        newErrored.forEach(id => {
+            if (this.items[id]) {
+                this.items[id].decorator.highlight('red');
+                this.hasError[id] = true;
+            }
+        });
+
+        fixedLayers.forEach(id => {
+            if (this.items[id]) {
+                this.items[id].clear();
+            }
+        });
+    };
+
     return ArchEditorWidget;
 });

src/visualizers/widgets/OperationInterfaceEditor/Buttons.js

@@ -86,6 +86,7 @@ define([
         } else {
             this.deleteNode(item.id);
         }
+        this.selectionManager.deselect();
     };
 
     return {

src/visualizers/widgets/PipelineEditor/OperationNode.js

@@ -122,8 +122,10 @@ define([
         this.hidePorts();
     };
 
-    OperationNode.prototype.update = function() {
+    OperationNode.prototype.update = function(desc) {
         DAGItem.prototype.update.apply(this, arguments);
+        this.inputs = desc.inputs;
+        this.outputs = desc.outputs;
         if (this._visiblePorts) {
             var areInputs = this._visiblePorts[1];
             this.showPorts.call(this, null, areInputs);

src/visualizers/widgets/PipelineEditor/PipelineEditorWidget.js

@@ -295,7 +295,6 @@ define([
             this.$execContent.empty();
             execs.forEach(html => this.$execContent.append(html));
 
-            this.$execContent.height(200);
             this.$execBody.show();
         } else {
             // Set the height to 0

src/visualizers/widgets/PipelineIndex/PipelineIndexWidget.js

@@ -145,7 +145,9 @@ define([
 
     PipelineIndexWidget.prototype.updateNode = function (desc) {
         if (desc && this.cards[desc.id]) {
-            this.cards[desc.id].outerHTML = PipelineTemplate(desc);
+            var Template = this.getCardTemplate(desc);
+
+            this.cards[desc.id].outerHTML = Template(desc);
             // Check if the preview changed
             if (desc.thumbnail !== this.nodes[desc.id].thumbnail) {
                 this.addThumbnail(desc.thumbnail, this.cards[desc.id]);

test/cli/cli.spec.js

@@ -1,58 +1,59 @@
-var mockery = require('mockery'),
-    assert = require('assert'),
-    path = require('path'),
-    nop = () => {},
-    cli;
+describe('cli', function() {
+    var mockery = require('mockery'),
+        assert = require('assert'),
+        path = require('path'),
+        nop = () => {},
+        cli;
 
-var callRegister = {
-    childProcess: {
-        execSync: []
-    }
-};
-
-var mocks = {
-    childProcess: {},
-    rimraf: {}
-};
-
-var childProcess = {
-    execSync: function(cmd) {
-        callRegister.childProcess.execSync.push(cmd);
-        if (mocks.childProcess.execSync) {
-            return mocks.childProcess.execSync.apply(this, arguments);
+    var callRegister = {
+        childProcess: {
+            execSync: []
         }
-    },
-    spawnSync: function(cmd) {
-        if (cmd === 'luarocks') {
+    };
+
+    var mocks = {
+        childProcess: {},
+        rimraf: {}
+    };
+
+    var childProcess = {
+        execSync: function(cmd) {
+            callRegister.childProcess.execSync.push(cmd);
+            if (mocks.childProcess.execSync) {
+                return mocks.childProcess.execSync.apply(this, arguments);
+            }
+        },
+        spawnSync: function(cmd) {
+            if (cmd === 'luarocks') {
+                return {
+                    stdout: 'rnn'
+                };
+            }
+            return {};
+        },
+        spawn: function() {
+            if (mocks.childProcess.spawn) {
+                mocks.childProcess.spawn.apply(this, arguments);
+            }
             return {
-                stdout: 'rnn'
+                on: () => {},
+                stdout: {
+                    on: () => {}
+                },
+                stderr: {
+                    on: () => {}
+                }
             };
         }
-        return {};
-    },
-    spawn: function() {
-        if (mocks.childProcess.spawn) {
-            mocks.childProcess.spawn.apply(this, arguments);
+    };
+    var rimraf = {};
+    rimraf.sync = function() {
+        if (mocks.rimraf.sync) {
+            mocks.rimraf.sync.apply(this, arguments);
         }
-        return {
-            on: () => {},
-            stdout: {
-                on: () => {}
-            },
-            stderr: {
-                on: () => {}
-            }
-        };
-    }
-};
-var rimraf = {};
-rimraf.sync = function() {
-    if (mocks.rimraf.sync) {
-        mocks.rimraf.sync.apply(this, arguments);
-    }
-};
+    };
+
 
-describe('cli', function() {
     before(function() {
         // create the mocks
         mockery.enable({
@@ -180,9 +181,11 @@ describe('cli', function() {
         });
 
         it('should update deepforge from git if --git set w/ npm', function() {
+            var repo = require('../../package.json').repository.url;
             mocks.childProcess.spawn = (cmd, args) => {
+                // check for the git repo
                 if (cmd === 'npm') {
-                    assert.notEqual(args.indexOf('dfst/deepforge'), -1);
+                    assert.notEqual(args.indexOf(repo), -1);
                 }
             };
             cli('update --git');

test/plugins/Export/Export.spec.js

@@ -1,19 +1,20 @@
 /*jshint node:true, mocha:true*/
 
 'use strict';
-describe('GenerateExecFile', function () {
+describe('Export', function () {
     var testFixture = require('../../globals'),
         lua = require('../../../src/common/lua'),
         path = testFixture.path,
+        assert = require('assert'),
         SEED_DIR = path.join(testFixture.DF_SEED_DIR, 'devProject'),
         gmeConfig = testFixture.getGmeConfig(),
         expect = testFixture.expect,
-        logger = testFixture.logger.fork('GenerateExecFile'),
+        logger = testFixture.logger.fork('Export'),
         PluginCliManager = testFixture.WebGME.PluginCliManager,
         manager = new PluginCliManager(null, logger, gmeConfig),
         BlobClient = require('webgme/src/server/middleware/blob/BlobClientWithFSBackend'),
         projectName = 'testProject',
-        pluginName = 'GenerateExecFile',
+        pluginName = 'Export',
         project,
         gmeAuth,
         storage,
@@ -139,19 +140,23 @@ describe('GenerateExecFile', function () {
                         expect(data).to.equal(saveData);
                         done();
                     });
-                    torch.set('class', (name) => {
-                        var cntr = context._G,
-                            classes,
-                            newClass = lua.newContext()._G;
+                    torch.set('class', name => {
+                        try {
+                            var cntr = context._G,
+                                classes,
+                                newClass = lua.newContext()._G;
 
-                        if (name.includes('.')) {
-                            classes = name.split('.');
-                            name = classes.pop();
-                            for (var i = 0; i < classes.length; i++) {
-                                cntr = cntr.get(classes[i]);
+                            if (name.includes('.')) {
+                                classes = name.split('.');
+                                name = classes.pop();
+                                for (var i = 0; i < classes.length; i++) {
+                                    cntr = cntr.get(classes[i]) || cntr;
+                                }
                             }
+                            cntr.set(name, newClass);
+                        } catch(e) {
+                            assert(!e, `Failed defining class ${name}: ${e}`);
                         }
-                        cntr.set(name, newClass);
                         return newClass;
                     });
                     context._G.set('torch', torch);

test/plugins/ValidateArchitecture/ValidateArchitecture.spec.js

@@ -0,0 +1,183 @@
+/*jshint node:true, mocha:true*/
+
+'use strict';
+var testFixture = require('../../globals');
+
+describe('ValidateArchitecture', function () {
+    var gmeConfig = testFixture.getGmeConfig(),
+        expect = testFixture.expect,
+        fs = require('fs'),
+        rm_rf = require('rimraf'),
+        mockery = require('mockery'),
+        logger = testFixture.logger.fork('ValidateArchitecture'),
+        PluginCliManager = testFixture.WebGME.PluginCliManager,
+        manager = new PluginCliManager(null, logger, gmeConfig),
+        projectName = 'testProject',
+        pluginName = 'ValidateArchitecture',
+        project,
+        gmeAuth,
+        storage,
+        commitHash;
+
+    before(function (done) {
+        testFixture.clearDBAndGetGMEAuth(gmeConfig, projectName)
+            .then(function (gmeAuth_) {
+                gmeAuth = gmeAuth_;
+                // This uses in memory storage. Use testFixture.getMongoStorage to persist test to database.
+                storage = testFixture.getMemoryStorage(logger, gmeConfig, gmeAuth);
+                return storage.openDatabase();
+            })
+            .then(function () {
+                var importParam = {
+                    projectSeed: testFixture.path.join(testFixture.DF_SEED_DIR, 'devProject', 'devProject.webgmex'),
+                    projectName: projectName,
+                    branchName: 'master',
+                    logger: logger,
+                    gmeConfig: gmeConfig
+                };
+
+                return testFixture.importProject(storage, importParam);
+            })
+            .then(function (importResult) {
+                project = importResult.project;
+                commitHash = importResult.commitHash;
+                return project.createBranch('test', commitHash);
+            })
+            .nodeify(done);
+    });
+
+    after(function (done) {
+        storage.closeDatabase()
+            .then(function () {
+                return gmeAuth.unload();
+            })
+            .nodeify(done);
+    });
+
+    var plugin,
+        preparePlugin = function(done) {
+            var context = {
+                project: project,
+                commitHash: commitHash,
+                namespace: 'nn',
+                branchName: 'test',
+                activeNode: '/4/n'  // "simple broken" architecture
+            };
+
+            return manager.initializePlugin(pluginName)
+                .then(plugin_ => {
+                    plugin = plugin_;
+                    plugin.setTorchInstalled(true);
+                    return manager.configurePlugin(plugin, {}, context);
+                })
+                .nodeify(done);
+        };
+    // check that each layer is validated
+    describe('simple broken test case', function() {
+        beforeEach(preparePlugin);
+
+        it('should validate each layer', function(done) {
+            var validated = {};
+
+            plugin.validateLayer = id => {
+                validated[id] = true;
+            };
+            plugin.main(err => {
+                expect(err).to.equal(null);
+                expect(Object.keys(validated).length).to.equal(5);
+                done();
+            });
+        });
+
+        it('should make tmp dir', function(done) {
+            var oldMkdir = fs.mkdir;
+            fs.mkdir = (dir, cb) => {
+                expect(dir).to.equal(plugin._tmpFileId);
+                return oldMkdir(dir, cb);
+            };
+            plugin.main(() => {
+                fs.mkdir = oldMkdir;
+                done();
+            });
+        });
+
+        it('should rm tmp dir', function(done) {
+            mockery.enable({
+                warnOnReplace: false,
+                warnOnUnregistered: false
+            });
+            mockery.registerMock('rimraf', (dir, cb) => {
+                expect(dir).to.equal(plugin._tmpFileId);
+                return rm_rf(dir, cb);
+            });
+            plugin.main(() => {
+                mockery.disable();
+                done();
+            });
+        });
+
+        // check that errors are returned in the message
+        it('should return two error messages', function(done) {
+            plugin.validateLayer = (id, code) => {
+                if (code.indexOf('Linear()') === -1) {
+                    return null;
+                } else {  // error!
+                    return {
+                        id: id,
+                        msg: 'invalid args'
+                    };
+                }
+            };
+            plugin.main((err, result) => {
+                var invalidLayers = result.messages[0].message.errors.map(msg => msg.id);
+                expect(result.messages[0]).to.not.equal(undefined);
+                expect(invalidLayers.length).to.equal(2);
+                done();
+            });
+        });
+
+        describe('w/o torch support', function() {
+            before(function(done) {
+                preparePlugin(() => {
+                    done();
+                });
+            });
+
+            it('should return "null" for messages', function(done) {
+                plugin.setTorchInstalled(false);
+                plugin.main((err, result) => {
+                    var errors = result.messages[0].message.errors;
+                    expect(errors).to.equal(null);
+                    done();
+                });
+            });
+        });
+    });
+
+    describe('custom layer test case', function() {
+        beforeEach(done => {
+            var context = {
+                project: project,
+                commitHash: commitHash,
+                namespace: 'nn',
+                branchName: 'test',
+                activeNode: '/4/5'  // "simple broken" architecture
+            };
+
+            return manager.initializePlugin(pluginName)
+                .then(plugin_ => {
+                    plugin = plugin_;
+                    return manager.configurePlugin(plugin, {}, context);
+                })
+                .nodeify(done);
+        });
+
+        it('should add custom layer def to test code', function(done) {
+            plugin.validateLayer = (id, code) => {
+                expect(code.indexOf('torch.class')).to.not.equal(-1);
+                return null;
+            };
+            plugin.main(done);
+        });
+    });
+});

utils/extender.js

@@ -0,0 +1,199 @@
+// Utility for applying and removing deepforge extensions
+// This utility is run by the cli when executing:
+//
+//     deepforge extensions add <project>
+//     deepforge extensions remove <name>
+//
+var path = require('path'),
+    fs = require('fs'),
+    npm = require('npm'),
+    Q = require('q'),
+    rm_rf = require('rimraf'),
+    exists = require('exists-file'),
+    makeTpl = require('lodash.template'),
+    CONFIG_DIR = path.join(process.env.HOME, '.deepforge'),
+    EXT_CONFIG_NAME = 'extension.json',
+    EXTENSION_REGISTRY_NAME = 'extensions.json',
+    extConfigPath = path.join(CONFIG_DIR, EXTENSION_REGISTRY_NAME),
+    allExtConfigs;
+
+var values = obj => Object.keys(obj).map(key => obj[key]);
+
+// Create the extensions.json if doesn't exist. Otherwise, load it
+if (!exists.sync(extConfigPath)) {
+    allExtConfigs = {};
+} else {
+    try {
+        allExtConfigs = JSON.parse(fs.readFileSync(extConfigPath, 'utf8'));
+    } catch (e) {
+        throw `Invalid config at ${extConfigPath}: ${e.toString()}`;
+    }
+}
+
+var persistExtConfig = () => {
+    fs.writeFileSync(extConfigPath, JSON.stringify(allExtConfigs, null, 2));
+};
+
+var extender = {};
+
+extender.EXT_CONFIG_NAME = EXT_CONFIG_NAME;
+
+extender.isSupportedType = function(type) {
+    return extender.install[type] && extender.uninstall[type];
+};
+
+extender.getExtensionsConfig = function() {
+    return allExtConfigs;
+};
+
+extender.getInstalledConfig = function(name) {
+    var group = values(allExtConfigs).find(typeGroup => {
+        return !!typeGroup[name];
+    });
+    return group && group[name];
+};
+
+extender.install = function(project, isReinstall) {
+    // Install the project
+    return Q.ninvoke(npm, 'load', {})
+        .then(() => Q.ninvoke(npm, 'install', project))
+        .then(results => {
+            var installed = results[0],
+                extProject,
+                extRoot;
+
+            extProject = installed[0][0];
+            extRoot = installed[0][1];
+
+            // Check for the extensions.json in the project (look up type, etc)
+            var extConfigPath = path.join(extRoot, extender.EXT_CONFIG_NAME),
+                extConfig,
+                extType;
+
+            // Check that the extensions file exists
+            if (!exists.sync(extConfigPath)) {
+                throw [
+                    `Could not find ${extender.EXT_CONFIG_NAME} for ${project}.`,
+                    '',
+                    `This is likely an issue w/ the deepforge extension (${project})`
+                ].join('\n');
+            }
+
+            try {
+                extConfig = JSON.parse(fs.readFileSync(extConfigPath, 'utf8'));
+            } catch(e) {  // Invalid JSON
+                throw `Invalid ${extender.EXT_CONFIG_NAME}: ${e}`;
+            }
+
+            // Try to add the extension to the project (using the extender)
+            extType = extConfig.type;
+            if (!extender.isSupportedType(extType)) {
+                throw `Unrecognized extension type: "${extType}"`;
+            }
+            extender.install[extType](extConfig, {
+                arg: project,
+                root: extRoot,
+                name: extProject
+            }, !!isReinstall);
+
+            return extConfig;
+        });
+};
+
+extender.uninstall = function(name) {
+    // Look up the extension in ~/.deepforge/extensions.json
+    var extConfig = extender.getInstalledConfig(name);
+    if (!extConfig) {
+        throw `Extension "${name}" not found`;
+    }
+
+    // Run the uninstaller using the extender
+    var extType = extConfig.type;
+    extender.uninstall[extType](name);
+};
+
+var makeInstallFor = function(typeCfg) {
+    var saveExtensions = () => {
+        // regenerate the format.js file from the template
+        var installedExts = values(allExtConfigs[typeCfg.type]),
+            formatTemplate = makeTpl(fs.readFileSync(typeCfg.template, 'utf8')),
+            formatsIndex = formatTemplate({path: path, formats: installedExts}),
+            dstPath = typeCfg.template.replace(/\.ejs$/, '');
+
+        fs.writeFileSync(dstPath, formatsIndex);
+        persistExtConfig();
+    };
+
+    // Given a...
+    //  - template file
+    //  - extension type
+    //  - target path tpl
+    // create the installation/uninstallation functions
+    extender.install[typeCfg.type] = (config, project, isReinstall) => {
+        var dstPath,
+            pkgJsonPath = path.join(project.root, 'package.json'),
+            pkgJson = JSON.parse(fs.readFileSync(pkgJsonPath, 'utf8')),
+            content;
+
+        // add the config to the current installed extensions of this type
+        project = project || config.project;
+        config.version = pkgJson.version;
+        config.project = project;
+
+        allExtConfigs[typeCfg.type] = allExtConfigs[typeCfg.type] || {};
+
+        if (allExtConfigs[typeCfg.type][config.name] && !isReinstall) {
+            // eslint-disable-next-line no-console
+            console.error(`Extension ${config.name} already installed. Reinstalling...`);
+        }
+
+        allExtConfigs[typeCfg.type][config.name] = config;
+
+        // copy the main script to src/plugins/Export/formats/<name>/<main>
+        dstPath = makeTpl(typeCfg.targetDir)(config);
+        if (!exists.sync(dstPath)) {
+            fs.mkdirSync(dstPath);
+        }
+
+        try {
+            content = fs.readFileSync(path.join(project.root, config.main), 'utf8');
+        } catch (e) {
+            throw 'Could not read the extension\'s main file: ' + e;
+        }
+        dstPath = path.join(dstPath, path.basename(config.main));
+        fs.writeFileSync(dstPath, content);
+
+        saveExtensions();
+    };
+
+    // uninstall
+    extender.uninstall['Export:Pipeline'] = name => {
+        // Remove from config
+        allExtConfigs[typeCfg.type] = allExtConfigs[typeCfg.type] || {};
+
+        if (!allExtConfigs[typeCfg.type][name]) {
+            // eslint-disable-next-line no-console
+            console.log(`Extension ${name} not installed`);
+            return;
+        }
+        var config = allExtConfigs[typeCfg.type][name],
+            dstPath = makeTpl(typeCfg.targetDir)(config);
+
+        // Remove the dstPath
+        delete allExtConfigs[typeCfg.type][name];
+        rm_rf.sync(dstPath);
+
+        // Re-generate template file
+        saveExtensions();
+    };
+
+};
+
+var PLUGIN_ROOT = path.join(__dirname, '..', 'src', 'plugins', 'Export');
+makeInstallFor({
+    type: 'Export:Pipeline',
+    template: path.join(PLUGIN_ROOT, 'format.js.ejs'),
+    targetDir: path.join(PLUGIN_ROOT, 'formats', '<%=name%>')
+});
+
+module.exports = extender;

utils/reinstall-extensions.js

@@ -0,0 +1,30 @@
+// Re-install all extensions
+var extender = require('./extender'),
+    Q = require('q'),
+    extConfig = extender.getExtensionsConfig(),
+    types,
+    names,
+    currentInstall = Q(),
+    installCount = 0,
+    config;
+
+// Read the extensions and reinstall each of them
+types = Object.keys(extConfig);
+for (var i = types.length; i--;) {
+    names = Object.keys(extConfig[types[i]]);
+    if (names.length) {
+        installCount += names.length;
+        for (var j = names.length; j--;) {
+            // eslint-disable-next-line no-console
+            console.log(`Re-installing ${names[j]} extension...`);
+            config = extConfig[types[i]][names[j]];
+            currentInstall = currentInstall
+                .then(() => extender.install(config.project.arg, true));
+        }
+    }
+}
+
+if (installCount) {
+    // eslint-disable-next-line no-console
+    currentInstall.then(() => console.log('Extensions reinstalled successfully'));
+}

webgme-setup.json

@@ -29,9 +29,9 @@
         "src": "src/plugins/CreateExecution",
         "test": "test/plugins/CreateExecution"
       },
-      "GenerateExecFile": {
-        "src": "src/plugins/GenerateExecFile",
-        "test": "test/plugins/GenerateExecFile"
+      "Export": {
+        "src": "src/plugins/Export",
+        "test": "test/plugins/Export"
       },
       "ImportArtifact": {
         "src": "src/plugins/ImportArtifact",
@@ -52,6 +52,10 @@
       "UpdateLibrarySeed": {
         "src": "src/plugins/UpdateLibrarySeed",
         "test": "test/plugins/UpdateLibrarySeed"
+      },
+      "ValidateArchitecture": {
+        "src": "src/plugins/ValidateArchitecture",
+        "test": "test/plugins/ValidateArchitecture"
       }
     },
     "layouts": {