v1.2.0

* Only retrieve member registry from nodes in set. Fixes #998

* WIP #998 Added test case

README.md

@@ -4,9 +4,6 @@
 [![Join the chat at https://gitter.im/deepforge-dev/deepforge](https://badges.gitter.im/deepforge-dev/deepforge.svg)](https://gitter.im/deepforge-dev/deepforge?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 [![Stories in Ready](https://badge.waffle.io/deepforge-dev/deepforge.png?label=ready&title=Ready)](https://waffle.io/deepforge-dev/deepforge)
 
-**Notice**: DeepForge is still a work in progress and in beta! That being said, any contributions and/or feedback is greatly appreciated. If you have any questions, check out the [wiki](https://github.com/dfst/deepforge/wiki/) or drop me a line on the gitter!
-
-
 # DeepForge
 DeepForge is an open-source visual development environment for deep learning providing end-to-end support for creating deep learning models. This is achieved through providing the ability to design **architectures**, create training **pipelines**, and then execute these pipelines over a cluster. Using a notebook-esque api, users can get real-time feedback about the status of any of their **executions** including compare them side-by-side in real-time.
 
@@ -49,6 +46,7 @@ Also, be sure to check out the other available features of the `deepforge` cli;
 - [Datamodel Developer Slides](https://docs.google.com/presentation/d/1hd3IyUlzW_TIPnzCnE-1pdz00Pw8WaIxYiOW_Hyog-M/edit#slide=id.p)
 
 ## Interested in contributing?
-Contributions are welcome! Either fork the project and submit some PR's or shoot me an email about getting more involved!
+Contributions are welcome! Either fork the project and submit some PR's or shoot me an email about getting more involved! If you have any questions, check out the [wiki](https://github.com/dfst/deepforge/wiki/) or drop me a line on the gitter!
+
 
 Sponsored by [Digital Reasoning](http://www.digitalreasoning.com/)

_config.yml

@@ -0,0 +1 @@
+theme: jekyll-theme-cayman

bin/deepforge

@@ -1,6 +1,7 @@
 #!/usr/bin/env node
 
 var Command = require('commander').Command,
+    tcpPortUsed = require('tcp-port-used'),
     program = new Command(),
     childProcess = require('child_process'),
     rawSpawn = childProcess.spawn,
@@ -8,7 +9,8 @@ var Command = require('commander').Command,
     execSync = childProcess.execSync,
     path = require('path'),
     fs = require('fs'),
-    version = require('../package.json').version,
+    pkgJson = require('../package.json'),
+    version = pkgJson.version,
     exists = require('exists-file'),
     DEFAULT_CONFIG = require('./config.json'),
     merge = require('lodash.merge'),
@@ -117,32 +119,35 @@ var isLocalUri = function(protocol, uri) {
         uri.indexOf(protocol + '://127.0.0.1') === 0;
 };
 
-var checkMongo = function(args, notSilent) {
+var checkMongo = function(args, notSilent, mongoUri) {
     // check the webgme config
-    var gmeConfig = require('../config'),
-        mongoUri = gmeConfig.mongo.uri;
+    var gmeConfig = require('../config');
+
+    mongoUri = mongoUri || gmeConfig.mongo.uri;
 
     if (isLocalUri('mongodb', mongoUri)) {
+        var match = mongoUri.match(/:([0-9]+)/),
+            port = '80';
+
+        if (match) {
+            port = match[1];
+        }
+
         // Make sure mongo is running locally (using pgrep)
         try {
             execSync('pgrep mongod').toString();
             console.log('MongoDB is already running!');
         } catch (e) {  // no pIds
             console.log('Starting MongoDB...');
-            var match = mongoUri.match(/:([0-9]+)/),
-                port = '80';
-
-            if (match) {
-                port = match[1];
-            }
-
             startMongo(args, port, !notSilent);
         }
+        return tcpPortUsed.waitUntilUsed(+port, 100, 1000);
     } else if (notSilent) {
         console.log(`Cannot start remote mongo locally: ${mongoUri}`);
     } else {
         console.log(`Using remote mongo: ${mongoUri}`);
     }
+    return Q();
 };
 
 var startMongo = function(args, port, silent) {
@@ -208,28 +213,34 @@ var installTorch = function() {
         args = `clone https://github.com/torch/distro.git ${tgtDir} --recursive`.split(' ');
 
         return spawn('git', args)
-            .then(code => {
-                if (code !== 0) {
-                    if (code === 128) {
-                        console.error(`${tgtDir} is not empty. ` +
-                            'Please empty it or change the torch directory:\n' +
-                            '\n    deepforge config torch.dir NEW/TORCH/PATH\n');
+            .catch(result => {
+                var error = result.error || result.stderr ||
+                    `Torch install failed with exit code ${result.code}`;
 
-                    }
+                if (result.stderr.includes('unable to access')) {
+                    error = `Could not access the torch repository. Are you ` +
+                        `connected to the internet?\n`;
+                } else if (result.code === 128) {
+                    error = `${tgtDir} is not empty. ` +
+                        'Please empty it or change the torch directory:\n' +
+                        '\n    deepforge config torch.dir NEW/TORCH/PATH\n';
 
-                    throw `Torch install Failed with exit code ${code}`;
-                } else {  // continue installation
-                    process.chdir(tgtDir);
-                    return spawn('bash', ['install-deps'])
-                        .then(() => spawn('bash', ['install.sh'], true))
-                        .then(() => {
-                            storeConfig('torch.dir', tgtDir);
-                            console.log('Installed torch. Please close and ' +
-                                're-open your terminal to use DeepForge w/ ' +
-                                'torch support!');
-                            process.exit(0);
-                        });
                 }
+
+                console.error(error);
+                process.exit(result.code);
+            })
+            .then((code, stderr) => {
+                process.chdir(tgtDir);
+                return spawn('bash', ['install-deps'])
+                    .then(() => spawn('bash', ['install.sh'], true))
+                    .then(() => {
+                        storeConfig('torch.dir', tgtDir);
+                        console.log('Installed torch. Please close and ' +
+                            're-open your terminal to use DeepForge w/ ' +
+                            'torch support!');
+                        process.exit(0);
+                    });
             });
     } else {
         return Q();
@@ -242,21 +253,32 @@ var spawn = function(cmd, args, opts) {
         spawnOpts = typeof opts === 'object' ? opts : null,
         forwardStdin = opts === true,
         isOpen = true,
+        stderr = '',
         err;
 
     args = args || [];
     job = spawnOpts ? rawSpawn(cmd, args, spawnOpts) : rawSpawn(cmd, args);
     job.stdout.on('data', data => process.stdout.write(data));
-    job.stderr.on('data', data => process.stderr.write(data));
+    job.stderr.on('data', data => {
+        stderr += data;
+        process.stderr.write(data);
+    });
+
     job.on('close', code => {
         isOpen = false;
-        if (err) {
-            deferred.reject(err, code);
+        if (err || code !== 0) {
+            deferred.reject({
+                code: code,
+                stderr: stderr,
+                error: err
+            });
         } else {
             deferred.resolve(code);
         }
     });
-    job.on('error', e => err = e);
+    job.on('error', e => {
+        err = e;
+    });
 
     if (forwardStdin) {
         process.stdin.on('data', data => {
@@ -276,42 +298,49 @@ program.command('start')
     .option('-w, --worker [url]', 'start a worker and connect to given url. Defaults to local deepforge')
     .option('-m, --mongo', 'start MongoDB')
     .action(args => {
-        var main = path.join(__dirname, 'start-local.js');
+        var main = path.join(__dirname, 'start-local.js'),
+            current = Q();
 
         if (args.port) {
             process.env.PORT = args.port;
         }
 
+        if (args.mongo) {
+            current = current.then(() => checkMongo(args, true));
+        }
+
         if (args.server) {
-            checkMongo(args);
-            main = path.join(__dirname, '..', 'app.js');
-            spawn('node', [main]);
+            current = current
+                .then(() => checkMongo(args))
+                .then(() => {
+                    main = path.join(__dirname, '..', 'app.js');
+                    return spawn('node', [main]);
+                });
         }
 
         if (args.worker) {
             if (hasTorch()) {
-                installTorchExtras().then(() => {
-                    main = path.join(__dirname, 'start-worker.js');
-                    if (args.worker !== true) {
-                        spawn('node', [main, args.worker]);
-                    } else {
-                        spawn('node', [main]);
-                    }
-                });
+                current
+                    .then(() => installTorchExtras())
+                    .then(() => {
+                        main = path.join(__dirname, 'start-worker.js');
+                        if (args.worker !== true) {
+                            spawn('node', [main, args.worker]);
+                        } else {
+                            spawn('node', [main]);
+                        }
+                    });
             } else {
                 installTorch();
             }
         }
 
-        if (args.mongo) {
-            checkMongo(args, true);
-        }
-
         if (!args.server && !args.worker && !args.mongo) {
             // Starting everything
-            checkMongo(args);
+            current = current.then(() => checkMongo(args));
             if (hasTorch()) {
-                installTorchExtras().then(() => spawn('node', [main]));
+                current.then(() => installTorchExtras())
+                    .then(() => spawn('node', [main]));
             } else {
                 installTorch();
             }
@@ -334,7 +363,7 @@ program
         if (!args.torch || args.server) {
 
             if (args.git) {
-                pkg = 'dfst/deepforge';
+                pkg = pkgJson.repository.url;
             } else {
                 // Check the version
                 try {
@@ -447,12 +476,17 @@ program
         }
     });
 
+// extensions
+program
+    .command('extensions <command>', 'Manage deepforge extensions');
+
 module.exports = function(cmd) {
     var cmds = cmd.split(/\s+/).filter(w => !!w);
     cmds.unshift('./bin/deepforge');
     cmds.unshift('node');
     program.parse(cmds);
 };
+module.exports.checkMongo = checkMongo;
 
 if (require.main === module) {
     program.parse(process.argv);

bin/deepforge-extensions

@@ -0,0 +1,67 @@
+#!/usr/bin/env node
+
+var Command = require('commander').Command,
+    program = new Command(),
+    extender = require('../utils/extender');
+
+// Supported commands
+//  - add
+//  - remove
+//  - list
+//  - update
+program
+    .command('add <project>')
+    .description('Add an extension to deepforge')
+    .option('-n, --name <name>', 'Project name (if different from <project>)')
+    .action(project => {
+        console.log('loading extension from: ' + project);
+        extender.install(project)
+            .then(extConfig => 
+                console.log(`The ${extConfig.name} extension has been added to deepforge.`))
+            .fail(err => {
+                console.error('Could not install extension:\n');
+                console.error(err);
+                process.exit(1);
+            });
+    });
+
+program
+    .command('remove <name>').alias('rm')
+    .description('Remove an extension from deepforge')
+    .action(name => {
+        try {
+            extender.uninstall(name);
+            console.log(`${name} has been successfully removed!`);
+        } catch (e) {
+            console.error('Could not remove extension:');
+            console.error(e);
+            process.exit(1);
+        }
+    });
+
+program
+    .command('list').alias('ls')
+    .description('List installed deepforge extensions')
+    .action(() => {
+        var allExtConfigs = extender.getExtensionsConfig(),
+            types = Object.keys(allExtConfigs),
+            hasContents = false,
+            names;
+
+        for (var i = types.length; i--;) {
+            names = Object.keys(allExtConfigs[types[i]]);
+            if (names.length) {
+                hasContents = true;
+                console.log(types[i]);
+                for (var j = names.length; j--;) {
+                    console.log(`  ${names[j]}`);
+                }
+            }
+        }
+
+        if (!hasContents) {
+            console.log('No installed extensions');
+        }
+    });
+
+program.parse(process.argv);

bin/start-local.js

@@ -13,8 +13,8 @@ if (gmeConfig.blob.type === 'FS') {
 }
 
 process.env.NODE_ENV = 'local';
-execJob = spawn('npm', [
-    'start'
+execJob = spawn('node', [
+    path.join(__dirname, '..', 'app.js')
 ], env);
 execJob.stdout.pipe(process.stdout);
 execJob.stderr.pipe(process.stderr);

bin/start-worker.js

@@ -73,6 +73,9 @@ var createConfigJson = function() {
 
     if (process.argv.length > 2) {
         address = process.argv[2];
+        if (!/^https?:\/\//.test(address)) {
+            address = 'http://' + address;
+        }
     }
 
     config[address] = {};

config/components.json

@@ -48,10 +48,6 @@
             "ImportTorch": {
                 "icon": "import_export",
                 "priority": -1
-            },
-            "GenerateExecFile": {
-                "icon": "play_for_work",
-                "priority": -1
             }
         }
     },

package.json

@@ -1,10 +1,15 @@
 {
   "name": "deepforge",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/deepforge-dev/deepforge.git"
+  },
   "bin": {
     "deepforge": "./bin/deepforge"
   },
   "scripts": {
-    "start": "node app.js",
+    "start": "./bin/deepforge start",
+    "postinstall": "node utils/reinstall-extensions.js",
     "start-dev": "NODE_ENV=dev node app.js",
     "local": "node ./bin/start-local.js",
     "worker": "node ./bin/start-worker.js",
@@ -12,7 +17,7 @@
     "watch-test": "nodemon --exec 'mocha --recursive test'",
     "build-nn": "node ./utils/nn-parser.js"
   },
-  "version": "0.22.0",
+  "version": "1.2.0",
   "dependencies": {
     "commander": "^2.9.0",
     "dotenv": "^2.0.0",
@@ -21,10 +26,13 @@
     "graceful-fs": "^4.1.10",
     "lodash.difference": "^4.1.2",
     "lodash.merge": "^4.5.1",
+    "lodash.template": "^4.4.0",
     "mongodb": "^2.2.10",
     "nodemon": "^1.9.2",
+    "npm": "^4.0.5",
     "q": "1.4.1",
     "rimraf": "^2.4.0",
+    "tcp-port-used": "^0.1.2",
     "webgme": "^2.7.1",
     "webgme-autoviz": "^2.2.0",
     "webgme-breadcrumbheader": "^2.1.1",

src/common/Constants.js

@@ -15,6 +15,7 @@
         CONTAINED_LAYER_INDEX: 'index',
 
         LINE_OFFSET: 'lineOffset',
+        DISPLAY_COLOR: 'displayColor',
 
         // DeepForge metadata creation in dist execution
         START_CMD: 'deepforge-cmd',
@@ -30,6 +31,10 @@
         GRAPH_CREATE: 'GRAPH',
         GRAPH_PLOT: 'PLOT',
         GRAPH_CREATE_LINE: 'LINE',
+        GRAPH_LABEL_AXIS: {
+            X: 'X',
+            Y: 'Y'
+        },
 
         // Code Generation Constants
         CTOR_ARGS_ATTR: 'ctor_arg_order',

src/common/globals.js

@@ -271,10 +271,8 @@ define([
             .filter(n => !n.getRegistry('isAbstract'))
             .map(node => node.getAttribute('name'));
 
-        //this.logger.info(`Found ${dataTypes.length} data types`);
-
-        // Add the target type to the pluginMetadata... hacky :/
-        var metadata = WebGMEGlobal.allPluginsMetadata[UPLOAD_PLUGIN], 
+        // Add the target type to the pluginMetadata...
+        var metadata = WebGMEGlobal.allPluginsMetadata[UPLOAD_PLUGIN],
             config = metadata.configStructure
                 .find(opt => opt.name === DATA_TYPE_CONFIG.name);
 

src/common/viz/NodePrompter.js

@@ -4,9 +4,11 @@
 // adding a "plus" button for creating new objects in line
 
 define([
+    'deepforge/Constants',
     'q',
     'css!./NodePrompter.css'
 ], function(
+    Constants,
     Q
 ) {
 
@@ -271,12 +273,14 @@ define([
     };
 
     var Container = function(svg, node) {  // used for positioning
+        var colorAttr = node.attributes[Constants.DISPLAY_COLOR];
         this.$el = svg.append('g');
         this.x = 0;
         this.y = 0;
         this.node = node;
         this.decorator = new node.Decorator({
             node: node,
+            color: colorAttr && colorAttr.value,
             parentEl: this.$el
         });
     };

src/common/viz/PipelineControl.js

@@ -103,6 +103,10 @@ define([
                 delete desc.attributes.code;
             }
 
+            // Handle the display color
+            desc.displayColor = desc.attributes[CONSTANTS.DISPLAY_COLOR].value;
+            delete desc.attributes[CONSTANTS.DISPLAY_COLOR];
+
         } else if (desc.isConnection) {
             // Set src, dst to siblings and add srcPort, dstPort
             desc.srcPort = desc.src;

src/decorators/ArtifactOpDecorator/EasyDAG/ArtifactOpDecorator.EasyDAGWidget.js

@@ -40,10 +40,40 @@ define([
     ArtifactOpDecorator.prototype.DECORATOR_ID = DECORATOR_ID;
 
     ArtifactOpDecorator.prototype.getTargetFilterFnFor = function() {
+        var currentNode,
+            pId,
+            peerIds,
+            currentDataId,
+            targetTypeIds = [];
+
+        // Get all connections from this node's data
+        if (this._node.outputs[0]) {
+            currentNode = this.client.getNode(this._node.id);
+            pId = currentNode.getParentId();
+            peerIds = this.client.getNode(pId).getChildrenIds();
+            currentDataId = this._node.outputs[0].id;
+
+            targetTypeIds = peerIds.map(id => this.client.getNode(id))
+                .filter(node => {
+                    var ptr = node.getPointer('src');
+                    return ptr.to === currentDataId;
+                })
+
+                // Get the target data types
+                .map(node => this.client.getNode(node.getPointer('dst').to).getMetaTypeId());
+        }
+
         return id => {
             var node = this.client.getNode(id),
-                isMetaTgt = node.getId() === node.getMetaTypeId();
-            return isMetaTgt === this.castOpts.metaTgt;
+                isMetaTgt = node.getId() === node.getMetaTypeId(),
+                isValidType = true;
+
+            // make sure it is a type of the target types
+            isValidType = targetTypeIds.reduce((passing, typeId) => {
+                return passing && node.isTypeOf(typeId);
+            }, true);
+
+            return isValidType && isMetaTgt === this.castOpts.metaTgt;
         };
     };
 

src/decorators/OperationDecorator/EasyDAG/OperationDecorator.EasyDAGWidget.js

@@ -2,9 +2,11 @@
 /*jshint browser: true, camelcase: false*/
 
 define([
+    'deepforge/Constants',
     'decorators/EllipseDecorator/EasyDAG/EllipseDecorator.EasyDAGWidget',
     'css!./OperationDecorator.EasyDAGWidget.css'
 ], function (
+    CONSTANTS,
     DecoratorBase
 ) {
 
@@ -13,6 +15,7 @@ define([
     var OperationDecorator,
         NAME_MARGIN = 25,
         DECORATOR_ID = 'OperationDecorator',
+        OPERATION_COLORS = {},
         PORT_TOOLTIP_OPTS = {
             tipJoint: 'left',
             removeElementsOnHide: true,
@@ -24,8 +27,10 @@ define([
     //     - highlight ports
     //     - unhighlight ports
     //     - report the location of specific ports
+    OPERATION_COLORS[CONSTANTS.OP.OUTPUT] = '#b0bec5';
+    OPERATION_COLORS[CONSTANTS.OP.INPUT] = '#b0bec5';
     OperationDecorator = function (options) {
-        options.color = options.color || '#78909c';
+        options.color = OPERATION_COLORS[options.node.name] || options.color || '#78909c';
         DecoratorBase.call(this, options);
 
         this.id = this._node.id;

src/plugins/ExecuteJob/ExecuteJob.Files.js

@@ -11,6 +11,15 @@ define([
     CONSTANTS
 ) {
 
+    var SKIP_ATTRIBUTES = [
+        'code',
+        'stdout',
+        'execFiles',
+        'jobId',
+        'secret',
+        CONSTANTS.LINE_OFFSET,
+        CONSTANTS.DISPLAY_COLOR
+    ];
     var ExecuteJob = function() {
     };
 
@@ -285,13 +294,12 @@ define([
     };
 
     ExecuteJob.prototype.createAttributeFile = function (node, files) {
-        var skip = ['code', 'stdout', 'execFiles', 'jobId', 'secret', CONSTANTS.LINE_OFFSET],
-            numOrBool = /^(-?\d+\.?\d*((e|e-)\d+)?|(true|false))$/,
+        var numOrBool = /^(-?\d+\.?\d*((e|e-)\d+)?|(true|false))$/,
             table;
 
         this.logger.info('Creating attributes file...');
         table = '{\n\t' + this.core.getAttributeNames(node)
-            .filter(attr => skip.indexOf(attr) === -1)
+            .filter(attr => SKIP_ATTRIBUTES.indexOf(attr) === -1)
             .map(name => {
                 var value = this.getAttribute(node, name);
                 if (!numOrBool.test(value)) {

src/plugins/ExecuteJob/ExecuteJob.Metadata.js

@@ -30,6 +30,30 @@ define([
         this._metadata[id] = graph;
     };
 
+    ExecuteJob.prototype[CONSTANTS.GRAPH_LABEL_AXIS.X] = function (job, id) {
+        var name = Array.prototype.slice.call(arguments, 2).join(' '),
+            jobId = this.core.getPath(job),
+            graph;
+
+        id = jobId + '/' + id;
+        this.logger.info(`Labeling the x-axis of ${id}: ${name}`);
+
+        graph = this._metadata[id];
+        this.setAttribute(graph, 'xlabel', name);
+    };
+
+    ExecuteJob.prototype[CONSTANTS.GRAPH_LABEL_AXIS.Y] = function (job, id) {
+        var name = Array.prototype.slice.call(arguments, 2).join(' '),
+            jobId = this.core.getPath(job),
+            graph;
+
+        id = jobId + '/' + id;
+        this.logger.info(`Labeling the y-axis of ${id}: ${name}`);
+
+        graph = this._metadata[id];
+        this.setAttribute(graph, 'ylabel', name);
+    };
+
     ExecuteJob.prototype[CONSTANTS.GRAPH_PLOT] = function (job, id, x, y) {
         var jobId = this.core.getPath(job),
             nonNum = /[^\d-\.]*/g,

src/plugins/ExecuteJob/templates/deepforge.ejs

@@ -52,6 +52,14 @@ function Graph:line(name, opts)
     return deepforge._Line(self.id, name, opts)
 end
 
+function Graph:xlabel(name)
+    deepforge._cmd('<%= GRAPH_LABEL_AXIS.X %>', self.id, name)
+end
+
+function Graph:ylabel(name)
+    deepforge._cmd('<%= GRAPH_LABEL_AXIS.Y %>', self.id, name)
+end
+
 -- Image support
 local function saveImage(name, tensor)
     require 'image'

src/plugins/Export/Export.js

@@ -4,19 +4,21 @@
 define([
     'text!./metadata.json',
     'text!./deepforge.ejs',
-    'text!./toboolean.lua',
+    './format',
     'plugin/PluginBase',
     'deepforge/plugin/PtrCodeGen',
     'deepforge/Constants',
+    'blob/BlobConfig',
     'underscore',
     'q'
 ], function (
     pluginMetadata,
-    deepForgeTxt,
-    TOBOOLEAN,
+    DeepForgeBaseCode,
+    FORMATS,
     PluginBase,
     PtrCodeGen,
     CONSTANTS,
+    BlobConfig,
     _,
     Q
 ) {
@@ -29,21 +31,16 @@ define([
             code: true
         },
         RESERVED = /^(and|break|do|else|elseifend|false|for|function|if|in|local|nil|not|orrepeat|return|then|true|until|while|print)$/,
-        INDENT = '   ',
-        INIT_CLASSES_FN = '__initClasses',
-        INIT_LAYERS_FN = '__initLayers',
-        DEEPFORGE_CODE = _.template(deepForgeTxt)({
-            initCode: `${INIT_CLASSES_FN}()\n${INDENT}${INIT_LAYERS_FN}()`
-        });
+        DeepForgeTpl = _.template(DeepForgeBaseCode);
 
     /**
-     * Initializes a new instance of GenerateExecFile.
+     * Initializes a new instance of Export.
      * @class
      * @augments {PluginBase}
-     * @classdesc This class represents the plugin GenerateExecFile.
+     * @classdesc This class represents the plugin Export.
      * @constructor
      */
-    var GenerateExecFile = function () {
+    var Export = function () {
         // Call base class' constructor.
         PluginBase.call(this);
         this.initRecords();
@@ -54,13 +51,13 @@ define([
      * This is also available at the instance at this.pluginMetadata.
      * @type {object}
      */
-    GenerateExecFile.metadata = pluginMetadata;
+    Export.metadata = pluginMetadata;
 
     // Prototypical inheritance from PluginBase.
-    GenerateExecFile.prototype = Object.create(PluginBase.prototype);
-    GenerateExecFile.prototype.constructor = GenerateExecFile;
+    Export.prototype = Object.create(PluginBase.prototype);
+    Export.prototype.constructor = Export;
 
-    GenerateExecFile.prototype.initRecords = function() {
+    Export.prototype.initRecords = function() {
         this.pluginMetadata = pluginMetadata;
 
         this._srcIdFor = {};  // input path -> output data node path
@@ -98,9 +95,7 @@ define([
      *
      * @param {function(string, plugin.PluginResult)} callback - the result callback
      */
-    GenerateExecFile.prototype.main = function (callback) {
-        var name = this.core.getAttribute(this.activeNode, 'name');
-
+    Export.prototype.main = function (callback) {
         this.initRecords();
 
         // Get all the children and call generate exec file
@@ -112,8 +107,8 @@ define([
         }
 
         return this.core.loadChildren(this.activeNode)
-            .then(nodes => this.createExecFile(nodes))
-            .then(code => this.blobClient.putFile(`${name}.lua`, code))
+            .then(nodes => this.generateOutputFiles(nodes))
+            .catch(err => callback(err))
             .then(hash => {
                 this.result.addArtifact(hash);
                 this.result.setSuccess(true);
@@ -122,62 +117,84 @@ define([
             .fail(err => callback(err));
     };
 
-    GenerateExecFile.prototype.createExecFile = function (children) {
+    Export.prototype.getCurrentConfig = function () {
+        var config = PluginBase.prototype.getCurrentConfig.call(this);
+        config.staticInputs = config.staticInputs || [];
+        return config;
+    };
+
+    Export.prototype.getExporterFor = function (name) {
+        var Exporter = function() {},
+            format = FORMATS[name],
+            exporter;
+
+        Exporter.prototype = this;
+        exporter = new Exporter();
+
+        if (typeof format === 'function') {
+            exporter.main = format;
+        } else {
+            _.extend(exporter, format);
+        }
+        return exporter;
+    };
+
+    Export.prototype.generateOutputFiles = function (children) {
+        var name = this.core.getAttribute(this.activeNode, 'name');
+
         return this.createCodeSections(children)
             .then(sections => {
-                var classes,
-                    initClassFn,
-                    initLayerFn,
-                    code = [];
+                // Get the selected format
+                var config = this.getCurrentConfig(),
+                    format = config.format || 'Basic CLI',
+                    exporter,
+                    staticInputs,
+                    files;
 
-                // concat all the sections into a single file
+                this.logger.info(`About to retrieve ${config.format} exporter`);
+                exporter = this.getExporterFor(format);
 
-                // wrap the class/layer initialization in a fn
-                // Add the classes ordered wrt their deps
-                classes = Object.keys(sections.classes)
-                    .sort((a, b) => {
-                        // if a depends on b, switch them (return 1)
-                        if (sections.classDependencies[a].includes(b)) {
-                            return 1;
+                staticInputs = config.staticInputs.map(id => {
+                    var opId = id.split('/').splice(0, this.activeNodeDepth).join('/'),
+                        port = this._portCache[id];
+
+                    return {
+                        portId: id,
+                        id: opId,
+                        hash: this.core.getAttribute(port, 'data'),
+                        name: this._nameFor[opId]
+                    };
+                });
+
+                this.logger.info('Invoking exporter "main" function...');
+                try {
+                    files = exporter.main(sections, staticInputs, config.extensionConfig);
+                } catch (e) {
+                    this.logger.error(`Exporter failed: ${e.toString()}`);
+                    throw e;
+                }
+                // If it returns a string, just put a single file
+                if (typeof files === 'string') {
+                    return this.blobClient.putFile(`${name}.lua`, files);
+                } else {  // filename -> content
+                    var artifact = this.blobClient.createArtifact(name),
+                        objects = {};
+
+                    Object.keys(files).forEach(key => {
+                        if (BlobConfig.hashRegex.test(files[key])) {
+                            objects[key] = files[key];
+                            delete files[key];
                         }
-                        return -1;
-                    })
-                    // Create fns from the classes
-                    .map(name => [
-                        `local function init${name}()`,
-                        indent(sections.classes[name]),
-                        'end',
-                        `init${name}()`
-                    ].join('\n'));
+                    });
 
-                initClassFn = [
-                    `local function ${INIT_CLASSES_FN}()`,
-                    indent(classes.join('\n\n')),
-                    'end'
-                ].join('\n');
-
-                code = code.concat(initClassFn);
-
-                // wrap the layers in a function
-                initLayerFn = [
-                    `local function ${INIT_LAYERS_FN}()`,
-                    indent(_.values(sections.layers).join('\n\n')),
-                    'end'
-                ].join('\n');
-                code = code.concat(initLayerFn);
-                code = code.concat(_.values(sections.operations));
-
-                code = code.concat(_.values(sections.pipelines));
-
-                code.push(DEEPFORGE_CODE);
-                code.push('deepforge.initialize()');
-                code.push(sections.main);
-
-                return code.join('\n\n');
+                    return artifact.addFiles(files)
+                        .then(() => artifact.addObjectHashes(objects))
+                        .then(() => artifact.save());
+                }
             });
     };
 
-    GenerateExecFile.prototype.createCodeSections = function (children) {
+    Export.prototype.createCodeSections = function (children) {
         // Convert opNodes' jobs to the nested operations
         var opNodes,
             nodes;
@@ -221,7 +238,7 @@ define([
             .fail(err => this.logger.error(err));
     };
 
-    GenerateExecFile.prototype.unpackJobs = function (nodes) {
+    Export.prototype.unpackJobs = function (nodes) {
         return Q.all(
             nodes.map(node => {
                 if (!this.isMetaTypeOf(node, this.META.Job)) {
@@ -235,7 +252,7 @@ define([
         );
     };
 
-    GenerateExecFile.prototype.sortOperations = function (operationDict, opIds) {
+    Export.prototype.sortOperations = function (operationDict, opIds) {
         var nextIds = [],
             sorted = opIds,
             dstIds,
@@ -263,7 +280,7 @@ define([
             .concat(this.sortOperations(operationDict, nextIds));
     };
 
-    GenerateExecFile.prototype.generateCodeSections = function(sortedOps) {
+    Export.prototype.generateCodeSections = function(sortedOps) {
         // Create the code sections:
         //  - operation definitions
         //  - pipeline definition
@@ -293,8 +310,12 @@ define([
         // Define the pipeline function
         code.pipelines = this.definePipelineFn(mainOps, outputOps);
 
-        // Define the main body
-        this.addCodeMain(code);
+        // Define the serializers/deserializers
+        this.addCodeSerializers(code);
+
+        // Define the main input names
+        code.pipelineName = Object.keys(code.pipelines)[0];
+        code.pipelineInputNames = Object.keys(this.isInputOp).map(id => this._nameFor[id]);
 
         // Add custom class definitions
         this.addCustomClasses(code);
@@ -305,11 +326,13 @@ define([
         return code;
     };
 
-    var indent = function(text) {
-        return text.replace(/^/mg, INDENT);
+    // expose this utility function to format extensions
+    var indent = Export.prototype.indent = function(text, spaces) {
+        spaces = spaces || 3;
+        return text.replace(/^/mg, new Array(spaces+1).join(' '));
     };
 
-    GenerateExecFile.prototype.defineOperationFn = function(operation) {
+    Export.prototype.defineOperationFn = function(operation) {
         var lines = [],
             args = operation.inputNames || [];
 
@@ -327,7 +350,7 @@ define([
         return lines.join('\n');
     };
 
-    GenerateExecFile.prototype.definePipelineFn = function(sortedOps, outputOps) {
+    Export.prototype.definePipelineFn = function(sortedOps, outputOps) {
         var inputArgs = Object.keys(this.isInputOp).map(id => this._nameFor[id]),
             name = this.core.getAttribute(this.activeNode, 'name'),
             safename = getUniqueName(name, this._opBaseNames),
@@ -353,7 +376,7 @@ define([
         return result;
     };
 
-    GenerateExecFile.prototype.getOutputPair = function(operation) {
+    Export.prototype.getOutputPair = function(operation) {
         var input = operation.inputValues[0].slice(),
             value;
 
@@ -363,39 +386,36 @@ define([
         return [this._nameFor[operation.id], value];
     };
 
-    GenerateExecFile.prototype.addCodeMain = function(sections) {
-        var pipelineName = Object.keys(sections.pipelines)[0],
-            hasBool = false,
-            code = [],
-            loadNodes = {},
-            args;
+    Export.prototype.addCodeSerializers = function(sections) {
+        var loadNodes = {},
+            saveNodes = {};
 
-        args = Object.keys(this.isInputOp).map((id, index) => {
-            var node = this.inputNode[id],
-                base = this.core.getBase(node),
-                type = this.core.getAttribute(base, 'name'),
-                arg = `arg[${index+1}]`;
+        // Add the serializer fn names for each input
+        sections.serializerFor = {};
+        sections.deserializerFor = {};
 
-            if (type === 'boolean') {
-                hasBool = true;
-                return `toboolean(${arg})`;
-            } else if (type === 'number') {
-                return `tonumber(${arg})`;
-            } else if (type === 'string') {
-                return arg;
-            } else {
-                loadNodes[id] = node;
-                return `load['${this._nameFor[id]}'](${arg})`;
-            }
+        Object.keys(this.isOutputOp).map(id => {
+            var name = this._nameFor[id];
+            sections.serializerFor[name] = `__save['${name}']`;
         });
 
-        // Handle the arg types
-        if (hasBool) {
-            // add toboolean def
-            code.push(TOBOOLEAN);
-        }
-        // Define the 'saveOutputs' method
-        var saveNodes = {};
+        // Add the serializer definitions
+        Object.keys(this.isInputOp).forEach(id => {
+            var node = this.inputNode[id],
+                name = this._nameFor[id];
+
+            loadNodes[id] = node;
+            sections.deserializerFor[name] = `__load['${this._nameFor[id]}']`;
+        });
+
+        sections.deserializers = this.createTorchFnDict(
+            '__load',
+            loadNodes,
+            'deserialize',
+            'path'
+        );
+
+        // Add the deserializer definitions
         Object.keys(this.outputDataToOpId).forEach(dataId => {
             var opId = this.outputDataToOpId[dataId];
             // The key is used for the output name resolution. The
@@ -405,32 +425,31 @@ define([
             saveNodes[opId] = this._portCache[this._srcIdFor[dataId]];
         });
 
-        // Add dictionary of serializers/deserializers
-        code.push(
-            this.createTorchFnDict('load', loadNodes, 'deserialize', 'path'),
-            this.createTorchFnDict('save', saveNodes, 'serialize', 'path, data')
+        sections.serializers = this.createTorchFnDict(
+            '__save',
+            saveNodes,
+            'serialize',
+            'path, data'
         );
 
         // Add a saveOutputs method for convenience
-        code.push([
-            'local function saveOutputs(data)',
+        sections.serializeOutputsDef = [
+            'local function __saveOutputs(data)',
             indent(Object.keys(this.isOutputOp).map(id => {
                 var name = this._nameFor[id];
-                return `print('saving ${name}...')\nsave['${name}']('${name}', data['${name}'])`;
+                return [
+                    `print('saving ${name}...')`,
+                    `${sections.serializerFor[name]}('${name}', data['${name}'])`
+                ].join('\n');
+
             }).join('\n')),
             'end'
-        ].join('\n'));
+        ].join('\n');
 
-        code.push(
-            `local outputs = ${pipelineName}(${args.join(', ')})\n` +
-            'saveOutputs(outputs)',
-            'return outputs'
-        );
-
-        sections.main = code.join('\n\n');
+        sections.serializeOutputs = '__saveOutputs(outputs)';
     };
 
-    GenerateExecFile.prototype.createTorchFnDict = function(name, nodeDict, attr, args) {
+    Export.prototype.createTorchFnDict = function(name, nodeDict, attr, args) {
         return [
             `local ${name} = {}`,
             Object.keys(nodeDict).map(id => {
@@ -444,7 +463,7 @@ define([
         ].join('\n');
     };
 
-    GenerateExecFile.prototype.addCustomClasses = function(sections) {
+    Export.prototype.addCustomClasses = function(sections) {
         var metaDict = this.core.getAllMetaNodes(this.rootNode),
             isClass,
             metanodes,
@@ -497,9 +516,19 @@ define([
 
                 sections.classes[name] = code;
             });
+
+        // order classes by dependency
+        sections.orderedClasses = Object.keys(sections.classes)
+            .sort((a, b) => {
+                // if a depends on b, switch them (return 1)
+                if (sections.classDependencies[a].includes(b)) {
+                    return 1;
+                }
+                return -1;
+            });
     };
 
-    GenerateExecFile.prototype.addCustomLayers = function(sections) {
+    Export.prototype.addCustomLayers = function(sections) {
         var metaDict = this.core.getAllMetaNodes(this.rootNode),
             isCustomLayer,
             metanodes,
@@ -523,7 +552,7 @@ define([
     };
 
 
-    GenerateExecFile.prototype.getTypeDictFor = function (name, metanodes) {
+    Export.prototype.getTypeDictFor = function (name, metanodes) {
         var isType = {};
         // Get all the custom layers
         for (var i = metanodes.length; i--;) {
@@ -541,7 +570,7 @@ define([
         return `"${attr}"`;
     };
 
-    GenerateExecFile.prototype.getOpInvocation = function(op) {
+    Export.prototype.getOpInvocation = function(op) {
         var lines = [],
             attrs,
             refInits = [],
@@ -574,13 +603,13 @@ define([
         return lines.join('\n');
     };
 
-    GenerateExecFile.prototype.getOutputName = function(node) {
+    Export.prototype.getOutputName = function(node) {
         var basename = this.core.getAttribute(node, 'saveName');
 
         return getUniqueName(basename, this._outputNames, true);
     };
 
-    GenerateExecFile.prototype.getVariableName = function (/*node*/) {
+    Export.prototype.getVariableName = function (/*node*/) {
         var c = Object.keys(this.isInputOp).length;
 
         if (c !== 1) {
@@ -590,7 +619,7 @@ define([
         return 'input';
     };
 
-    GenerateExecFile.prototype.registerNode = function (node) {
+    Export.prototype.registerNode = function (node) {
         if (this.isMetaTypeOf(node, this.META.Operation)) {
             return this.registerOperation(node);
         } else if (this.isMetaTypeOf(node, this.META.Transporter)) {
@@ -619,7 +648,7 @@ define([
         return name;
     };
 
-    GenerateExecFile.prototype.registerOperation = function (node) {
+    Export.prototype.registerOperation = function (node) {
         var name = this.core.getAttribute(node, 'name'),
             id = this.core.getPath(node),
             base = this.core.getBase(node),
@@ -681,7 +710,7 @@ define([
             });
     };
 
-    GenerateExecFile.prototype.registerTransporter = function (node) {
+    Export.prototype.registerTransporter = function (node) {
         var outputData = this.core.getPointerPath(node, 'src'),
             inputData = this.core.getPointerPath(node, 'dst'),
             srcOpId = this.getOpIdFor(outputData),
@@ -700,7 +729,7 @@ define([
         this._incomingCnts[dstOpId]++;
     };
 
-    GenerateExecFile.prototype.getOpIdFor = function (dataId) {
+    Export.prototype.getOpIdFor = function (dataId) {
         var ids = dataId.split('/'),
             depth = ids.length;
 
@@ -715,7 +744,7 @@ define([
     //   - add the references
     //     - generate the code
     //     - replace the `return <thing>` w/ `<ref-name> = <thing>`
-    GenerateExecFile.prototype.createOperation = function (node) {
+    Export.prototype.createOperation = function (node) {
         var id = this.core.getPath(node),
             baseId = this.core.getPath(this.core.getBase(node)),
             attrNames = this.core.getValidAttributeNames(node),
@@ -790,12 +819,12 @@ define([
             });
     };
 
-    GenerateExecFile.prototype.genPtrSnippet = function (ptrName, pId) {
+    Export.prototype.genPtrSnippet = function (ptrName, pId) {
         return this.getPtrCodeHash(pId)
             .then(hash => this.blobClient.getObjectAsString(hash));
     };
 
-    GenerateExecFile.prototype.createHeader = function (title, length) {
+    Export.prototype.createHeader = function (title, length) {
         var len;
         title = ` ${title} `;
         length = length || HEADER_LENGTH;
@@ -813,7 +842,7 @@ define([
 
     };
 
-    GenerateExecFile.prototype.genOperationCode = function (operation) {
+    Export.prototype.genOperationCode = function (operation) {
         var header = this.createHeader(`"${operation.name}" Operation`),
             codeParts = [],
             body = [];
@@ -840,15 +869,57 @@ define([
         return operation;
     };
 
-    GenerateExecFile.prototype.assignResultToVar = function (code, name) {
-        var i = code.lastIndexOf('return');
+    _.extend(Export.prototype, PtrCodeGen.prototype);
 
-        return code.substring(0, i) +
-            code.substring(i)
-                .replace('return', `${name} = `);
+    // Extra utilities for export types
+    Export.prototype.INIT_CLASSES_FN = '__init_classes';
+    Export.prototype.INIT_LAYERS_FN = '__init_layers';
+    Export.prototype.getAllDefinitions = function (sections) {
+        var code = [],
+            classes,
+            initClassFn,
+            initLayerFn;
+
+        classes = sections.orderedClasses
+            // Create fns from the classes
+            .map(name => this.indent(sections.classes[name])).join('\n');
+
+        initClassFn = [
+            `local function ${this.INIT_CLASSES_FN}()`,
+            this.indent(classes),
+            'end'
+        ].join('\n');
+
+        code = code.concat(initClassFn);
+
+        // wrap the layers in a function
+        initLayerFn = [
+            `local function ${this.INIT_LAYERS_FN}()`,
+            this.indent(_.values(sections.layers).join('\n\n')),
+            'end'
+        ].join('\n');
+        code = code.concat(initLayerFn);
+
+        // Add operation fn definitions
+        code = code.concat(_.values(sections.operations));
+        code = code.concat(_.values(sections.pipelines));
+
+        // define deserializers, serializers
+        code.push(sections.deserializers);
+        code.push(sections.serializers);
+
+        code.push(this.getDeepforgeObject());
+        code.push('deepforge.initialize()');
+
+        code.push(sections.serializeOutputsDef);
+        return code.join('\n\n');
     };
 
-    _.extend(GenerateExecFile.prototype, PtrCodeGen.prototype);
+    Export.prototype.getDeepforgeObject = function (content) {
+        content = content || {};
+        content.initCode = content.initCode || `${this.INIT_CLASSES_FN}()\n${'   '}${this.INIT_LAYERS_FN}()`;
+        return DeepForgeTpl(content);
+    };
 
-    return GenerateExecFile;
+    return Export;
 });

src/plugins/Export/format.js

@@ -0,0 +1,13 @@
+
+/* globals define*/
+// The supported export formats and metadata
+define([
+    './formats/cli/cli'
+], function(
+    Format0
+) {
+
+    return {
+        'Basic CLI': Format0
+    };
+});

src/plugins/Export/format.js.ejs

@@ -0,0 +1,21 @@
+<% // Add default format
+formats.unshift({ name: 'cli', main: 'cli.js', displayName: 'Basic CLI' })
+%>
+/* globals define*/
+// The supported export formats and metadata
+define([
+<%= formats.map(function(format) {
+    return '    \'./formats/' + format.name + '/' +
+        path.basename(format.main.replace(/\.js$/, '')) + '\''
+    })
+    .join(',\n') %>
+], function(
+<%= formats.map(function(f, index) { return '    Format' + index; }).join(',\n') %>
+) {
+
+    return {
+<%= formats.map(function(f, index) {
+        return '        \'' + f.displayName + '\': Format' + index;
+    }).join(',\n') %>
+    };
+});

src/plugins/Export/formats/cli/cli.js

@@ -0,0 +1,103 @@
+/*globals define*/
+// Simple torch cli for the given pipeline
+define([
+], function(
+) {
+
+    var TOBOOLEAN = 
+`local function toboolean(str)
+    if str == 'true' then
+        return true
+    elseif str == 'false' then
+        return false
+    end
+end`;
+
+    var CliExporter = {};
+
+    CliExporter.deserializersFromString = function(sections) {
+        var hasBool = false;
+
+        // Add serializers given cli string input
+        Object.keys(this.isInputOp).forEach(id => {
+            var node = this.inputNode[id],
+                base = this.core.getBase(node),
+                type = this.core.getAttribute(base, 'name'),
+                name = this._nameFor[id];
+
+            if (type === 'boolean') {
+                hasBool = true;
+                sections.deserializerFor[name] = 'toboolean';
+            } else if (type === 'number') {
+                sections.deserializerFor[name] = 'tonumber';
+            } else if (type === 'string') {
+                sections.deserializerFor[name] = 'tostring';
+            }
+        });
+
+        if (hasBool) {
+            sections.deserializers += '\n' + TOBOOLEAN;
+        }
+
+        return sections;
+    };
+
+    CliExporter.main = function (sections, staticInputs) {
+        var code = [];
+
+        // Update deserializers for cli input
+        this.deserializersFromString(sections);
+
+        // Define all the operations, pipelines, etc
+        // 'getAllDefinitions' is provided as part of the public api
+        code.push(this.getAllDefinitions(sections));
+
+        // Command line specific stuff
+        var files = {},
+            main,
+            args,
+            staticNames = staticInputs.map(input => input.name),
+            varDefs,
+            index = 1;
+
+        // Create some names for the inputs
+        args = sections.pipelineInputNames.map(name => `${sections.deserializerFor[name]}(${name})`);
+
+        main = `local outputs = ${sections.pipelineName}(${args.join(', ')})`;
+
+        // Grab the args from the cli
+        code.push(sections.pipelineInputNames.map((name, index) => {
+            return `local ${name} = arg[${index + 1}]`;
+        }).join('\n'));
+
+        // Add the hash for each of the static inputs and reference them
+        staticInputs.forEach(input => {
+            files[`res/${input.name}`] = input.hash;
+        });
+
+        varDefs = staticNames.map(name => {
+            return `local ${name} = './res/${name}'`;
+        });
+
+        // Grab the remaining args from the cli
+        varDefs = varDefs.concat(sections.pipelineInputNames.map(name => {
+            if (!staticNames.includes(name)) {
+                return `local ${name} = arg[${index++}]`;
+            }
+        }));
+
+        // Add the main fn
+        code.push(varDefs.join('\n'));
+        code.push(main);
+
+        // Save outputs to disk
+        code.push(sections.serializeOutputs);
+
+        files['init.lua'] = code.join('\n\n');
+
+        // if no extra assets, just return the main file
+        return staticInputs.length ? files : files['init.lua'];
+    };
+
+    return CliExporter;
+});

src/plugins/Export/metadata.json

@@ -1,7 +1,7 @@
 {
-  "id": "GenerateExecFile",
-  "name": "Generate Execution File",
-  "version": "0.1.0",
+  "id": "Export",
+  "name": "Export",
+  "version": "1.0.0",
   "description": "",
   "icon": {
     "class": "glyphicon glyphicon-cog",

src/plugins/GenerateArchitecture/GenerateArchitecture.js

@@ -130,7 +130,7 @@ define([
         var args = this.createArgString(layer),
             def = `nn.${layer.name}${args}`,
             type = layer.base.base.name,
-            addedIds,
+            memberIds,
             node,
             name,
             children,
@@ -141,28 +141,32 @@ define([
         // each nested architecture's code to the given container
         if (type === 'Container') {
             // Get the members of the 'addLayers' set
-            addedIds = {};
+            memberIds = {};
             id = layer[SimpleNodeConstants.NODE_PATH];
             node = this._nodeCache[id];
             this.core.getMemberPaths(node, Constants.CONTAINED_LAYER_SET)
-                .forEach(id => addedIds[id] = true);
+                .forEach(id => memberIds[id] = true);
 
             // Get the (sorted) children
             children = layer[SimpleNodeConstants.CHILDREN]
                 .map(child => {  // get (child, index) tuples
-                    var index;
+                    var index = null;
 
                     id = child[SimpleNodeConstants.NODE_PATH];
-                    index = this.core.getMemberRegistry(node, Constants.CONTAINED_LAYER_SET, id, Constants.CONTAINED_LAYER_INDEX);
+                    if (memberIds[id]) {
+                        index = this.core.getMemberRegistry(node,
+                            Constants.CONTAINED_LAYER_SET, id, Constants.CONTAINED_LAYER_INDEX);
+                    }
                     return [child, index];
                 })
-                .filter(pair => pair[1] !== undefined)  // remove non-members
+                .filter(pair => pair[1] !== null)  // remove non-members
                 .sort((a, b) => a[1] < b[1] ? -1 : 1)  // sort by 'index'
                 .map(pair => pair[0]);
 
 
             var addedLayerDefs = '',
                 firstLayer;
+
             for (var i = 0; i < children.length; i++) {
                 id = children[i][SimpleNodeConstants.NODE_PATH];
                 // Get the children!

src/seeds/pipeline/version.txt

@@ -1 +1 @@
-0.4.1
+0.6.0

src/visualizers/panels/ArchIndex/ArchIndexControl.js

@@ -0,0 +1,55 @@
+/* globals define */
+define([
+    'panels/PipelineIndex/PipelineIndexControl'
+], function(
+    PipelineIndexControl
+) {
+    var ArchIndexControl = function() {
+        PipelineIndexControl.apply(this, arguments);
+    };
+
+    ArchIndexControl.prototype = Object.create(PipelineIndexControl.prototype);
+    ArchIndexControl.prototype._getObjectDescriptor = function (nodeId) {
+        var node = this._client.getNode(nodeId),
+            base,
+            desc;
+
+        if (node) {
+            base = this._client.getNode(node.getBaseId());
+            desc = {
+                id: node.getId(),
+                name: node.getAttribute('name'),
+                parentId: node.getParentId(),
+                thumbnail: node.getAttribute('thumbnail'),
+                type: base.getAttribute('name')
+            };
+        }
+
+        return desc;
+    };
+
+    ArchIndexControl.prototype._initWidgetEventHandlers = function () {
+        this._widget.deletePipeline = id => {
+            var node = this._client.getNode(id),
+                name = node.getAttribute('name'),
+                msg = `Deleted "${name}" architecture`;
+
+
+            this._client.startTransaction(msg);
+            this._client.deleteNode(id);
+            this._client.completeTransaction();
+        };
+
+        this._widget.setName = (id, name) => {
+            var oldName = this._client.getNode(id).getAttribute('name'),
+                msg = `Renaming architecture: "${oldName}" -> "${name}"`;
+
+            if (oldName !== name && !/^\s*$/.test(name)) {
+                this._client.startTransaction(msg);
+                this._client.setAttribute(id, 'name', name);
+                this._client.completeTransaction();
+            }
+        };
+    };
+    return ArchIndexControl;
+});

src/visualizers/panels/ArchIndex/ArchIndexPanel.js

@@ -5,7 +5,7 @@ define([
     'js/PanelBase/PanelBaseWithHeader',
     'js/PanelManager/IActivePanel',
     'widgets/ArchIndex/ArchIndexWidget',
-    'panels/PipelineIndex/PipelineIndexControl'
+    'panels/ArchIndex/ArchIndexControl'
 ], function (
     PanelBaseWithHeader,
     IActivePanel,

src/visualizers/panels/ArtifactIndex/ArtifactIndexControl.js

@@ -80,16 +80,16 @@ define([
     // This next function retrieves the relevant node information for the widget
     ArtifactIndexControl.prototype._getObjectDescriptor = function (nodeId) {
         var node = this._client.getNode(nodeId),
-            base,
+            type,
             hash,
             objDescriptor;
 
         if (node) {
-            base = this._client.getNode(node.getBaseId());
+            type = this._client.getNode(node.getMetaTypeId());
             hash = node.getAttribute('data');
             objDescriptor = {
                 id: node.getId(),
-                type: base ? base.getAttribute('name') : 'n/a',
+                type: type ? type.getAttribute('name') : 'n/a',
                 name: node.getAttribute('name'),
                 createdAt: node.getAttribute('createdAt'),
                 dataURL: this.blobClient.getDownloadURL(hash),

src/visualizers/panels/ExecutionView/ExecutionViewControl.js

@@ -116,6 +116,7 @@ define([
                     allAttrs = {},
                     hiddenAttrs = [
                         CONSTANTS.LINE_OFFSET,
+                        CONSTANTS.DISPLAY_COLOR,
                         'code',
                         'name'
                     ],

src/visualizers/panels/ForgeActionButton/Actions.js

@@ -5,7 +5,8 @@ define([
     'panel/FloatingActionButton/styles/Materialize',
     'q',
     'js/RegistryKeys',
-    'deepforge/globals'
+    'deepforge/globals',
+    'deepforge/Constants'
 ], function(
     Materialize,
     Q,
@@ -266,29 +267,23 @@ define([
                 }
             },
             {
-                name: 'Export for local execution',
-                icon: 'play_for_work',
+                name: 'Export Pipeline',
+                icon: 'launch',
                 priority: -1,
                 action: function() {
-                    var pluginId = 'GenerateExecFile',
-                        context = this.client.getCurrentPluginContext(pluginId);
-
-                    // Run the plugin in the browser (set namespace)
-                    context.managerConfig.namespace = 'pipeline';
-                    context.pluginConfig = {};
-                    Q.ninvoke(this.client, 'runBrowserPlugin', pluginId, context)
-                        .then(res => {
-                            var id = this._currentNodeId,
-                                node = this.client.getNode(id),
-                                base = this.client.getNode(node.getBaseId()),
-                                type = base.getAttribute('name'),
-                                name = node.getAttribute('name');
-
-                            // Get the file and download it
-                            this.downloadFromBlob(res.artifacts[0]);
-                            Materialize.toast(`Exported ${name} ${type}!`, 2000);
+                    this.exportPipeline()
+                        .then(result => {
+                            Materialize.toast('Export successful!', 2000);
+                            // Download the result!
+                            this.downloadFromBlob(result.artifacts[0]);
+                            result.__unread = true;
+                            this.results.push(result);
+                            this._updatePluginBtns();
                         })
-                        .fail(err => Materialize.toast(`Export failed: ${err}`, 2000));
+                        .fail(err => {
+                            this.logger.warn('Pipeline export failed:', err);
+                            Materialize.toast(`Export failed: ${err}`, 4000);
+                        });
                 }
             }
         ],

src/visualizers/panels/ForgeActionButton/ConfigDialog.css

@@ -0,0 +1,5 @@
+.config-section-header {
+  margin-top: 0;
+  color: #888;
+  font-style: italic;
+}

src/visualizers/panels/ForgeActionButton/ConfigDialog.js

@@ -0,0 +1,222 @@
+/* globals define, $*/
+define([
+    'js/Dialogs/PluginConfig/PluginConfigDialog',
+    'text!js/Dialogs/PluginConfig/templates/PluginConfigDialog.html',
+    'plugin/Export/Export/format',
+    'css!./ConfigDialog.css'
+], function(
+    PluginConfigDialog,
+    pluginConfigDialogTemplate,
+    ExportFormats
+) {
+    var SECTION_DATA_KEY = 'section',
+        ATTRIBUTE_DATA_KEY = 'attribute',
+    //jscs:disable maximumLineLength
+        PLUGIN_CONFIG_SECTION_BASE = $('<div><fieldset><form class="form-horizontal" role="form"></form><fieldset></div>'),
+        ENTRY_BASE = $('<div class="form-group"><div class="row"><label class="col-sm-4 control-label">NAME</label><div class="col-sm-8 controls"></div></div><div class="row description"><div class="col-sm-4"></div></div></div>'),
+    //jscs:enable maximumLineLength
+        DESCRIPTION_BASE = $('<div class="desc muted col-sm-8"></div>'),
+        SECTION_HEADER = $('<h6 class="config-section-header">');
+
+    var ConfigDialog = function(client, nodeId) {
+        PluginConfigDialog.call(this, {client: client});
+        this._widgets = {};
+        this._node = this._client.getNode(nodeId);
+    };
+
+    ConfigDialog.prototype = Object.create(PluginConfigDialog.prototype);
+
+    ConfigDialog.prototype.getFormatOptions = function() {
+        this._exportFormats = Object.keys(ExportFormats);
+
+        this._formatOptions = [];
+        if (this._exportFormats.length > 1) {
+            this._formatOptions.push({  // format options
+                name: 'exportFormat',
+                displayName: 'Export Format',
+                valueType: 'string',
+                value: this._exportFormats[0],
+                valueItems: this._exportFormats,
+                readOnly: false
+            });
+        }
+    };
+
+    ConfigDialog.prototype.show = function(pluginMetadata, callback) {
+        this._pluginMetadata = pluginMetadata;
+
+        this.getFormatOptions();
+        this._initDialog();
+
+        this._dialog.on('shown', () => {
+            this._dialog.find('input').first().focus();
+        });
+
+        this._btnSave.on('click', event => {
+            this.submit(callback);
+            event.stopPropagation();
+            event.preventDefault();
+        });
+
+        //save&run on CTRL + Enter
+        this._dialog.on('keydown.PluginConfigDialog', event => {
+            if (event.keyCode === 13 && (event.ctrlKey || event.metaKey)) {
+                event.stopPropagation();
+                event.preventDefault();
+                this.submit(callback);
+            }
+        });
+        this._dialog.modal('show');
+    };
+
+    ConfigDialog.prototype._initDialog = function() {
+        this._dialog = $(pluginConfigDialogTemplate);
+
+        this._btnSave = this._dialog.find('.btn-save');
+        this._divContainer = this._dialog.find('.modal-body');
+        this._saveConfigurationCb = this._dialog.find('.save-configuration');
+        this._modalHeader = this._dialog.find('.modal-header');
+
+        // Create the header
+        var iconEl = $('<i/>', {
+            class: this._pluginMetadata.icon.class || 'glyphicon glyphicon-cog'
+        });
+        iconEl.addClass('plugin-icon pull-left');
+        this._modalHeader.prepend(iconEl);
+        this._title = this._modalHeader.find('.modal-title');
+        this._title.text(this._pluginMetadata.id + ' ' + 'v' + this._pluginMetadata.version);
+
+        // Generate the config options
+        var sectionHeader = SECTION_HEADER.clone();
+
+        sectionHeader.text('Static Artifacts');
+        this._divContainer.append(sectionHeader);
+        this.generateConfigSection(this._pluginMetadata);
+
+        if (this._exportFormats.length > 1) {
+            this._divContainer.append($('<hr class="extension-config-divider">'));
+            sectionHeader = SECTION_HEADER.clone();
+            sectionHeader.text('Export Options');
+            this._divContainer.append(sectionHeader);
+
+            this.generateConfigSection({
+                id: 'FormatOptions',
+                configStructure: this._formatOptions
+            });
+            this._widgets.FormatOptions.exportFormat.el.find('select').on('change', event => {
+                var format = event.target.value;
+                // Update the ext config
+                this.updateExtConfig(format);
+            });
+        }
+
+        this.updateExtConfig(this._exportFormats[0]);
+    };
+
+    ConfigDialog.prototype.submit = function (callback) {
+        var config = this._getAllConfigValues();
+        this._dialog.modal('hide');
+
+        if (this._exportFormats.length === 1) {
+            config.FormatOptions = {
+                exportFormat: this._exportFormats[0]
+            };
+        }
+        return callback(config);
+    };
+
+    ConfigDialog.prototype._getAllConfigValues = function () {
+        var settings = {};
+
+        Object.keys(this._widgets).forEach(namespace => {
+            settings[namespace] = {};
+
+            Object.keys(this._widgets[namespace]).forEach(name => {
+                settings[namespace][name] = this._widgets[namespace][name].getValue();
+            });
+        });
+
+        return settings;
+    };
+
+    ConfigDialog.prototype.updateExtConfig = function (format) {
+        var extConfig = {
+            id: 'extensionConfig',
+            class: 'extension-config',
+            configStructure: ExportFormats[format].getConfigStructure ?
+                ExportFormats[format].getConfigStructure(this._node, this._client) : []
+        };
+        this._divContainer.find('.extension-config').remove();
+
+        if (extConfig.configStructure.length) {
+            this.generateConfigSection(extConfig);
+        }
+    };
+
+    ConfigDialog.prototype.generateConfigSection = function (metadata) {
+        var len = metadata.configStructure.length,
+            i,
+            el,
+            pluginConfigEntry,
+            widget,
+            descEl,
+            containerEl,
+            pluginSectionEl = PLUGIN_CONFIG_SECTION_BASE.clone();
+
+        pluginSectionEl.data(SECTION_DATA_KEY, metadata.id);
+        this._divContainer.append(pluginSectionEl);
+        containerEl = pluginSectionEl.find('.form-horizontal');
+
+        if (metadata.class) {
+            pluginSectionEl.addClass(metadata.class);
+        }
+
+        this._widgets[metadata.id] = {};
+        for (i = 0; i < len; i += 1) {
+            pluginConfigEntry = metadata.configStructure[i];
+            descEl = undefined;
+
+            // Make sure not modify the global metadata.
+            pluginConfigEntry = JSON.parse(JSON.stringify(pluginConfigEntry));
+            if (this._client.getProjectAccess().write === false && pluginConfigEntry.writeAccessRequired === true) {
+                pluginConfigEntry.readOnly = true;
+            }
+
+            widget = this._propertyGridWidgetManager.getWidgetForProperty(pluginConfigEntry);
+            this._widgets[metadata.id][pluginConfigEntry.name] = widget;
+
+            el = ENTRY_BASE.clone();
+            el.data(ATTRIBUTE_DATA_KEY, pluginConfigEntry.name);
+
+            el.find('label.control-label').text(pluginConfigEntry.displayName);
+
+            if (pluginConfigEntry.description && pluginConfigEntry.description !== '') {
+                descEl = descEl || DESCRIPTION_BASE.clone();
+                descEl.text(pluginConfigEntry.description);
+            }
+
+            if (pluginConfigEntry.minValue !== undefined &&
+                pluginConfigEntry.minValue !== null &&
+                pluginConfigEntry.minValue !== '') {
+                descEl = descEl || DESCRIPTION_BASE.clone();
+                descEl.append(' The minimum value is: ' + pluginConfigEntry.minValue + '.');
+            }
+
+            if (pluginConfigEntry.maxValue !== undefined &&
+                pluginConfigEntry.maxValue !== null &&
+                pluginConfigEntry.maxValue !== '') {
+                descEl = descEl || DESCRIPTION_BASE.clone();
+                descEl.append(' The maximum value is: ' + pluginConfigEntry.maxValue + '.');
+            }
+
+            el.find('.controls').append(widget.el);
+            if (descEl) {
+                el.find('.description').append(descEl);
+            }
+
+            containerEl.append(el);
+        }
+    };
+
+    return ConfigDialog;
+});

src/visualizers/panels/ForgeActionButton/ForgeActionButton.js

@@ -1,9 +1,10 @@
-/*globals $, window, define, _ */
+/*globals $, window, define, _, WebGMEGlobal */
 /*jshint browser: true*/
 
 define([
     'blob/BlobClient',
     'js/Utils/SaveToDisk',
+    './ConfigDialog',
     'js/Constants',
     'panel/FloatingActionButton/FloatingActionButton',
     'deepforge/viz/PipelineControl',
@@ -14,11 +15,14 @@ define([
     'js/RegistryKeys',
     'js/Panels/MetaEditor/MetaEditorConstants',
     'q',
-    'deepforge/globals'
+    'deepforge/globals',
+    'deepforge/Constants',
+    'plugin/Export/Export/format'
 ], function (
     BlobClient,
     SaveToDisk,
-    CONSTANTS,
+    ConfigDialog,
+    GME_CONSTANTS,
     PluginButton,
     PipelineControl,
     NodePrompter,
@@ -28,7 +32,9 @@ define([
     REGISTRY_KEYS,
     META_CONSTANTS,
     Q,
-    DeepForge
+    DeepForge,
+    Constants,
+    ExportFormatDict
 ) {
     'use strict';
 
@@ -142,19 +148,19 @@ define([
 
     // Helper functions REMOVE! FIXME
     ForgeActionButton.prototype.addToMetaSheet = function(nodeId, metasheetName) {
-        var root = this.client.getNode(CONSTANTS.PROJECT_ROOT_ID),
+        var root = this.client.getNode(GME_CONSTANTS.PROJECT_ROOT_ID),
             metatabs = root.getRegistry(REGISTRY_KEYS.META_SHEETS),
             metatab = metatabs.find(tab => tab.title === metasheetName) || metatabs[0],
             metatabId = metatab.SetID;
 
         // Add to the general meta
         this.client.addMember(
-            CONSTANTS.PROJECT_ROOT_ID,
+            GME_CONSTANTS.PROJECT_ROOT_ID,
             nodeId,
             META_CONSTANTS.META_ASPECT_SET_NAME
         );
         this.client.setMemberRegistry(
-            CONSTANTS.PROJECT_ROOT_ID,
+            GME_CONSTANTS.PROJECT_ROOT_ID,
             nodeId,
             META_CONSTANTS.META_ASPECT_SET_NAME,
             REGISTRY_KEYS.POSITION,
@@ -165,9 +171,9 @@ define([
         );
 
         // Add to the specific sheet
-        this.client.addMember(CONSTANTS.PROJECT_ROOT_ID, nodeId, metatabId);
+        this.client.addMember(GME_CONSTANTS.PROJECT_ROOT_ID, nodeId, metatabId);
         this.client.setMemberRegistry(
-            CONSTANTS.PROJECT_ROOT_ID,
+            GME_CONSTANTS.PROJECT_ROOT_ID,
             nodeId,
             metatabId,
             REGISTRY_KEYS.POSITION,
@@ -363,18 +369,131 @@ define([
     };
 
     ForgeActionButton.prototype.downloadFromBlob = function(hash) {
-        var name;
-
         this._blobClient.getMetadata(hash)
             .then(metadata => {
-                name = metadata.name;
-                return this._blobClient.getObjectAsString(hash);
-            })
-            .then(text => {
-                SaveToDisk.downloadTextAsFile(name, text);
+                var url = this._blobClient.getDownloadURL(hash),
+                    name = metadata.name,
+                    save = document.createElement('a');
+
+                save.href = url;
+                save.target = '_self';
+                save.download = name;
+
+                save.click();
+                (window.URL || window.webkitURL).revokeObjectURL(save.href);
             })
             .fail(err => this.logger.error(`Blob download failed: ${err}`));
     };
 
+    /// Export Pipeline Support
+    ForgeActionButton.prototype.exportPipeline = function() {
+        var deferred = Q.defer(),
+            pluginId = 'Export',
+            metadata = WebGMEGlobal.allPluginsMetadata[pluginId],
+            id = this._currentNodeId,
+            node = this.client.getNode(id),
+            inputData,
+            inputNames;
+
+        inputData = node.getChildrenIds()
+            .map(id => this.client.getNode(id))
+            .filter(node => {
+                var typeId = node.getMetaTypeId(),
+                    type = this.client.getNode(typeId).getAttribute('name');
+
+                return type === Constants.OP.INPUT;
+            })
+            .map(input => {
+                var outputCntr,
+                    outputIds;
+
+                outputCntr = input.getChildrenIds()
+                    .map(id => this.client.getNode(id))
+                    .find(node => {
+                        var typeId = node.getMetaTypeId(),
+                            type = this.client.getNode(typeId).getAttribute('name');
+                        return type === 'Outputs';
+                    });
+
+                // input operations only have a single output
+                outputIds = outputCntr.getChildrenIds();
+
+                if (outputIds.length === 1) {
+                    return outputIds[0];
+                } else if (outputIds.length > 1) {
+                    this.logger.warn(`Found multiple ids for input op: ${outputIds.join(', ')}`);
+                    return;
+                }
+            })
+            .filter(outputId => !!outputId)
+            .map(id => this.client.getNode(id))
+            .filter(output => output.getAttribute('data'));
+
+        // get the name of node referenced from the input op
+        inputNames = inputData
+            .map(node => {
+                var cntrId = node.getParentId(),
+                    opId = this._client.getNode(cntrId).getParentId(),
+                    inputOp = this._client.getNode(opId),
+                    targetNodeId = inputOp.getPointer('artifact').to;
+
+                return this._client.getNode(targetNodeId).getAttribute('name');
+            })
+            .sort();
+
+        // create config options from inputs
+        var inputOpts = inputNames.map((input, index) => {
+            return {
+                name: inputData[index].getId(),
+                displayName: input,
+                description: `Export ${input} as static (non-input) content`,
+                value: false,
+                valueType: 'boolean',
+                readOnly: false
+            };
+        });
+
+        var exportFormats = Object.keys(ExportFormatDict),
+            configDialog = new ConfigDialog(this.client, this._currentNodeId),
+            inputConfig = _.extend({}, metadata);
+
+        inputConfig.configStructure = inputOpts;
+
+        // Try to get the extension options
+        if (inputOpts.length || exportFormats.length > 1) {
+            configDialog.show(inputConfig, (allConfigs) => {
+                var context = this.client.getCurrentPluginContext(pluginId),
+                    exportFormat = allConfigs.FormatOptions.exportFormat,
+                    staticInputs = Object.keys(allConfigs[pluginId]).filter(input => allConfigs[pluginId][input]);
+
+                this.logger.debug('Exporting pipeline to format', exportFormat);
+                this.logger.debug('static inputs:', staticInputs);
+
+                context.managerConfig.namespace = 'pipeline';
+                context.pluginConfig = {
+                    format: exportFormat,
+                    staticInputs: staticInputs,
+                    extensionConfig: allConfigs.extensionConfig
+                };
+                return Q.ninvoke(this.client, 'runBrowserPlugin', pluginId, context)
+                    .then(deferred.resolve)
+                    .fail(deferred.reject);
+            });
+        } else {  // no options - just run the plugin!
+            var context = this.client.getCurrentPluginContext(pluginId);
+
+            this.logger.debug('Exporting pipeline to format', exportFormats[0]);
+
+            context.managerConfig.namespace = 'pipeline';
+            context.pluginConfig = {
+                format: exportFormats[0],
+                staticInputs: []
+            };
+            return Q.ninvoke(this.client, 'runBrowserPlugin', pluginId, context);
+        }
+
+        return deferred.promise;
+    };
+
     return ForgeActionButton;
 });

src/visualizers/panels/LineGraph/LineGraphControl.js

@@ -99,6 +99,9 @@ define([
 
                 desc.type = 'line';
                 desc.points = points;
+            } else {
+                desc.xlabel = node.getAttribute('xlabel');
+                desc.ylabel = node.getAttribute('ylabel');
             }
         }
 

src/visualizers/panels/OperationInterfaceEditor/OperationInterfaceEditorControl.js

@@ -155,6 +155,8 @@ define([
             // Remove DeepForge hidden attributes
             delete desc.attributes.code;
             delete desc.attributes[CONSTANTS.LINE_OFFSET];
+            desc.displayColor = desc.attributes[CONSTANTS.DISPLAY_COLOR].value;
+            delete desc.attributes[CONSTANTS.DISPLAY_COLOR];
         }
 
         // Extra decoration for data

src/visualizers/panels/PipelineEditor/PipelineEditorControl.js

@@ -32,9 +32,7 @@ define([
             SRC: 'src',
             DST: 'dst'
         },
-        DECORATORS = {
-            ArtifactFinder: 'ArtifactOpDecorator'
-        },
+        DECORATORS = {},
         WIDGET_NAME = 'EasyDAG';
 
     DECORATORS[CONSTANTS.OP.INPUT] = 'ArtifactOpDecorator';

src/visualizers/panels/PipelineIndex/PipelineIndexControl.js

@@ -103,11 +103,11 @@ define([
     PipelineIndexControl.prototype._getObjectDescriptor = function (nodeId) {
         var node = this._client.getNode(nodeId),
             base,
-            objDescriptor;
+            desc;
 
         if (node) {
             base = this._client.getNode(node.getBaseId());
-            objDescriptor = {
+            desc = {
                 id: node.getId(),
                 name: node.getAttribute(nodePropertyNames.Attributes.name),
                 parentId: node.getParentId(),
@@ -115,10 +115,9 @@ define([
                 type: base.getAttribute('name'),
                 executionCount: node.getMemberIds('executions').length
             };
-
         }
 
-        return objDescriptor;
+        return desc;
     };
 
     /* * * * * * * * Node Event Handling * * * * * * * */

src/visualizers/widgets/ExecutionView/SelectionManager.js

@@ -23,7 +23,9 @@ define([
         var jobName = this.selectedItem.desc.name;
 
         // Check if it is an Input or Output job
-        if (jobName !== CONSTANTS.OP.INPUT && jobName !== CONSTANTS.OP.OUTPUT) {
+        if (!this.selectedItem.isConnection && jobName !== CONSTANTS.OP.INPUT &&
+            jobName !== CONSTANTS.OP.OUTPUT) {
+
             new Buttons.Enter({
                 context: this._widget,
                 $pEl: this.$selection,

src/visualizers/widgets/LineGraph/LineGraphWidget.js

@@ -42,17 +42,16 @@ define([
 
             chart.xAxis
                 .tickFormat(d3.format(',r'));
-
-            if (this.options.xAxis) {
-                chart.xAxis
-                    .axisLabel(this.options.xAxis);
-            }
-
             chart.yAxis.tickFormat(d3.format('.02f'));
+
             if (this.options.yAxis) {
                 chart.yAxis
                     .axisLabel(this.options.yAxis);
             }
+            if (this.options.xAxis) {
+                chart.xAxis
+                    .axisLabel(this.options.xAxis);
+            }
 
             var myData = this.getData();
 
@@ -83,6 +82,9 @@ define([
                     key: desc.name,
                     values: desc.points
                 };
+            } else {
+                this.options.xAxis = desc.xlabel;
+                this.options.yAxis = desc.ylabel;
             }
             this.refreshChart();
         }
@@ -94,11 +96,14 @@ define([
     };
 
     LineGraphWidget.prototype.updateNode = function (desc) {
-        if (desc && this.lineData[desc.id]) {
+        if (this.lineData[desc.id]) {
             this.lineData[desc.id].values = desc.points;
             this.lineData[desc.id].key = desc.name;
-            this.refreshChart();
+        } else {
+            this.options.xAxis = desc.xlabel;
+            this.options.yAxis = desc.ylabel;
         }
+        this.refreshChart();
     };
 
     LineGraphWidget.prototype.onWidgetContainerResize = function(width, height) {
@@ -111,6 +116,17 @@ define([
 
     LineGraphWidget.prototype.updateChartData = function () {
         if (this.$chart && this.chart) {
+
+            if (this.options.yAxis) {
+                this.chart.yAxis
+                    .axisLabel(this.options.yAxis || '');
+            }
+            if (this.options.xAxis) {
+                this.chart.xAxis
+                    .axisLabel(this.options.xAxis || '');
+            }
+
+
             this.$chart
                 .datum(this.getData())
                 .call(this.chart);

src/visualizers/widgets/OperationInterfaceEditor/Buttons.js

@@ -1,16 +1,23 @@
 /*globals define*/
 define([
     'deepforge/viz/Buttons',
+    'deepforge/Constants',
     'widgets/EasyDAG/Buttons',
     'widgets/EasyDAG/Icons',
-    'underscore'
+    'underscore',
+    './lib/spectrum.min'
 ], function(
     CommonButtons,
+    Constants,
     EasyDAGButtons,
     Icons,
     _
 ) {
 
+    var COLOR_PALETTE = [
+        ['#78909c', '#ce93d8', '#ff9100', '#ffb74d', '#ffe0b2'],
+        ['#42a5f5', '#80deea', '#80cbc4', '#a5d6a7', '#69f0ae']
+    ];
     var AddOutput = function(params) {
         params.title = params.title || 'Add operation output';
         EasyDAGButtons.Add.call(this, params);
@@ -89,9 +96,54 @@ define([
         this.selectionManager.deselect();
     };
 
+    // Set the color
+    var SetColor = function(params) {
+        params.title = params.title || 'Set operation color';
+        EasyDAGButtons.Add.call(this, params);
+
+        // Add the click handling
+        var currentColor = this.item.desc.displayColor;
+        $('.set-color-icon').spectrum({
+            change: color => this.onColorChanged(color.toHexString()),
+            showPaletteOnly: true,
+            showPalette: true,
+            clickoutFiresChange: true,
+            hideAfterPaletteSelect: true,
+
+            color: currentColor,
+            palette: COLOR_PALETTE
+        });
+    };
+    _.extend(SetColor.prototype, EasyDAGButtons.Add.prototype);
+
+    SetColor.prototype.BTN_CLASS = 'set-color-icon';
+    SetColor.prototype._onClick = function() {};
+
+    SetColor.prototype.onColorChanged = function(color) {
+        // Set the displayColor attribute to the given hex value
+        this.context.saveAttributeForNode(this.item.id, Constants.DISPLAY_COLOR, color);
+    };
+
+    SetColor.prototype._render = function() {
+        var lineRadius = EasyDAGButtons.Add.SIZE - EasyDAGButtons.Add.BORDER,
+            btnColor = '#ffcc80';
+
+        if (this.disabled) {
+            btnColor = '#e0e0e0';
+        }
+
+        this.$el
+            .append('circle')
+            .attr('r', EasyDAGButtons.Add.SIZE)
+            .attr('fill', btnColor);
+
+        Icons.addIcon('brush', this.$el, {radius: lineRadius});
+    };
+
     return {
         AddOutput: AddOutput,
         AddInput: AddInput,
+        SetColor: SetColor,
         AddRef: AddRef,
         GoToBase: CommonButtons.GoToBase,
         Delete: Delete

src/visualizers/widgets/OperationInterfaceEditor/Item.js

@@ -9,6 +9,8 @@ define([
     
     var Item = function(parentEl, desc) {
         DAGItem.call(this, parentEl, desc);
+        this.decorator.color = desc.displayColor || this.decorator.color;
+
 
         // Show the warnings
         this.$warning = null;
@@ -19,6 +21,7 @@ define([
     _.extend(Item.prototype, DAGItem.prototype);
     
     Item.prototype.update = function(desc) {
+        this.decorator.color = desc.displayColor || this.decorator.color;
         DAGItem.prototype.update.call(this, desc);
         this.updateWarnings();
     };

src/visualizers/widgets/OperationInterfaceEditor/SelectionManager.js

@@ -17,6 +17,12 @@ define([
 
     _.extend(SelectionManager.prototype, EasyDAGSelectionManager.prototype);
 
+    SelectionManager.prototype.deselect = function() {
+        // this would be better in a 'destroy' method...
+        $('.set-color-icon').spectrum('hide');
+        EasyDAGSelectionManager.prototype.deselect.call(this);
+    };
+
     SelectionManager.prototype.createActionButtons = function(width, height) {
         var selectedType = this.selectedItem.desc.baseName,
             dataNodes,
@@ -53,6 +59,14 @@ define([
                 x: 2*width/3,
                 y: 0
             });
+
+            new Buttons.SetColor({  // Set the operation color
+                context: this._widget,
+                $pEl: this.$selection,
+                item: this.selectedItem,
+                x: 0,
+                y: height
+            });
         } else {  // Data or pointer...
             new Buttons.Delete({
                 context: this._widget,

src/visualizers/widgets/PipelineEditor/OperationNode.js

@@ -9,7 +9,10 @@ define([
 
     'use strict';
     var OperationNode = function(parentEl, desc) {
-        DAGItem.call(this, parentEl, desc);
+        var decoratorOpts = {
+            color: desc.displayColor
+        };
+        DAGItem.call(this, parentEl, desc, decoratorOpts);
         this.inputs = desc.inputs;
         this.outputs = desc.outputs;
         this._visiblePorts = null;
@@ -122,8 +125,10 @@ define([
         this.hidePorts();
     };
 
-    OperationNode.prototype.update = function() {
+    OperationNode.prototype.update = function(desc) {
         DAGItem.prototype.update.apply(this, arguments);
+        this.inputs = desc.inputs;
+        this.outputs = desc.outputs;
         if (this._visiblePorts) {
             var areInputs = this._visiblePorts[1];
             this.showPorts.call(this, null, areInputs);

src/visualizers/widgets/PipelineEditor/PipelineEditorWidget.js

@@ -64,6 +64,17 @@ define([
         return 'PipelineEditor';
     };
 
+    PipelineEditorWidget.prototype.onCreateInitialNode = function() {
+        var initialNodes = this.getValidInitialNodes().map(node => {
+            var colorAttr = node.attributes[CONSTANTS.DISPLAY_COLOR];
+            node.decoratorOpts = {color: colorAttr && colorAttr.value};
+            return {node};
+        });
+
+        AddNodeDialog.prompt(initialNodes)
+            .then(selected => this.createNode(selected.node.id));
+    };
+
     PipelineEditorWidget.prototype.setupItemCallbacks = function() {
         ThumbnailWidget.prototype.setupItemCallbacks.call(this);
         this.ItemClass.prototype.connectPort =
@@ -295,7 +306,6 @@ define([
             this.$execContent.empty();
             execs.forEach(html => this.$execContent.append(html));
 
-            this.$execContent.height(200);
             this.$execBody.show();
         } else {
             // Set the height to 0

src/visualizers/widgets/PipelineIndex/PipelineIndexWidget.js

@@ -145,7 +145,9 @@ define([
 
     PipelineIndexWidget.prototype.updateNode = function (desc) {
         if (desc && this.cards[desc.id]) {
-            this.cards[desc.id].outerHTML = PipelineTemplate(desc);
+            var Template = this.getCardTemplate(desc);
+
+            this.cards[desc.id].outerHTML = Template(desc);
             // Check if the preview changed
             if (desc.thumbnail !== this.nodes[desc.id].thumbnail) {
                 this.addThumbnail(desc.thumbnail, this.cards[desc.id]);

test/cli/cli.spec.js

@@ -81,12 +81,37 @@ describe('cli', function() {
             cli = require('../../bin/deepforge');
         });
 
-        it('should check for running mongo', function() {
-            var calls;
-            callRegister.childProcess.execSync = [];
-            cli('start');
-            calls = callRegister.childProcess.execSync;
-            assert.notEqual(calls.indexOf('pgrep mongod'), -1);
+        it('should check for running mongo', function(done) {
+            var mongoListening = false,
+                mongoUri = 'mongodb://127.0.0.1:2016/deepforge-test',
+                net = require('net'),
+                server = net.createServer(function(socket) {
+                    socket.on('error', err => {
+                        // Only worry about mock server errors if the test hasn't completed
+                        assert(mongoListening);
+                    });
+                    socket.pipe(socket);
+                }),
+                mockStartMongo = function(port) {
+                    server.listen(+port, '127.0.0.1');
+                    mongoListening = true;
+                };
+
+            // Check that 'spawn' node happens after the tcp port has been bound
+            mocks.childProcess.spawn = function(cmd, opts) {
+                if (cmd === 'mongod') {
+                    setTimeout(mockStartMongo, 250, opts[3]);
+                }
+            };
+
+            cli.checkMongo({}, false, mongoUri)
+                .then(() => {
+                    console.log('closing server');
+                    server.close();
+                    assert(mongoListening);
+                    done();
+                })
+                .catch(err => console.error(err));
         });
 
         it('should start mongo if no running mongo', function() {
@@ -181,9 +206,11 @@ describe('cli', function() {
         });
 
         it('should update deepforge from git if --git set w/ npm', function() {
+            var repo = require('../../package.json').repository.url;
             mocks.childProcess.spawn = (cmd, args) => {
+                // check for the git repo
                 if (cmd === 'npm') {
-                    assert.notEqual(args.indexOf('dfst/deepforge'), -1);
+                    assert.notEqual(args.indexOf(repo), -1);
                 }
             };
             cli('update --git');

test/plugins/Export/Export.spec.js

@@ -1,7 +1,7 @@
 /*jshint node:true, mocha:true*/
 
 'use strict';
-describe('GenerateExecFile', function () {
+describe('Export', function () {
     var testFixture = require('../../globals'),
         lua = require('../../../src/common/lua'),
         path = testFixture.path,
@@ -9,12 +9,12 @@ describe('GenerateExecFile', function () {
         SEED_DIR = path.join(testFixture.DF_SEED_DIR, 'devProject'),
         gmeConfig = testFixture.getGmeConfig(),
         expect = testFixture.expect,
-        logger = testFixture.logger.fork('GenerateExecFile'),
+        logger = testFixture.logger.fork('Export'),
         PluginCliManager = testFixture.WebGME.PluginCliManager,
         manager = new PluginCliManager(null, logger, gmeConfig),
         BlobClient = require('webgme/src/server/middleware/blob/BlobClientWithFSBackend'),
         projectName = 'testProject',
-        pluginName = 'GenerateExecFile',
+        pluginName = 'Export',
         project,
         gmeAuth,
         storage,

test/plugins/GenerateArchitecture/GenerateArchitecture.spec.js

@@ -90,6 +90,7 @@ describe('GenerateArchitecture', function () {
             //['/8', 'basic-transfers.lua'],
             //['/M', 'concat-parallel.lua'],
             ['/Q', 'basiccontainer.lua'],
+            ['/P', 'ContainerWithLayerArgs.lua'],
             ['/4', 'requiredOmitted.lua'],
             ['/e', 'googlenet.lua'],
             ['/X', 'overfeat.lua']

test/test-cases/generated-code/ContainerWithLayerArgs.lua

@@ -0,0 +1,13 @@
+require 'nn'
+require 'rnn'
+
+local m = nn.Sequential()
+m:add(nn.Linear(100, 50))
+m:add(nn.LeakyReLU())
+m:add(nn.Linear(50, 10))
+
+
+local net = nn.Sequential()
+net:add(nn.Bottle(m, 2))
+
+return net

utils/extender.js

@@ -0,0 +1,199 @@
+// Utility for applying and removing deepforge extensions
+// This utility is run by the cli when executing:
+//
+//     deepforge extensions add <project>
+//     deepforge extensions remove <name>
+//
+var path = require('path'),
+    fs = require('fs'),
+    npm = require('npm'),
+    Q = require('q'),
+    rm_rf = require('rimraf'),
+    exists = require('exists-file'),
+    makeTpl = require('lodash.template'),
+    CONFIG_DIR = path.join(process.env.HOME, '.deepforge'),
+    EXT_CONFIG_NAME = 'extension.json',
+    EXTENSION_REGISTRY_NAME = 'extensions.json',
+    extConfigPath = path.join(CONFIG_DIR, EXTENSION_REGISTRY_NAME),
+    allExtConfigs;
+
+var values = obj => Object.keys(obj).map(key => obj[key]);
+
+// Create the extensions.json if doesn't exist. Otherwise, load it
+if (!exists.sync(extConfigPath)) {
+    allExtConfigs = {};
+} else {
+    try {
+        allExtConfigs = JSON.parse(fs.readFileSync(extConfigPath, 'utf8'));
+    } catch (e) {
+        throw `Invalid config at ${extConfigPath}: ${e.toString()}`;
+    }
+}
+
+var persistExtConfig = () => {
+    fs.writeFileSync(extConfigPath, JSON.stringify(allExtConfigs, null, 2));
+};
+
+var extender = {};
+
+extender.EXT_CONFIG_NAME = EXT_CONFIG_NAME;
+
+extender.isSupportedType = function(type) {
+    return extender.install[type] && extender.uninstall[type];
+};
+
+extender.getExtensionsConfig = function() {
+    return allExtConfigs;
+};
+
+extender.getInstalledConfig = function(name) {
+    var group = values(allExtConfigs).find(typeGroup => {
+        return !!typeGroup[name];
+    });
+    return group && group[name];
+};
+
+extender.install = function(project, isReinstall) {
+    // Install the project
+    return Q.ninvoke(npm, 'load', {})
+        .then(() => Q.ninvoke(npm, 'install', project))
+        .then(results => {
+            var installed = results[0],
+                extProject,
+                extRoot;
+
+            extProject = installed[0][0];
+            extRoot = installed[0][1];
+
+            // Check for the extensions.json in the project (look up type, etc)
+            var extConfigPath = path.join(extRoot, extender.EXT_CONFIG_NAME),
+                extConfig,
+                extType;
+
+            // Check that the extensions file exists
+            if (!exists.sync(extConfigPath)) {
+                throw [
+                    `Could not find ${extender.EXT_CONFIG_NAME} for ${project}.`,
+                    '',
+                    `This is likely an issue w/ the deepforge extension (${project})`
+                ].join('\n');
+            }
+
+            try {
+                extConfig = JSON.parse(fs.readFileSync(extConfigPath, 'utf8'));
+            } catch(e) {  // Invalid JSON
+                throw `Invalid ${extender.EXT_CONFIG_NAME}: ${e}`;
+            }
+
+            // Try to add the extension to the project (using the extender)
+            extType = extConfig.type;
+            if (!extender.isSupportedType(extType)) {
+                throw `Unrecognized extension type: "${extType}"`;
+            }
+            extender.install[extType](extConfig, {
+                arg: project,
+                root: extRoot,
+                name: extProject
+            }, !!isReinstall);
+
+            return extConfig;
+        });
+};
+
+extender.uninstall = function(name) {
+    // Look up the extension in ~/.deepforge/extensions.json
+    var extConfig = extender.getInstalledConfig(name);
+    if (!extConfig) {
+        throw `Extension "${name}" not found`;
+    }
+
+    // Run the uninstaller using the extender
+    var extType = extConfig.type;
+    extender.uninstall[extType](name);
+};
+
+var makeInstallFor = function(typeCfg) {
+    var saveExtensions = () => {
+        // regenerate the format.js file from the template
+        var installedExts = values(allExtConfigs[typeCfg.type]),
+            formatTemplate = makeTpl(fs.readFileSync(typeCfg.template, 'utf8')),
+            formatsIndex = formatTemplate({path: path, formats: installedExts}),
+            dstPath = typeCfg.template.replace(/\.ejs$/, '');
+
+        fs.writeFileSync(dstPath, formatsIndex);
+        persistExtConfig();
+    };
+
+    // Given a...
+    //  - template file
+    //  - extension type
+    //  - target path tpl
+    // create the installation/uninstallation functions
+    extender.install[typeCfg.type] = (config, project, isReinstall) => {
+        var dstPath,
+            pkgJsonPath = path.join(project.root, 'package.json'),
+            pkgJson = JSON.parse(fs.readFileSync(pkgJsonPath, 'utf8')),
+            content;
+
+        // add the config to the current installed extensions of this type
+        project = project || config.project;
+        config.version = pkgJson.version;
+        config.project = project;
+
+        allExtConfigs[typeCfg.type] = allExtConfigs[typeCfg.type] || {};
+
+        if (allExtConfigs[typeCfg.type][config.name] && !isReinstall) {
+            // eslint-disable-next-line no-console
+            console.error(`Extension ${config.name} already installed. Reinstalling...`);
+        }
+
+        allExtConfigs[typeCfg.type][config.name] = config;
+
+        // copy the main script to src/plugins/Export/formats/<name>/<main>
+        dstPath = makeTpl(typeCfg.targetDir)(config);
+        if (!exists.sync(dstPath)) {
+            fs.mkdirSync(dstPath);
+        }
+
+        try {
+            content = fs.readFileSync(path.join(project.root, config.main), 'utf8');
+        } catch (e) {
+            throw 'Could not read the extension\'s main file: ' + e;
+        }
+        dstPath = path.join(dstPath, path.basename(config.main));
+        fs.writeFileSync(dstPath, content);
+
+        saveExtensions();
+    };
+
+    // uninstall
+    extender.uninstall['Export:Pipeline'] = name => {
+        // Remove from config
+        allExtConfigs[typeCfg.type] = allExtConfigs[typeCfg.type] || {};
+
+        if (!allExtConfigs[typeCfg.type][name]) {
+            // eslint-disable-next-line no-console
+            console.log(`Extension ${name} not installed`);
+            return;
+        }
+        var config = allExtConfigs[typeCfg.type][name],
+            dstPath = makeTpl(typeCfg.targetDir)(config);
+
+        // Remove the dstPath
+        delete allExtConfigs[typeCfg.type][name];
+        rm_rf.sync(dstPath);
+
+        // Re-generate template file
+        saveExtensions();
+    };
+
+};
+
+var PLUGIN_ROOT = path.join(__dirname, '..', 'src', 'plugins', 'Export');
+makeInstallFor({
+    type: 'Export:Pipeline',
+    template: path.join(PLUGIN_ROOT, 'format.js.ejs'),
+    targetDir: path.join(PLUGIN_ROOT, 'formats', '<%=name%>')
+});
+
+module.exports = extender;

utils/reinstall-extensions.js

@@ -0,0 +1,30 @@
+// Re-install all extensions
+var extender = require('./extender'),
+    Q = require('q'),
+    extConfig = extender.getExtensionsConfig(),
+    types,
+    names,
+    currentInstall = Q(),
+    installCount = 0,
+    config;
+
+// Read the extensions and reinstall each of them
+types = Object.keys(extConfig);
+for (var i = types.length; i--;) {
+    names = Object.keys(extConfig[types[i]]);
+    if (names.length) {
+        installCount += names.length;
+        for (var j = names.length; j--;) {
+            // eslint-disable-next-line no-console
+            console.log(`Re-installing ${names[j]} extension...`);
+            config = extConfig[types[i]][names[j]];
+            currentInstall = currentInstall
+                .then(() => extender.install(config.project.arg, true));
+        }
+    }
+}
+
+if (installCount) {
+    // eslint-disable-next-line no-console
+    currentInstall.then(() => console.log('Extensions reinstalled successfully'));
+}

webgme-setup.json

@@ -29,9 +29,9 @@
         "src": "src/plugins/CreateExecution",
         "test": "test/plugins/CreateExecution"
       },
-      "GenerateExecFile": {
-        "src": "src/plugins/GenerateExecFile",
-        "test": "test/plugins/GenerateExecFile"
+      "Export": {
+        "src": "src/plugins/Export",
+        "test": "test/plugins/Export"
       },
       "ImportArtifact": {
         "src": "src/plugins/ImportArtifact",