WIP #1024 fixed the code update on new data added

and support for return values if no current ones

.gitignore

@@ -35,6 +35,3 @@ test-tmp/
 blob-local-storage/
 src/seeds/nn/hash.txt
 src/seeds/pipeline/hash.txt
-
-notes/
-src/worker

.npmignore

@@ -38,6 +38,3 @@ src/seeds/pipeline/hash.txt
 
 # docs
 images/
-
-notes/
-src/worker

bin/deepforge

@@ -185,6 +185,7 @@ var startMongo = function(args, port, silent) {
 };
 
 var hasTorch = function() {
+    return true;
     var result = childProcess.spawnSync('th', ['--help']);
     return !result.error;
 };

bin/start-worker.js

@@ -39,13 +39,13 @@ try {
 }
 
 // Check torch support
-var result = childProcess.spawnSync('th', ['--help']);
-if (result.error) {
-    console.error('Checking Torch7 dependency failed. Do you have Torch7 installed ' + 
-        'and in your PATH?\n\nFor Torch7 installation instructions, check out ' +
-        'http://torch.ch/docs/getting-started.html');
-    process.exit(1);
-}
+//var result = childProcess.spawnSync('th', ['--help']);
+//if (result.error) {
+    //console.error('Checking Torch7 dependency failed. Do you have Torch7 installed ' + 
+        //'and in your PATH?\n\nFor Torch7 installation instructions, check out ' +
+        //'http://torch.ch/docs/getting-started.html');
+    //process.exit(1);
+//}
 
 var cleanUp = function() {
     console.log('removing worker directory ', workerPath);

config/config.test.js

@@ -9,7 +9,8 @@ var config = require('./config.default'),
     path = require('path');
 
 config.server.port = 9001;
-config.mongo.uri = 'mongodb://127.0.0.1:27017/webgme_tests';
+config.mongo.uri = process.env.MONGO_URI || 'mongodb://127.0.0.1:27017';
+config.mongo.uri = config.mongo.uri.replace(/\/[a-zA-Z_\-]*$/,  '') + '/deepforge_tests';
 config.blob.fsDir = path.join(__dirname, '..', 'test-tmp', 'blob');
 
 module.exports = config;

config/config.webgme.js

@@ -22,6 +22,7 @@ config.seedProjects.basePaths.push(__dirname + '/../src/seeds/project');
 config.seedProjects.basePaths.push(__dirname + '/../src/seeds/cifar10');
 config.seedProjects.basePaths.push(__dirname + '/../src/seeds/xor');
 config.seedProjects.basePaths.push(__dirname + '/../src/seeds/devProject');
+config.seedProjects.basePaths.push(__dirname + '/../src/seeds/minimal');
 
 
 

package.json

@@ -17,7 +17,7 @@
     "watch-test": "nodemon --exec 'mocha --recursive test'",
     "build-nn": "node ./utils/nn-parser.js"
   },
-  "version": "1.4.0",
+  "version": "1.3.0",
   "dependencies": {
     "commander": "^2.9.0",
     "dotenv": "^2.0.0",
@@ -43,6 +43,7 @@
     "webgme-simple-nodes": "^2.1.0"
   },
   "devDependencies": {
+    "brython": "^3.2.7",
     "chai": "^3.0.0",
     "jszip": "^2.5.0",
     "mocha": "^2.2.5",

src/common/LayerParser.js

@@ -1,378 +1,127 @@
 /* globals define*/
 (function(root, factory){
     if(typeof define === 'function' && define.amd) {
-        define(['./lua'], function(luajs){
-            return (root.LayerParser = factory(luajs));
+        // TODO: Load the brython script
+        define(['./lua'], function(brython){
+            return (root.LayerParser = factory(brython, console.assert));
         });
     } else if(typeof module === 'object' && module.exports) {
-        var luajs = require('./lua');
-        module.exports = (root.LayerParser = factory(luajs));
+        var brython = require('./node-brython'),
+            assert = require('assert');
+
+        module.exports = (root.LayerParser = factory(brython, assert));
     }
-}(this, function(luajs) {
+}(this, function(brython, assert) {
     var LayerParser = {};
 
-    //////////////////////// Setters //////////////////////// 
-    var returnsSelf = function(fnNode){
-        var stats = fnNode.block.stats,
-            last = stats[stats.length-1];
+    function build_ast(src) {
+        brython.$py_module_path['__main__']='./'
+        return brython.py2js(src,'__main__', '__main__', '__builtins__')
+    }
 
-        if (last.type === 'stat.return') {
-            return last.nret[0].type === 'variable' && last.nret[0].val === 'self';
-        }
-        return false;
-    };
-
-    var isAttrSetter = function(node){
-        if (node.type === 'stat.assignment' && node.lefts.length === 1) {
-            var left = node.lefts[0];
-            return left.type === 'expr.index' && left.self.val === 'self';
-        }
-        return false;
-    };
-
-    var getSettingAttrName = function(node){
-        if (isAttrSetter(node)) {
-            var left = node.lefts[0];
-            return left.key.val;
-        }
-        return null;
-    };
-
-    var getSettingAttrValue = function(node){
-        if (isAttrSetter(node)) {
-            return node.right;
-        }
-        return null;
-    };
-
-    var isSetterMethod = function(curr, parent, className){
-        if (parent && parent.type === 'stat.method') {
-            // is it a fn w/ two statements (stats)
-            if (parent.self.val === className && curr.type === 'function' &&
-                curr.block.stats.length === 2) {
-                // Is the first statement setting a value?
-                return returnsSelf(curr) && getSettingAttrName(curr.block.stats[0]);  // does it return itself?
+    // The provided tree gives us contexts which can have associated 'C'
+    function traverse(node, fn) {
+        var i;
+        if (node.children) {
+            for (i = node.children.length; i--;) {
+                traverse(node.children[i], fn);
+                fn(node.children[i]);
             }
         }
-        return false;
-    };
-
-    var isFnArg = function(method, name) {
-        return method.args.indexOf(name) !== -1;
-    };
-
-    var getSetterSchema = function(node, method) {
-        var setterType,
-            setterFn,
-            value = getSettingAttrValue(node);
-
-        if (value[0].type === 'variable' && isFnArg(method.func, value[0].val)) {
-            setterType = 'arg';
-            setterFn = method.key.val;
-        } else {
-            setterType = 'const';
-            setterFn = {};
-            setterFn[value[0].val] = method.key.val;
-        }
-
-        return {
-            setterType,
-            setterFn
-        };
-    };
-
-    //////////////////////// Setters END //////////////////////// 
-
-    var isInitFn = function(node, className) {
-        if (node.type === 'stat.method' && node.self.val === className) {
-            return node.key.val === '__init';
-        }
-        return false;
-    };
-
-    var getClassAttrDefs = function(method) {
-        var fn = method.func,
-            dict = {},
-            attr,
-            right,
-            value;
-
-        luajs.codegen.traverse(curr => {
-            if (isAttrSetter(curr)) {
-                // Store the value if it is set to a constant
-                attr = curr.lefts[0].key.val;
-                right = curr.right[0];
-                if (right.type.indexOf('const.') !== -1) {
-                    value = right.val;
-
-                    if (right.type === 'const.nil') {
-                        value = null;
-                    }
-
-                    dict[attr] = value;
-                }
-            }
-        })(fn);
-
-        return dict;
-    };
-
-    var getAttrsAndVals = function(method) {
-        // Given a method, get the 'self' attributes and the default values
-        var fn = method.func,
-            dict = {},
-            varName,
-            value,
-            varUsageCnt = {};
-
-        // Get the variables that are used only once (or updating themselves)
-        luajs.codegen.traverse(curr => {
-            if (curr.type === 'variable') {
-                varUsageCnt[curr.val] = varUsageCnt[curr.val] ?
-                    varUsageCnt[curr.val] + 1 : 1;
-            }
-        })(method);
-
-        luajs.codegen.traverse(curr => {
-            // If the variable is only used once and is 'or'-ed w/ a constant
-            // during this use, we can infer that this is the default value
-            if (curr.type === 'expr.op' && curr.op === 'op.or' &&
-                curr.left.type === 'variable' && curr.right.type.indexOf('const') !== -1) {
-                varName = curr.left.val;
-                if (varUsageCnt[varName] === 1) {
-                    value = curr.right.type === 'const.nil' ? null : curr.right;
-                    dict[varName] = value;
-                }
-            }
-        })(fn);
-
-        return dict;
-    };
-
-    var copyNodeValues = function(attrs, from, to) {
-        var value;
-        for (var i = attrs.length; i--;) {
-            value = from[attrs[i]] || null;
-            if (value) {
-                value = (value && value.hasOwnProperty('val')) ? value.val : value;
-                to[attrs[i]] = value;
+        if (node.C && node.C.tree) {
+            for (i = node.C.tree.length; i--;) {
+                traverse(node.C.tree[i], fn);
+                fn(node.C.tree[i]);
             }
         }
-        return to;
-    };
+    }
 
-    var getTypeCheckInfo = function(cond) {
-        var caller,
-            method,
-            target,
-            expType;
+    var types = {},
+        layers = [],
+        pCtx,
+        classNode,
+        params;
 
-        // Check for torch.isTypeOf:
-        if (cond.type === 'expr.call' && cond.func.type === 'expr.index') {
-            caller = cond.func.self.val;
-            method = cond.func.key.val;
+    function isClass(node) {
+        return node.type === 'class';
+    }
 
-            if (cond.type === 'expr.call' && caller === 'torch') {
-                target = cond.args[0].val;
-                if (method === 'isTypeOf' && target) {
-                    expType = cond.args[1].val;
-                    return {
-                        target,
-                        type: expType
-                    };
-                }
-            }
-        } else if (cond.type === 'expr.op') {  // torch.type() === ''
-            // Check right side, too!
-            var sides = [cond.left, cond.right],
-                side,
-                otherSide;
+    function isInitFn(node) {
+        return node.type === 'def' && node.name === '__init__';
+    }
 
-            for (var i = sides.length; i--;) {
-                side = sides[i];
-                otherSide = sides[(i+1)%2];
-                if (side.type === 'expr.call' && side.func.type === 'expr.index') {
-                    // Is it torch?
-                    caller = side.func.self.val;
-                    method = side.func.key.val;
-                    if (caller === 'torch' && method === 'type') {
-                        if (side.args[0].type === 'variable') {
-                            target = side.args[0].val;
-                            if (otherSide.type === 'const.string') {
-                                expType = otherSide.val;
+    function getBaseClass(node) {
+        assert(node.type === 'class');
+        return node.args.tree[0].tree[0].tree[0].value;
+    }
 
-                                return {
-                                    target: target,
-                                    type: expType
-                                };
+    function findTorchLayers(root) {
+        var defaults = {},
+            layers = [],
+            defTypes,
+            args,
+            def;
+
+        traverse(root, node => {
+            // Get the class for the given function
+            if (isInitFn(node)) {
+            // TODO: What if there is no constructor? Is this a potential problem?
+                pCtx = node.parent.node.parent;
+                classNode = pCtx.C.tree[0];
+
+                if (isClass(classNode)) {
+                    // remove the 'self' variable
+                    // TODO: May need to update this for kwargs
+                    // (use positional_list)
+                    args = node.tree[1].tree;
+                    defaults = {};
+                    params = node.args.slice(1);
+                    defTypes = {};
+                    for (var i = args.length; i--;) {
+                        if (args[i].tree[0]) {
+                            def = args[i].tree[0].tree[0];
+                            if (def.type === 'int') {
+                                defaults[params[i-1]] = parseInt.apply(null, def.value.reverse());
+                            } else {
+                                defaults[params[i-1]] = def.value;
+                            }
+                            if (/^(True|False)$/.test(defaults[params[i-1]])) {
+                                defTypes[params[i-1]] = 'boolean';
+                            } else {
+                                defTypes[params[i-1]] = def.type;
                             }
                         }
-                        
                     }
+                    layers.push({
+                        name: classNode.name,
+                        baseType: getBaseClass(classNode),
+                        //doc: classNode.doc_string || '',
+                        defaults: defaults,
+                        types: defTypes,
+                        setters: {},
+                        params: params
+                    });
                 }
             }
-            return null;
-        }
-    };
-
-    var isError = function(stat) {
-        var fn;
-        if (stat.type === 'stat.expr' && stat.expr.type === 'expr.call') {
-            fn = stat.expr.func.val;
-            return fn === 'error';
-        }
-        return false;
-    };
-
-    var inferParamTypes = function(node, paramDefs) {
-        var types = {},
-            check,
-            cond;
-
-        // Infer from assertions
-        luajs.codegen.traverse(curr => {
-            // check for 'assert's that check type
-            if (curr.type === 'expr.call' && curr.func.val === 'assert') {
-                cond = curr.args[0];
-                check = getTypeCheckInfo(cond);
-                if (check) {
-                    types[check.target] = check.type;
-                }
-            } else if (curr.type === 'stat.if' && curr.cond.op === 'uop.not') {
-                // if statements throwing errors on type mismatch
-                cond = curr.cond.operand;  // non-negated version
-                // Check that it throws an error on true
-                if (curr.tblock.stats.some(isError)) {
-                    check = getTypeCheckInfo(cond);
-                    if (check) {
-                        types[check.target] = check.type;
-                    }
-                }
-            }
-        })(node);
-
-        // Infer from defaults
-        Object.keys(paramDefs).forEach(param => {
-            var val = paramDefs[param];
-            if (val) {  // initialized to 'null' doesn't help us...
-                types[param] = val.type.replace('const.', '');
-            }
         });
 
-        return types;
-    };
+        return layers;
+    }
 
-    var findTorchClass = function(ast){ 
-        var torchClassArgs,  // args for `torch.class(...)`
-            name = '',
-            alias,
-            baseType,
-            params,
-            setters = {},
-            defaults = {},
-            paramDefs,
-            attrDefs;
-
-        if(ast.type == 'function'){
-            ast.block.stats.forEach(function(func){
-                if(func.type == 'stat.local' && func.right && func.right[0] &&
-                  func.right[0].func && func.right[0].func.self &&
-                  func.right[0].func.self.val == 'torch' &&
-                  func.right[0].func.key.val == 'class'){
-
-                    torchClassArgs = func.right[0].args.map(arg => arg.val);
-                    name = torchClassArgs[0];
-                    if(name !== ''){
-                        name = name.replace('nn.', '');
-                        alias = func.names[0] || name;
-                        if (torchClassArgs.length > 1) {
-                            baseType = torchClassArgs[1].replace('nn.', '');
-                        }
-                    }
-                }
-            });
-        }
-
-        // Get the setters, defaults and type info (inferred)
-        var setterNames,
-            schema,
-            types,
-            values;
-
-        luajs.codegen.traverse((curr, parent) => {
-            var firstLine,
-                attrName;
-
-            // Record the setter functions
-            if (isSetterMethod(curr, parent, alias)) {
-                firstLine = curr.block.stats[0];
-                // just use the attribute attrName for now...
-                attrName = getSettingAttrName(firstLine);
-
-                // merge schemas
-                schema = getSetterSchema(firstLine, parent);
-                if (setters[attrName] && setters[attrName].setterType === 'const') {  // merge
-                    for (var val in schema.setterFn) {
-                        setters[attrName].setterFn[val] = schema.setterFn[val];
-                    }
-                } else {
-                    setters[attrName] = schema;
-                }
-            } else if (isInitFn(curr, alias)) {  // Record the defaults
-                paramDefs = getAttrsAndVals(curr);
-                attrDefs = getClassAttrDefs(curr);
-                types = inferParamTypes(curr, paramDefs);
-
-                // get ctor args
-                params = curr.func.args;
-                if(params.length === 0 && curr.func.varargs){
-                    params.push('params');
-                }
-            }
-
-        })(ast);
-
-        // Get the defaults for the params from defs
-        if (paramDefs && params) {
-            copyNodeValues(params, paramDefs, defaults);
-        }
-
-        // Get the defaults for the setters from attrDefs
-        if (attrDefs) {
-            setterNames = Object.keys(setters);
-            copyNodeValues(setterNames, attrDefs, defaults);
-        }
-
-        // Remove any const setters w/ only one value and no default
-        setterNames = Object.keys(setters);
-        for (var i = setterNames.length; i--;) {
-            schema = setters[setterNames[i]];
-            if (schema.setterType === 'const') {
-                values = Object.keys(schema.setterFn);
-                if (values.length === 1 &&
-                    // boolean setters can have the default value inferred
-                    values[0] !== 'true' && values[0] !== 'false' &&
-                    !defaults[setterNames[i]]) {
-
-                    delete setters[setterNames[i]];
-                }
-            }
-        }
-
-        return {
-            name,
-            baseType,
-            params,
-            setters,
-            types,
-            defaults
-        };
-    };
-
-    LayerParser.parse = function(text) {
+    // Try to find the class definitions...
+    //
+    //  Need to create:
+    //
+    //    setters: (I don't think these are used in pytorch!
+    //    types: 
+    //    type: 
+    //////////////////////// Setters //////////////////////// 
+    LayerParser.parse = function(src) {
         try {
-            var ast = luajs.parser.parse(text);
-            return findTorchClass(ast);
+            brython.$py_module_path['__main__']='./';
+            var ast = brython.py2js(src,'__main__', '__main__', '__builtins__');
+            var layers = findTorchLayers(ast);
+            return layers;
         } catch (e) {
             return null;
         }

src/common/OperationCode.js

@@ -0,0 +1,331 @@
+/*globals Sk, define*/
+var isNodeJs = typeof module === 'object' && module.exports;
+(function(root, factory){
+    if(typeof define === 'function' && define.amd) {
+        define(['./skulpt.min'], function(){
+            return (root.OperationParser = factory(Sk));
+        });
+    } else if(isNodeJs) {
+        require('./skulpt.min');
+
+        module.exports = (root.OperationParser = factory(Sk));
+    }
+}(this, function(Sk) {
+    var MAIN_FN = 'execute';
+    var CTOR_FN = '__init__';
+    var OperationCode = function(code, filename) {
+        this._lines = code.split('\n');
+        this.filename = filename;
+    };
+
+    OperationCode.prototype.getName = function() {
+        if (!this._schema) this.updateSchema();
+
+        return this._schema.name;
+    };
+
+    OperationCode.prototype.getBase = function() {
+        if (!this._schema) this.updateSchema();
+
+        return this._schema.base;
+    };
+
+    OperationCode.prototype.getArguments = function(method) {
+        if (!this._schema) this.updateSchema();
+        if (!this._schema.methods[method]) return null;
+
+        return this._schema.methods[method].inputs.slice();
+    };
+
+    OperationCode.prototype.getReturnValues = function(method) {
+        if (!this._schema) this.updateSchema();
+        if (!this._schema.methods[method]) return null;
+
+        return this._schema.methods[method].outputs.slice();
+    };
+
+    OperationCode.prototype.getOutputs = function() {
+        return this.getReturnValues(MAIN_FN);
+    };
+
+    OperationCode.prototype.getInputs = function() {
+        return this.getArguments(MAIN_FN);
+    };
+
+    OperationCode.prototype.removeInput = function(name) {
+        return this._removeIOCode(this.getInputs(), name);
+    };
+
+    OperationCode.prototype.removeOutput = function(name) {
+        return this._removeIOCode(this.getOutputs(), name);
+    };
+
+    OperationCode.prototype._removeIOCode = function(ios, name) {
+        var match,
+            prev,
+            line,
+            startIndex,
+            endIndex;
+
+        for (var i = 0; i < ios.length; i++) {
+            match = ios[i];
+            prev = ios[i-1];
+
+            if (match.name === name) {
+                line = this._lines[match.pos.line-1];
+
+                startIndex = prev ? prev.pos.col + prev.value.toString().length : match.pos.col;
+                // only remove the following ',' if first input/output
+                endIndex = i === 0 && i < ios.length-1 ? ios[i+1].pos.col :
+                    match.pos.col + match.value.toString().length;
+                this._lines[match.pos.line-1] = line.substring(0, startIndex) +
+                    line.substring(endIndex);
+
+                this.clearSchema();
+                return match;
+            }
+        }
+        return null;
+    };
+
+    OperationCode.prototype.addInput = function(name) {
+        return this.addArgument(MAIN_FN, name);
+    };
+
+    OperationCode.prototype.addOutput = function(name) {
+        return this.addReturnValue(MAIN_FN, name);
+    };
+
+    OperationCode.prototype.addArgument = function(method, name) {
+        return this._addIOCode(method, name, true);
+    };
+
+    OperationCode.prototype.addReturnValue = function(method, name) {
+        return this._addIOCode(method, name, false);
+    };
+
+    OperationCode.prototype.addMethod = function(method) {
+        // TODO: get the position at the top of the class def
+        var line = this._schema.body.pos.line - 1,
+            indentSize = this._schema.body.pos.col,
+            indent = new Array(indentSize+1).join(' '),
+            snippet = indent + `def ${method}():`,
+            body = new Array(indentSize+5).join(' ') + 'return';
+
+        this._lines.splice(line-1, 0, '');
+        this._lines.splice(line-1, 0, snippet);
+        this._lines.splice(line, 0, body);
+
+        this.clearSchema();
+    };
+
+    OperationCode.prototype.hasMethod = function(method) {
+        if (!this._schema) this.updateSchema();
+        return this._schema.methods[method];
+    };
+
+    OperationCode.prototype._addIOCode = function(method, name, isInput) {
+        if (!this.hasMethod(method)) this.addMethod(method);
+
+        this.updateSchema();
+
+        var ios = this._schema.methods[method][isInput ? 'inputs' : 'outputs'].slice(),
+            node = this._schema.methods[method].node,
+            body = node.body,
+            content = name,
+            line,
+            startIndex,
+            endIndex,
+            lineIndex;
+
+        if (ios.length) {
+            var pos = ios[ios.length-1].pos;
+            var argLen = ios[ios.length-1].name.length;
+
+            line = this._lines[pos.line-1];
+            startIndex = pos.col + argLen;
+            endIndex = pos.col + argLen;
+            content = ', ' + name;
+            lineIndex = pos.line - 1;
+        } else if (isInput) {
+            var first = body[0];
+            lineIndex = first.lineno - 2;
+            line = this._lines[lineIndex];
+            this._lines[lineIndex] = line.replace(/\).*?:/, name + '):');
+
+            return this.clearSchema();
+        } else {
+            var ret = body.find(node => this._isNodeType(node, 'Return_'));
+            if (ret) {
+                lineIndex = ret.lineno-1;
+                startIndex = endIndex = ret.col_offset + 6;
+                content = ' ' + content;
+            } else {  // add to the end of the body (no return statement)
+                var lastNode = body[body.length-1];
+                var indent = new Array(lastNode.col_offset+1).join(' ');
+
+                lineIndex = lastNode.lineno;
+                this._lines.splice(lineIndex, 0, '');
+                startIndex = endIndex = 0;
+                content = indent + 'return ' + content;
+            }
+        }
+
+        line = this._lines[lineIndex];
+        this._lines[lineIndex] = line.substring(0, startIndex) + content +
+            line.substring(endIndex);
+
+        this.clearSchema();
+    };
+
+    OperationCode.prototype.rename = function(oldName, name) {
+        if (!this.hasMethod(MAIN_FN)) return;
+
+        var fnSchema = this._schema.methods[MAIN_FN];
+        var startLine = fnSchema.bounds.start.line - 1;
+        var endLine = fnSchema.bounds.end ? fnSchema.bounds.end.line - 1 : this._lines.length;
+        var pattern = new RegExp('\\b' + oldName + '\\b');
+
+        for (var i = startLine; i < endLine; i++) {
+            this._lines[i] = this._lines[i].replace(pattern, name);
+        }
+        this.clearSchema();
+    };
+
+    OperationCode.prototype.getCode = function() {
+        return this._lines.join('\n');
+    };
+
+    OperationCode.prototype.getAst = function () {
+        if (this._ast) return this._ast;
+
+        var filename = this.filename || 'operation.py';
+        var cst = Sk.parse(filename, this.getCode()).cst;
+        var ast = Sk.astFromParse(cst, filename);
+        return this._ast = ast;
+    };
+
+    OperationCode.prototype._isNodeType = function (node, name) {
+        return node.constructor.name === name;
+    };
+
+    OperationCode.prototype._parseFn = function (node, schema, next) {
+        var name = node.name.v;
+
+        schema.methods[name] = {};
+        // add inputs
+        schema.methods[name].inputs = node.args.args.map(arg => {
+            return {
+                name: arg.id.v,
+                value: arg.id.v,
+                pos: {
+                    line: arg.lineno,
+                    col: arg.col_offset
+                }
+            };
+        });
+
+        // add outputs
+        var ret = node.body.find(node => this._isNodeType(node, 'Return_'));
+        var retVals = [];
+        if (ret) {
+            retVals = ret.value && this._isNodeType(ret.value, 'Tuple') ?
+                ret.value.elts : [ret.value];
+        }
+
+        schema.methods[name].outputs = retVals
+            .filter(node => !!node)
+            .map((arg, index) => {
+                var isNameNode = this._isNodeType(arg, 'Name');
+                var name = isNameNode ? arg.id.v : 'result';
+                if (!isNameNode && index > 0) {
+                    name += '_' + index;
+                }
+
+                var value = this._isNodeType(arg, 'Num') ? arg.n.v : name;
+
+                return {
+                    name: name,
+                    value: value,
+                    pos: {
+                        line: arg.lineno,
+                        col: arg.col_offset
+                    }
+                };
+            });
+
+        // Get the function location
+        schema.methods[name].bounds = {};
+        schema.methods[name].bounds.start = {
+            line: node.lineno,
+            col: node.col_offset
+        };
+
+        if (next) {
+            schema.methods[name].bounds.end = {
+                line: next.lineno,
+                col: next.col_offset
+            };
+        }
+
+        schema.methods[name].node = node;
+    };
+
+    OperationCode.prototype.updateSchema = function () {
+        if (!this._schema) this._schema = this.getSchema();
+    };
+
+    OperationCode.prototype.clearSchema = function () {
+        this._ast = null;
+        this._schema = null;
+    };
+
+    OperationCode.prototype.getSchema = function () {
+        var schema = {
+            name: null,
+            base: null,
+            methods: {}
+        };
+        var ast = this.getAst();
+
+        // Find the class definition
+        var classDef = ast.body.find(node => this._isNodeType(node, 'ClassDef'));
+        if (classDef) {
+            schema.name = classDef.name.v;
+
+            // TODO: what if fn is inherited?
+            var nodes = classDef.body;
+            for (var i = 0; i < nodes.length; i++) {
+                if (this._isNodeType(nodes[i], 'FunctionDef')) {
+                    this._parseFn(nodes[i], schema, nodes[i+1]);
+                }
+            }
+            schema.body = {
+                pos: {
+                    line: nodes[0].lineno,
+                    col: nodes[0].col_offset,
+                }
+            };
+
+        }
+
+        schema.ast = ast;
+
+        return schema;
+    };
+
+    /////////////////////// Attributes /////////////////////// 
+    OperationCode.prototype.addAttribute = function(name, value) {
+        return this._addIOCode(CTOR_FN, name, true);
+    };
+
+    OperationCode.prototype.removeAttribute = function(name) {
+        // TODO
+    };
+
+    OperationCode.prototype.getAttributes = function() {
+        return this.getArguments(CTOR_FN);
+    };
+
+    return OperationCode;
+}));

src/common/OperationParser.js

@@ -0,0 +1,126 @@
+/* globals define, Sk*/
+var isNodeJs = typeof module === 'object' && module.exports;
+(function(root, factory){
+    if(typeof define === 'function' && define.amd) {
+        define(['./skulpt.min'], function(){
+            return (root.OperationParser = factory(Sk));
+        });
+    } else if(isNodeJs) {
+        require('./skulpt.min');
+
+        module.exports = (root.OperationParser = factory(Sk));
+    }
+}(this, function(Sk) {
+    Sk.python3 = true;
+    var OperationParser = {};
+
+    // The provided tree gives us contexts which can have associated 'C'
+    function traverse(node, fn) {
+        var i;
+        if (node.children) {
+            for (i = node.children.length; i--;) {
+                traverse(node.children[i], fn);
+                fn(node.children[i]);
+            }
+        }
+        if (node.C && node.C.tree) {
+            for (i = node.C.tree.length; i--;) {
+                traverse(node.C.tree[i], fn);
+                fn(node.C.tree[i]);
+            }
+        }
+    }
+
+    function isNodeType(node, name) {
+        return node.constructor.name === name;
+    }
+
+    function parseFn(node, schema) {
+        var name = node.name.v;
+
+        schema.methods[name] = {};
+        // add inputs
+        schema.methods[name].inputs = node.args.args.map(arg => {
+            return {
+                name: arg.id.v,
+                value: arg.id.v,
+                pos: {
+                    line: arg.lineno,
+                    col: arg.col_offset
+                }
+            };
+        });
+
+        // add outputs
+        var ret = node.body.find(node => isNodeType(node, 'Return_'));
+        var retVals = [];
+        if (ret) {
+            retVals = ret.value && isNodeType(ret.value, 'Tuple') ?
+                ret.value.elts : [ret.value];
+        }
+
+        schema.methods[name].outputs = retVals.map((arg, index) => {
+            var isNameNode = isNodeType(arg, 'Name');
+            var name = isNameNode ? arg.id.v : 'result';
+            if (!isNameNode && index > 0) {
+                name + '_' + index;
+            }
+
+            var value = isNodeType(arg, 'Num') ? arg.n.v : name;
+
+            return {
+                name: name,
+                value: value,
+                pos: {
+                    line: arg.lineno,
+                    col: arg.col_offset
+                }
+            };
+        });
+    }
+
+    function parseOperationAst(ast) {
+        var schema = {
+            name: null,
+            base: null,
+            methods: {}
+        };
+
+        // Find the class definition
+        var classDef = ast.body.find(node => isNodeType(node, 'ClassDef'));
+        if (classDef) {
+            schema.name = classDef.name.v;
+
+            // TODO: what if fn is inherited?
+            classDef.body
+                .filter(node => isNodeType(node, 'FunctionDef'))
+                .forEach(node => parseFn(node, schema));
+
+        }
+
+        schema.inputs = schema.methods.execute.inputs;
+        schema.outputs = schema.methods.execute.outputs;
+        schema.ast = ast;
+        return schema;
+    }
+
+    OperationParser._traverse = traverse;
+    OperationParser._getAst = function(src, filename) {
+        filename = filename || 'operation.py';
+        var cst = Sk.parse(filename, src).cst;
+        var ast = Sk.astFromParse(cst, filename);
+        return ast;
+    };
+
+    OperationParser.parse = function(src, filename) {
+        //try {
+            var ast = this._getAst(src, filename);
+            return  parseOperationAst(ast);
+        //} catch (e) {
+            //console.error('operation parsing failed:', e);
+            //return null;
+        //}
+    };
+
+    return OperationParser;
+}));

src/common/globals.js

@@ -59,8 +59,14 @@ define([
     var createNamedNode = function(baseId, parentId, isMeta) {
         var newId = client.createNode({parentId, baseId}),
             baseNode = client.getNode(baseId),
-            basename = 'New' + baseNode.getAttribute('name'),
-            newName = getUniqueName(parentId, basename);
+            basename,
+            newName;
+
+        basename = 'New';
+        if (baseNode.getAttribute('name') !== 'Operation') {
+            basename += baseNode.getAttribute('name');
+        }
+        newName = getUniqueName(parentId, basename);
 
         // If instance, make the first char lowercase
         if (!isMeta) {

src/common/lua.js

@@ -3234,7 +3234,7 @@ function runScripts(){
 
 if (window.addEventListener){
     window.addEventListener("DOMContentLoaded", runScripts, false);
-} else {
+} else if (window.attachEvent) {
     window.attachEvent("onload", runScripts);
 }
 

src/common/node-brython.js

@@ -0,0 +1,177 @@
+/*
+Author: Billy Earney
+Date: 04/19/2013
+License: MIT
+
+Description: This file can work as a "bridge" between nodejs and brython
+ so that client side brython code can be executed on the server side.
+Will brython replace Cython one day?  Only time will tell.
+:)
+
+*/
+
+var fs = require('fs'),
+    path = require('path'),
+    //brythonSrcPath = path.join(__dirname, '..', '..', 'node_modules', 'brython', 'www', 'src', 'brython.js');
+    brythonSrcPath = path.join(__dirname, 'brython.js');
+
+document={};
+document.getElementsByTagName = () => [{src: ''}];
+window={};
+window.location = {href: ''};
+window.navigator={}
+window.confirm = () => true;
+window.console = console;
+document.$py_src = {}
+document.$debug = 0
+
+self={};
+__BRYTHON__={}
+__BRYTHON__.$py_module_path = {}
+__BRYTHON__.$py_module_alias = {}
+__BRYTHON__.$py_next_hash = -Math.pow(2,53)
+__BRYTHON__.exception_stack = []
+__BRYTHON__.scope = {}
+__BRYTHON__.modules = {}
+
+// Read and eval library
+jscode = fs.readFileSync(brythonSrcPath, 'utf8');
+eval(jscode);
+
+//function node_import(module,alias,names) {
+function $import_single(module) {
+  var search_path=['../src/libs', '../src/Lib'];
+  var ext=['.js', '.py'];
+  var mods=[module, module+'/__init__'];
+
+  for(var i=0, _len_i = search_path.length; i < _len_i; i++) {
+     for (var j=0, _len_j = ext.length; j < _len_j; j++) {
+         for (var k=0, _len_k = mods.length; k < _len_k; k++) {
+             var path=search_path[i]+'/'+mods[k]+ext[j]
+
+             //console.log("searching for " + path);
+             var module_contents;
+             try {
+               module_contents=fs.readFileSync(path, 'utf8')
+             } catch(err) {}
+             if (module_contents !== undefined) {
+                console.log("imported " + module)
+                //console.log(module_contents);
+                if (ext[j] == '.js') {
+                   return $import_js_module(module,alias,names,path,module_contents)
+                }
+                return $import_py_module(module,alias,names,path,module_contents)
+             }
+         }
+     }
+  }
+  console.log("error time!");
+  res = Error()
+  res.name = 'NotFoundError'
+  res.message = "No module named '"+module+"'"
+  throw res
+}
+
+$compile_python=function(module_contents,module) {
+    var root = __BRYTHON__.py2js(module_contents,module)
+    var body = root.children
+    root.children = []
+    // use the module pattern : module name returns the results of an anonymous function
+    var mod_node = new $Node('expression')
+    //if(names!==undefined){alias='$module'}
+    new $NodeJSCtx(mod_node,'$module=(function()')
+    root.insert(0,mod_node)
+    mod_node.children = body
+    // search for module-level names : functions, classes and variables
+    var mod_names = []
+    for(var i=0, _len_i = mod_node.children.length; i < _len_i;i++){
+        var node = mod_node.children[i]
+        // use function get_ctx() 
+        // because attribute 'context' is renamed by make_dist...
+        var ctx = node.get_ctx().tree[0]
+        if(ctx.type==='def'||ctx.type==='class'){
+            if(mod_names.indexOf(ctx.name)===-1){mod_names.push(ctx.name)}
+        } else if(ctx.type==='from') {
+            for (var j=0, _len_j = ctx.names.length; j < _len_j; j++) {
+                var name=ctx.names[j];
+                if (name === '*') {
+                   // just pass, we don't want to include '*'
+                } else if (ctx.aliases[name] !== undefined) {
+                   if (mod_names.indexOf(ctx.aliases[name])===-1){
+                      mod_names.push(ctx.aliases[name])
+                   }
+                } else {
+                   if (mod_names.indexOf(ctx.names[j])===-1){
+                     mod_names.push(ctx.names[j])
+                   }
+                }
+            }
+        }else if(ctx.type==='assign'){
+            var left = ctx.tree[0]
+            if(left.type==='expr'&&left.tree[0].type==='id'&&left.tree[0].tree.length===0){
+                var id_name = left.tree[0].value
+                if(mod_names.indexOf(id_name)===-1){mod_names.push(id_name)}
+            }
+        }
+    }
+    // create the object that will be returned when the anonymous function is run
+    var ret_code = 'return {'
+    for(var i=0, _len_i = mod_names.length; i < _len_i;i++){
+        ret_code += mod_names[i]+':'+mod_names[i]+','
+    }
+    ret_code += '__getattr__:function(attr){return this[attr]},'
+    ret_code += '__setattr__:function(attr,value){this[attr]=value}'
+    ret_code += '}'
+    var ret_node = new $Node('expression')
+    new $NodeJSCtx(ret_node,ret_code)
+    mod_node.add(ret_node)
+    // add parenthesis for anonymous function execution
+    
+    var ex_node = new $Node('expression')
+    new $NodeJSCtx(ex_node,')()')
+    root.add(ex_node)
+    
+    try{
+        var js = root.to_js()
+        return js;
+    }catch(err){
+        eval('throw '+err.name+'(err.message)')
+    }
+    return undefined;
+}
+
+function build_ast(src) {
+  __BRYTHON__.$py_module_path['__main__']='./'
+  return __BRYTHON__.py2js(src,'__main__', '__main__', '__builtins__')
+}
+
+function execute_python_script(filename) {
+  _py_src=fs.readFileSync(filename, 'utf8')
+  var root = build_ast(_py_src)
+  var js = root.to_js()
+  //eval(js);
+}
+
+//console.log("try to execute compile script");
+
+__BRYTHON__.$py_module_path = __BRYTHON__.$py_module_path || {}
+__BRYTHON__.$py_module_alias = __BRYTHON__.$py_module_alias || {}
+__BRYTHON__.exception_stack = __BRYTHON__.exception_stack || []
+__BRYTHON__.scope = __BRYTHON__.scope || {}
+__BRYTHON__.imported = __BRYTHON__.imported || {}
+__BRYTHON__.modules = __BRYTHON__.modules || {}
+__BRYTHON__.compile_python=$compile_python
+
+__BRYTHON__.debug = 0
+__BRYTHON__.$options = {}
+__BRYTHON__.$options.debug = 0
+
+// other import algs don't work in node
+//import_funcs=[node_import]
+
+if (!module.parent) {
+    var filename=process.argv[2];
+    execute_python_script(filename)
+}
+
+module.exports = __BRYTHON__;

src/common/viz/OperationControl.js

@@ -38,5 +38,103 @@ define([
         return typeId ? this._client.getNode(typeId).getChildrenIds() : [];
     };
 
+    OperationControl.prototype.createIONode = function(opId, typeId, isInput, baseName, silent) {
+        var cntrId = this.getDataContainerId(opId, isInput),
+            name = this._client.getNode(opId).getAttribute('name'),
+            dataName,
+            msg;
+
+        baseName = baseName || this._client.getNode(typeId).getAttribute('name').toLowerCase();
+        dataName = this._getDataName(cntrId, baseName);
+
+        msg = `Adding ${isInput ? 'input' : 'output'} "${dataName}" to ${name} interface`;
+        if (!silent) {
+            this._client.startTransaction(msg);
+        }
+
+        var id = this._client.createNode({
+            parentId: cntrId,
+            baseId: typeId
+        });
+
+        // Set the name of the new input
+        this._client.setAttribute(id, 'name', dataName);
+
+        if (!silent) {
+            this._client.completeTransaction();
+        }
+        return id;
+    };
+
+    OperationControl.prototype._getDataName = function(cntrId, baseName) {
+        var otherNames = this._getDataNames(cntrId),
+            name = baseName,
+            i = 1;
+
+        while (otherNames.indexOf(name) !== -1) {
+            i++;
+            name = baseName + '_' + i;
+        }
+        return name;
+    };
+
+    OperationControl.prototype._getDataNames = function(cntrId) {
+        var otherIds = this._client.getNode(cntrId).getChildrenIds();
+
+        return otherIds.map(id => this._client.getNode(id).getAttribute('name'));
+    };
+
+    OperationControl.prototype.getDataNames = function(opId, isInput) {
+        return this._getDataNames(this.getDataContainerId(opId, isInput));
+    };
+
+    OperationControl.prototype.getDataContainerId = function(opId, isInput) {
+        var node = this._client.getNode(opId),
+            cntrs = node.getChildrenIds(),
+            cntrType = isInput ? 'Inputs' : 'Outputs';
+
+        return cntrs.find(id => this.hasMetaName(id, cntrType));
+    };
+
+    OperationControl.prototype.getDataTypeId = function() {
+        var dataNode = this._client.getAllMetaNodes()
+            .find(node => node.getAttribute('name') === 'Data');
+
+        return dataNode.getId();
+    };
+
+    OperationControl.prototype.addInputData = function(opId, name) {
+        return this.createIONode(opId, this.getDataTypeId(), true, name, true);
+    };
+
+    OperationControl.prototype.removeInputData = function(opId, name) {
+        var cntrId = this.getDataContainerId(opId, true),
+            otherIds = this._client.getNode(cntrId).getChildrenIds(),
+            dataId = otherIds.find(id => this._client.getNode(id).getAttribute('name') === name);
+
+        if (dataId) {  // ow, data not found
+            this._client.deleteNode(dataId);
+        }
+    };
+
+    OperationControl.prototype.addOutputData = function(opId, name) {
+        return this.createIONode(opId, this.getDataTypeId(), false, name, true);
+    };
+
+    OperationControl.prototype.removeOutputData = function(opId, name) {
+        var cntrId = this.getDataContainerId(opId),
+            otherIds = this._client.getNode(cntrId).getChildrenIds(),
+            dataId = otherIds.find(id => this._client.getNode(id).getAttribute('name') === name);
+
+        if (dataId) {  // ow, data not found
+            this._client.deleteNode(dataId);
+        }
+    };
+
+    OperationControl.prototype.isInputData = function(nodeId) {
+        var node = this._client.getNode(nodeId);
+        return this.hasMetaName(node.getParentId(), 'Inputs');
+    };
+
     return OperationControl;
 });

src/decorators/OpIntDecorator/EasyDAG/OpIntDecorator.EasyDAGWidget.js

@@ -103,6 +103,7 @@ define([
             () => this.deleteAttribute(name));
     };
 
+    // TODO: implement this in the widget controller (so we can update the op code)
     OpIntDecorator.prototype.deleteAttribute = function(name) {
         var opName = this._node.attributes.name.value,
             msg = `Deleting "${name}" attribute from "${opName}" operation`;
@@ -113,6 +114,7 @@ define([
         this.client.completeTransaction();
     };
 
+    // TODO: implement this in the widget controller (so we can update the op code)
     OpIntDecorator.prototype.setAttributeMeta = function(name, desc) {
         var schema,
             opName = this._node.attributes.name.value,

src/plugins/CreateTorchMeta/CreateTorchMeta.js

@@ -205,9 +205,10 @@ define([
     };
 
     // Some helper methods w/ attribute handling
-    var LUA_TO_GME = {
+    var PYTHON_TO_GME = {
         boolean: 'boolean',
-        number: 'float',
+        float: 'float',
+        int: 'integer',
         string: 'string'
     };
 
@@ -301,7 +302,7 @@ define([
             attrs.forEach(name => {
                 desc = {};
                 defVal = defaults.hasOwnProperty(name) ? defaults[name] : '';
-                type = LUA_TO_GME[types[name]];
+                type = PYTHON_TO_GME[types[name]];
                 if (type) {
                     desc.type = type;
                 }
@@ -376,6 +377,11 @@ define([
             // Set the min, max
             schema.max = +schema.max;
         }
+        // Add the enum for booleans so we use python style True/False
+        if (schema.type === 'boolean') {
+            schema.enum = ['True', 'False'];
+            schema.type = 'string';
+        }
 
         // Create the attribute and set the schema
         this.core.setAttributeMeta(node, name, schema);

src/plugins/CreateTorchMeta/metadata.json

@@ -17,7 +17,6 @@
       "value": "all",
       "valueItems": [
         "nn",
-        "rnn",
         "all"
       ],
       "valueType": "string",

src/plugins/CreateTorchMeta/schemas/index.js

@@ -1,13 +1,10 @@
 /*globals define*/
 define([
-    'text!./nn.json',
-    'text!./rnn.json'
+    'text!./nn.json'
 ], function(
-    nn,
-    rnn
+    nn
 ) {
     return {
-        nn: nn,
-        rnn: rnn
+        nn: nn
     };
 });

src/plugins/CreateTorchMeta/schemas/rnn.json

@@ -1,178 +0,0 @@
-[
-  {
-    "name": "CopyGrad",
-    "baseType": "Identity",
-    "setters": {},
-    "defaults": {},
-    "type": "RNN"
-  },
-  {
-    "name": "FastLSTM",
-    "baseType": "LSTM",
-    "params": [
-      "inputSize",
-      "outputSize",
-      "rho",
-      "eps",
-      "momentum",
-      "affine"
-    ],
-    "setters": {},
-    "types": {
-      "eps": "number",
-      "momentum": "number"
-    },
-    "defaults": {
-      "momentum": 0.1,
-      "eps": 0.1
-    },
-    "type": "RNN"
-  },
-  {
-    "name": "LSTM",
-    "baseType": "AbstractRecurrent",
-    "params": [
-      "inputSize",
-      "outputSize",
-      "rho",
-      "cell2gate"
-    ],
-    "setters": {},
-    "types": {
-      "rho": "number"
-    },
-    "defaults": {
-      "rho": 9999
-    },
-    "type": "RNN"
-  },
-  {
-    "name": "LinearNoBias",
-    "baseType": "Linear",
-    "params": [
-      "inputSize",
-      "outputSize"
-    ],
-    "setters": {},
-    "types": {},
-    "defaults": {},
-    "type": "Simple"
-  },
-  {
-    "name": "LookupTableMaskZero",
-    "baseType": "LookupTable",
-    "params": [
-      "nIndex",
-      "nOutput"
-    ],
-    "setters": {},
-    "types": {},
-    "defaults": {},
-    "type": "RNN"
-  },
-  {
-    "name": "NormStabilizer",
-    "baseType": "AbstractRecurrent",
-    "params": [
-      "beta"
-    ],
-    "setters": {},
-    "defaults": {},
-    "type": "RNN"
-  },
-  {
-    "name": "Recurrent",
-    "baseType": "AbstractRecurrent",
-    "params": [
-      "start",
-      "input",
-      "feedback",
-      "transfer",
-      "rho",
-      "merge"
-    ],
-    "setters": {},
-    "types": {
-      "start": "nn.Module",
-      "transfer": "nn.Module",
-      "feedback": "nn.Module",
-      "input": "nn.Module"
-    },
-    "defaults": {},
-    "type": "RNN"
-  },
-  {
-    "name": "SAdd",
-    "baseType": "Module",
-    "params": [
-      "addend",
-      "negate"
-    ],
-    "setters": {},
-    "types": {},
-    "defaults": {},
-    "type": "RNN"
-  },
-  {
-    "name": "SeqBRNN",
-    "baseType": "Container",
-    "params": [
-      "inputDim",
-      "hiddenDim",
-      "batchFirst"
-    ],
-    "setters": {},
-    "types": {},
-    "defaults": {},
-    "type": "RNN"
-  },
-  {
-    "name": "SeqGRU",
-    "baseType": "Module",
-    "params": [
-      "inputSize",
-      "outputSize"
-    ],
-    "setters": {},
-    "types": {},
-    "defaults": {},
-    "type": "RNN"
-  },
-  {
-    "name": "SeqLSTM",
-    "baseType": "Module",
-    "params": [
-      "inputsize",
-      "hiddensize",
-      "outputsize"
-    ],
-    "setters": {},
-    "types": {},
-    "defaults": {},
-    "type": "RNN"
-  },
-  {
-    "name": "SeqLSTMP",
-    "baseType": "SeqLSTM",
-    "params": [
-      "inputsize",
-      "hiddensize",
-      "outputsize"
-    ],
-    "setters": {},
-    "types": {},
-    "defaults": {},
-    "type": "RNN"
-  },
-  {
-    "name": "SeqReverseSequence",
-    "baseType": "Module",
-    "params": [
-      "dim"
-    ],
-    "setters": {},
-    "types": {},
-    "defaults": {},
-    "type": "RNN"
-  }
-]

src/plugins/GenerateArchitecture/GenerateArchitecture.js

@@ -49,10 +49,6 @@ define([
         this.LayerDict = createLayerDict(this.core, this.META);
         this.uniqueId = 2;
         this.varnames = {net: true};
-        this.definitions = [
-            'require \'nn\'',
-            'require \'rnn\''
-        ];
 
         return PluginBase.prototype.main.apply(this, arguments);
     };
@@ -76,6 +72,11 @@ define([
             result = {},
             code = '';
 
+        this.definitions = [
+            'import torch',
+            'import torch.nn as nn'
+        ];
+
         // Add an index to each layer
         layers.forEach((l, index) => l[INDEX] = index);
 
@@ -85,6 +86,7 @@ define([
             code += this.genLayerDefinitions(layers);
         }
 
+        // TODO: Define the network w/ 'class ARCHITECTURE_NAME'
         this.logger.debug('Generating architecture code...');
         code += this.genArchCode(layers);
         this.logger.debug('Prepending hoisted code...');

src/plugins/GenerateJob/GenerateJob.js

@@ -107,16 +107,24 @@ define([
             .then(mds => {
                 // Record the large files
                 var inputData = {},
-                    runsh = '# Bash script to download data files and run job\n' +
-                    'if [ -z "$DEEPFORGE_URL" ]; then\n  echo "Please set DEEPFORGE_URL and' +
-                    ' re-run:"\n  echo ""  \n  echo "  DEEPFORGE_URL=http://my.' +
-                    'deepforge.server.com:8080 bash run.sh"\n  echo ""\n exit 1\nfi\n';
+                    runsh = [
+                        '# Bash script to download data files and run job',
+                        'if [ -z "$DEEPFORGE_URL" ]; then',
+                        '  echo "Please set DEEPFORGE_URL and re-run:"',
+                        '  echo ""',
+                        '  echo "  DEEPFORGE_URL=http://my.deepforge.server.com:8080 bash run.sh"',
+                        '  echo ""',
+                        '  exit 1',
+                        'fi',
+                        'mkdir outputs\n'
+                    ].join('\n');
 
                 mds.forEach((metadata, i) => {
                     // add the hashes for each input
                     var input = inputs[i], 
                         hash = files.inputAssets[input],
-                        dataPath = 'inputs/' + input + '/data',
+                        dataDir = 'inputs/' + input + '/',
+                        dataPath = dataDir + 'data',
                         url = this.blobClient.getRelativeDownloadURL(hash);
 
                     inputData[dataPath] = {
@@ -125,12 +133,13 @@ define([
                     };
 
                     // Add to the run.sh file
+                    runsh += `mkdir -p ${dataDir}\n`;
                     runsh += `wget $DEEPFORGE_URL${url} -O ${dataPath}\n`;
                 });
 
                 delete files.inputAssets;
                 files['input-data.json'] = JSON.stringify(inputData, null, 2);
-                runsh += 'th init.lua';
+                runsh += 'python main.py';
                 files['run.sh'] = runsh;
 
                 // Add pointer assets
@@ -209,23 +218,21 @@ define([
     GenerateJob.prototype.createOperationFiles = function (node) {
         var files = {};
         // For each operation, generate the output files:
-        //   inputs/<arg-name>/init.lua  (respective data deserializer)
-        //   pointers/<name>/init.lua  (result of running the main plugin on pointer target - may need a rename)
+        //   inputs/<arg-name>/init.py  (respective data deserializer)
+        //   pointers/<name>/init.py  (result of running the main plugin on pointer target - may need a rename)
         //   outputs/<name>/  (make dirs for each of the outputs)
-        //   outputs/init.lua  (serializers for data outputs)
+        //   outputs/init.py  (serializers for data outputs)
         //
-        //   attributes.lua (returns lua table of operation attributes)
-        //   init.lua (main file -> calls main and serializes outputs)
-        //   <name>.lua (entry point -> calls main operation code)
+        //   attributes.py (returns py table of operation attributes)
+        //   init.py (main file -> calls main and serializes outputs)
+        //   <name>.py (entry point -> calls main operation code)
 
         // add the given files
-        this.logger.info('About to create dist execution files');
-        files['start.js'] = _.template(Templates.START)(CONSTANTS);
+        this.logger.info('About to generate operation execution files');
         return this.createEntryFile(node, files)
             .then(() => this.createClasses(node, files))
             .then(() => this.createCustomLayers(node, files))
             .then(() => this.createInputs(node, files))
-            .then(() => this.createOutputs(node, files))
             .then(() => this.createMainFile(node, files))
             .then(() => {
                 this.createAttributeFile(node, files);
@@ -238,7 +245,8 @@ define([
     };
 
     GenerateJob.prototype.createEntryFile = function (node, files) {
-        this.logger.info('Creating entry files...');
+        this.logger.info('Creating deepforge.py file...');
+        files['deepforge.py'] = _.template(Templates.DEEPFORGE)(CONSTANTS);
         return this.getOutputs(node)
             .then(outputs => {
                 var name = this.getAttribute(node, 'name'),
@@ -246,12 +254,9 @@ define([
 
                 // inputs and outputs
                 content.name = name;
-                content.outputs = outputs;
-
-                files['init.lua'] = _.template(Templates.ENTRY)(content);
+                content.outputs = outputs.map(output => output[0]);
 
                 // Create the deepforge file
-                files['deepforge.lua'] = _.template(Templates.DEEPFORGE)(CONSTANTS);
             });
     };
 
@@ -294,17 +299,18 @@ define([
 
             return inheritanceLvl[aId] > inheritanceLvl[bId];
         }).map(node =>
-            `require './${this.getAttribute(node, 'name')}.lua'`
+            // FIXME: update this
+            `require './${this.getAttribute(node, 'name')}.py'`
         ).join('\n');
 
         // Create the class files
         classNodes.forEach(node => {
             var name = this.getAttribute(node, 'name');
-            files[`classes/${name}.lua`] = this.getAttribute(node, 'code');
+            files[`classes/${name}.py`] = this.getAttribute(node, 'code');
         });
 
         // Create the custom layers file
-        files['classes/init.lua'] = code;
+        files['classes/init.py'] = code;
     };
 
     GenerateJob.prototype.getTypeDictFor = function (name, metanodes) {
@@ -318,6 +324,7 @@ define([
         return isType;
     };
 
+    // TODO: update this to nn modules
     GenerateJob.prototype.createCustomLayers = function (node, files) {
         var metaDict = this.core.getAllMetaNodes(this.rootNode),
             isCustomLayer,
@@ -337,7 +344,7 @@ define([
             .map(node => this.getAttribute(node, 'code')).join('\n');
 
         // Create the custom layers file
-        files['custom-layers.lua'] = code;
+        files['custom-layers.py'] = code;
     };
 
     GenerateJob.prototype.getConnectionContainer = function () {
@@ -374,11 +381,15 @@ define([
                 //
                 // For each input,
                 //  - create the deserializer
-                //  - put it in inputs/<name>/init.lua
-                //  - copy the data asset to /inputs/<name>/init.lua
+                //  - put it in inputs/<name>/init.py
+                //  - copy the data asset to /inputs/<name>/init.py
                 inputs = allInputs
                     .filter(pair => !!this.getAttribute(pair[2], 'data'));  // remove empty inputs
 
+                files['start.js'] = _.template(Templates.START)({
+                    CONSTANTS,
+                    inputs: inputs.map(pair => pair[0])
+                });
                 files.inputAssets = {};  // data assets
                 return Q.all(inputs.map(pair => {
                     var name = pair[0],
@@ -414,67 +425,25 @@ define([
             })
             .then(_tplContents => {
                 tplContents = _tplContents;
-                var hashes = inputs.map(pair => {
+                inputs.forEach(pair => {
                     var hash = this.getAttribute(pair[2], 'data');
                     files.inputAssets[pair[0]] = hash;
-                    return {
-                        hash: hash,
-                        name: pair[0]
-                    };
                 });
 
-                return Q.all(hashes.map(pair => 
-                    this.blobClient.getMetadata(pair.hash)
-                        .fail(() => {
-                            throw Error(`BLOB_FETCH_FAILED:${pair.name}`);
-                        })));
-            })
-            .then(metadatas => {
-                // Create the deserializer
-                tplContents.forEach((ctnt, i) => {
-                    // Get the name of the given asset
-                    ctnt.filename = metadatas[i].name;
-                    files['inputs/' + ctnt.name + '/init.lua'] = _.template(Templates.DESERIALIZE)(ctnt);
-                });
                 return files;
             });
     };
 
-    GenerateJob.prototype.createOutputs = function (node, files) {
-        // For each of the output types, grab their serialization functions and
-        // create the `outputs/init.lua` file
-        this.logger.info('Creating outputs/init.lua...');
-        return this.getOutputs(node)
-            .then(outputs => {
-                var outputTypes = outputs
-                // Get the serialize functions for each
-                    .map(tuple => {
-                        var node = tuple[2],
-                            serFn = this.getAttribute(node, 'serialize');
-
-                        if (this.isMetaTypeOf(node, this.META.Complex)) {
-                            // Complex objects are expected to define their own
-                            // serialize methods
-                            serFn = 'if data ~= nil then data:serialize(path) end';
-                        }
-
-                        return [tuple[1], serFn];
-                    });
-
-                files['outputs/init.lua'] = _.template(Templates.SERIALIZE)({types: outputTypes});
-            });
-    };
-
     GenerateJob.prototype.createMainFile = function (node, files) {
         this.logger.info('Creating main file...');
+        var content = {};
         return this.getInputs(node)
             .then(inputs => {
                 var name = this.getAttribute(node, 'name'),
                     code = this.getAttribute(node, 'code'),
-                    pointers = this.core.getPointerNames(node).filter(ptr => ptr !== 'base'),
-                    content = {
-                        name: name
-                    };
+                    pointers = this.core.getPointerNames(node).filter(ptr => ptr !== 'base');
+
+                content.name = name;
 
                 // Get input data arguments
                 content.inputs = inputs
@@ -486,11 +455,16 @@ define([
 
                 // Add remaining code
                 content.code = code;
+                return this.getOutputs(node);
+            })
+            .then(outputs => {
+                content.outputs = outputs.map(output => output[0]);
 
-                files['main.lua'] = _.template(Templates.MAIN)(content);
+                files['main.py'] = _.template(Templates.MAIN)(content);
+                files['operations.py'] = content.code;
 
                 // Set the line offset
-                var lineOffset = this.getLineOffset(files['main.lua'], code);
+                var lineOffset = 0;
                 this.setAttribute(node, CONSTANTS.LINE_OFFSET, lineOffset);
             });
     };
@@ -504,6 +478,7 @@ define([
 
     GenerateJob.prototype.createAttributeFile = function (node, files) {
         var numOrBool = /^(-?\d+\.?\d*((e|e-)\d+)?|(true|false))$/,
+            isBool = /^(true|false)$/,
             table;
 
         this.logger.info('Creating attributes file...');
@@ -514,12 +489,16 @@ define([
                 if (!numOrBool.test(value)) {
                     value = `"${value}"`;
                 }
-                return [`['${name}']`, value];
+                if (isBool.test(value)) {  // Convert to python bool
+                    value = value.toString();
+                    value = value[0].toUpperCase() + value.slice(1);
+                }
+                return [`'${name}'`, value];
             })
-            .map(pair => pair.join(' = '))
-            .join(',\n\t') + '\n}';
+            .map(pair => pair.join(': '))
+            .join(',\n    ') + '\n}';
 
-        files['attributes.lua'] = `-- attributes of ${this.getAttribute(node, 'name')}\nreturn ${table}`;
+        files['attributes.py'] = `# attributes of ${this.getAttribute(node, 'name')}\nattributes = ${table}`;
     };
 
     GenerateJob.prototype.createPointers = function (node, files, cb) {
@@ -540,7 +519,7 @@ define([
             var name = this.getAttribute(node, 'name');
             this.logger.info(`Pointer generation for ${name} FINISHED!`);
             resultHashes.forEach((hash, index) => {
-                files.ptrAssets[`pointers/${pointers[index]}/init.lua`] = hash;
+                files.ptrAssets[`pointers/${pointers[index]}/init.py`] = hash;
             });
             return cb(null, files);
         })

src/plugins/GenerateJob/templates/deepforge.ejs

@@ -1,105 +1,105 @@
--- Instantiate the deepforge object
-deepforge = {}
+# Instantiate the deepforge object
 
-function deepforge.initialize()
-    require 'nn'
-    require 'rnn'
-    require './classes/init'
-    require './custom-layers'
-end
+#class deepforge()
+# function deepforge.initialize()
+    # require 'nn'
+    # require 'rnn'
+    # require './classes/init'
+    # require './custom-layers'
+# end
 
-function deepforge.id()
-    if __deepforge_id == nil then
-        __deepforge_id = 0
-    end
-    __deepforge_id = __deepforge_id + 1
-    return __deepforge_id
-end
-
-function deepforge._cmd(...)
-    local cmd = '<%= START_CMD %>'
-    local arg = {...}
-    local n = #arg
-    for i=1,n do
-        cmd = cmd .. ' ' .. tostring(arg[i])
-    end
-    print(cmd .. ' ')  -- guarantee ends w/ space
-end
-
--- Graph support
-Graph = torch.class('deepforge.Graph')
-
-function Graph:__init(name)
-    self.id = deepforge.id()
-    self.name = name
-    deepforge._cmd('<%= GRAPH_CREATE %>', self.id, name)
-end
-
-_Line = torch.class('deepforge._Line')
-
-function _Line:__init(graphId, name, opts)
-    self.id = deepforge.id()
-    self.name = name
-    deepforge._cmd('<%= GRAPH_CREATE_LINE %>', graphId, self.id, name)
-end
-
-function _Line:add(x, y)
-    assert(type(x) == "number" and type(y) == "number", "adding point (" .. tostring(x) .. ", " .. tostring(y) .. ") to " .. self.name .. " failed: expected (number, number)")
-    deepforge._cmd('<%= GRAPH_PLOT %>', self.id, x, y)
-end
-
-function Graph:line(name, opts)
-    return deepforge._Line(self.id, name, opts)
-end
-
-function Graph:xlabel(name)
-    deepforge._cmd('<%= GRAPH_LABEL_AXIS.X %>', self.id, name)
-end
-
-function Graph:ylabel(name)
-    deepforge._cmd('<%= GRAPH_LABEL_AXIS.Y %>', self.id, name)
-end
-
--- Image support
-local function saveImage(name, tensor)
-    require 'image'
-    require 'paths'
-
-    -- save it in the tmp directory
-    local filename = name .. '.png'
-    local path = paths.concat('metadata', filename)
-
-    if paths.dir('metadata') == nil then
-       paths.mkdir('metadata')
-    end
-
-    image.save(path, tensor)
-end
-
-function deepforge.image(name, tensor)
-    saveImage(name, tensor)
-    deepforge._cmd("<%= IMAGE.BASIC %>", deepforge.id(), name)
-end
-
-Image = torch.class('deepforge.Image')
-function Image:__init(name, tensor)
-    self.id = deepforge.id()
-    self.name = name
-
-    if tensor ~= nil then
-        saveImage(name, tensor)
-        deepforge._cmd('<%= IMAGE.CREATE %>', self.id, self.name)
-    end
-end
-
-function Image:update(tensor)
-    saveImage(self.name, tensor)
-    deepforge._cmd('<%= IMAGE.UPDATE %>', self.id, self.name)
-end
-
-function Image:title(name)
-    self.name = name
-    deepforge._cmd('<%= IMAGE.NAME %>', self.id, self.name)
-end
-
-return deepforge
+# function deepforge.id()
+    # if __deepforge_id == nil then
+        # __deepforge_id = 0
+    # end
+    # __deepforge_id = __deepforge_id + 1
+    # return __deepforge_id
+# end
+# 
+# function deepforge._cmd(...)
+    # local cmd = '<%= START_CMD %>'
+    # local arg = {...}
+    # local n = #arg
+    # for i=1,n do
+        # cmd = cmd .. ' ' .. tostring(arg[i])
+    # end
+    # print(cmd .. ' ')  # guarantee ends w/ space
+# end
+# 
+# # Graph support
+# Graph = torch.class('deepforge.Graph')
+# 
+# function Graph:__init(name)
+    # self.id = deepforge.id()
+    # self.name = name
+    # deepforge._cmd('<%= GRAPH_CREATE %>', self.id, name)
+# end
+# 
+# _Line = torch.class('deepforge._Line')
+# 
+# function _Line:__init(graphId, name, opts)
+    # self.id = deepforge.id()
+    # self.name = name
+    # deepforge._cmd('<%= GRAPH_CREATE_LINE %>', graphId, self.id, name)
+# end
+# 
+# function _Line:add(x, y)
+    # assert(type(x) == "number" and type(y) == "number", "adding point (" .. tostring(x) .. ", " .. tostring(y) .. ") to " .. self.name .. " failed: expected (number, number)")
+    # deepforge._cmd('<%= GRAPH_PLOT %>', self.id, x, y)
+# end
+# 
+# function Graph:line(name, opts)
+    # return deepforge._Line(self.id, name, opts)
+# end
+# 
+# function Graph:xlabel(name)
+    # deepforge._cmd('<%= GRAPH_LABEL_AXIS.X %>', self.id, name)
+# end
+# 
+# function Graph:ylabel(name)
+    # deepforge._cmd('<%= GRAPH_LABEL_AXIS.Y %>', self.id, name)
+# end
+# 
+# # Image support
+# local function saveImage(name, tensor)
+    # require 'image'
+    # require 'paths'
+# 
+    # # save it in the tmp directory
+    # local filename = name .. '.png'
+    # local path = paths.concat('metadata', filename)
+# 
+    # if paths.dir('metadata') == nil then
+       # paths.mkdir('metadata')
+    # end
+# 
+    # image.save(path, tensor)
+# end
+# 
+# function deepforge.image(name, tensor)
+    # saveImage(name, tensor)
+    # deepforge._cmd("<%= IMAGE.BASIC %>", deepforge.id(), name)
+# end
+# 
+# Image = torch.class('deepforge.Image')
+# function Image:__init(name, tensor)
+    # self.id = deepforge.id()
+    # self.name = name
+# 
+    # if tensor ~= nil then
+        # saveImage(name, tensor)
+        # deepforge._cmd('<%= IMAGE.CREATE %>', self.id, self.name)
+    # end
+# end
+# 
+# function Image:update(tensor)
+    # saveImage(self.name, tensor)
+    # deepforge._cmd('<%= IMAGE.UPDATE %>', self.id, self.name)
+# end
+# 
+# function Image:title(name)
+    # self.name = name
+    # deepforge._cmd('<%= IMAGE.NAME %>', self.id, self.name)
+# end
+# 
+# return deepforge

src/plugins/GenerateJob/templates/entry.ejs

@@ -1,16 +1,7 @@
--- Instantiate the deepforge object
-require './deepforge'
+import main
+import pickle
 
--- run the <%= name %> and serialize the results
-print('\n############### Running "<%= name.replace(/'/g, '\\\'') %>" Operation ############### ')
-results = require './main'
-print('############### "<%= name.replace(/'/g, '\\\'') %>" Operation Complete! ###############')
+# run the <%= name %> and serialize the results
 
--- serialize by type
-outputs = require './outputs'
-<% outputs.forEach(pair => { 
-    var name = pair[0],
-        type = pair[1];
-%>
-outputs.<%= type %>('<%= name %>', results.<%= name %>)
-<% }); %>
+# serialize by type
+import outputs

src/plugins/GenerateJob/templates/index.js

@@ -1,14 +1,12 @@
 /*globals define*/
 define([
     'text!./start.ejs',
-    'text!./entry.ejs',
     'text!./main.ejs',
     'text!./deepforge.ejs',
     'text!./serialize.ejs',
     'text!./deserialize.ejs'
 ], function(
     START,
-    ENTRY,
     MAIN,
     DEEPFORGE,
     SERIALIZE,
@@ -17,7 +15,6 @@ define([
 
     return {
         START,
-        ENTRY,
         MAIN,
         SERIALIZE,
         DEEPFORGE,

src/plugins/GenerateJob/templates/main.ejs

@@ -1,12 +1,29 @@
--- load custom layers and classes
-deepforge.initialize()
+import pickle
+# load custom layers and classes
+# from deepforge import deepforge
 
--- input data<% inputs.forEach(function(pair) { var input = pair[0], isNil = pair[1];%>
-local <%= input %> = <% if (isNil) { %>nil<% } else { %>require './inputs/<%= input %>'<%}}); %>
+# deepforge.initialize()
 
--- load references<% pointers.forEach(function(pair) { var pointer = pair[0], isNil = pair[1];%>
-local <%= pointer %> = <% if (isNil) { %>nil<% } else { %>require './pointers/<%= pointer %>'<%}}); %>
-local attributes = require './attributes'
+# input data<% inputs.forEach(function(pair) { var input = pair[0], isNil = pair[1];%>
+<%= input %> = <% if (isNil) { %>None<% } else { %>pickle.load(open('./inputs/<%= input %>/data', 'rb')) <%}}); %>
 
--- main operation code for <%= name %>
-<%= code %>
+# load references<% pointers.forEach(function(pair) { var pointer = pair[0], isNil = pair[1];%>
+from pointers import <%= pointer %>
+<%}); %>
+
+from operations import <%= name %>Operation
+from attributes import attributes
+
+# main operation code for <%= name %>
+operation = <%= name %>Operation()
+print('\n############### Running "<%= name.replace(/'/g, '\\\'') %>" Operation ############### ')
+<%= outputs.length ? outputs.join(', ') : 'result' %> = operation.execute(<%= inputs.map(function(pair) {
+    var name = pair[0],
+        isNone = pair[1];
+    return isNone ? 'None' : name;
+})%>)
+print('############### "<%= name.replace(/'/g, '\\\'') %>" Operation Complete! ###############')
+
+<% outputs.forEach(name => { %>
+pickle.dump(<%= name %>, open('outputs/<%= name %>', 'wb'))
+<% }); %>

src/plugins/GenerateJob/templates/serialize.ejs

@@ -1,16 +1,17 @@
--- Serialization functions for: <%= types.map(function(type) {return type[0];}).join('\n--   ') %>
-require 'paths'
+# Serialization functions for: <%= types.map(function(type) {return type[0];}).join('\n--   ') %>
 
-local serializer = {}
+serializer = {}
 <% types.forEach(function(pair) {
 var type = pair[0],
-    fn = pair[1];
-%>
-function serializer.<%= type %> (name, data)
-    local path = 'outputs/' .. name
-    local abs_path = paths.concat('outputs', name)
-    <%= fn.replace('\n', '\n    ') %>
-end
-<% }); %>
+    fn = pair[1],
+    safeType = type.replace(/[^a-zA-Z\d_]/g, '_');
 
-return serializer
+%>
+
+def <%= safeType %> (name, data):
+    path = 'outputs/' .. name
+    <%= fn.replace('\n', '\n    ') %>
+
+serializer['<%= type %>'] = <%= safeType %>
+
+<% }); %>

src/plugins/GenerateJob/templates/start.ejs

@@ -11,8 +11,8 @@ var spawn = require('child_process').spawn,
 ['error', 'warn', 'info', 'log', 'debug'].forEach(method => logger[method] = log);
 
 // Get the BlobClient...
-var COMMAND_PREFIX = '<%= START_CMD %>',
-    IMAGE = '<%= IMAGE.PREFIX %>',
+var COMMAND_PREFIX = '<%= CONSTANTS.START_CMD %>',
+    IMAGE = '<%= CONSTANTS.IMAGE.PREFIX %>',
     requirejs = require('webgme').requirejs,
     remainingImageCount = 0,
     exitCode = null;
@@ -35,14 +35,23 @@ requirejs([
 
     // Create CACHE_DIR if it doesn't exist
     var prepareCache = function() {
-    var dirs = CACHE_DIR.replace(/\/$/, '').split('/'),
-        cacheParent;
+        var dirs = CACHE_DIR.replace(/\/$/, '').split('/'),
+            cacheParent;
 
         dirs.pop();
         cacheParent = dirs.join('/');
         return makeIfNeeded(cacheParent).then(() => makeIfNeeded(CACHE_DIR));
     };
 
+    var prepareInputsOutputs = function() {
+        var dirs = ['inputs', <% inputs.forEach(function(input) { %>
+            'inputs/<%= input %>',
+            <% }) %>
+            'outputs'];
+        return Q.all(dirs.map(dir => makeIfNeeded(dir)));
+    };
+
+
     var makeIfNeeded = function(dir) {
         var deferred = Q.defer(),
             job;
@@ -109,7 +118,7 @@ requirejs([
     var onStderr = function(data) {
         var text = data.toString();
         // Filter out directory label from stack traces
-        process.stdout.write(text.replace(/\.\.\.\/.*\/(main|deepforge|init).lua/g, '$1'));
+        process.stdout.write(text.replace(/\.\.\.\/.*\/(main|deepforge|init).py/g, '$1'));
     };
 
     var onStdout = function(data) {
@@ -229,17 +238,19 @@ requirejs([
         cleanup();
         process.exit(130);
     });
-    process.on('uncaughtException', () => {
+    process.on('uncaughtException', err => {
         log('received "uncaughtException" event')
+        log(err);
         cleanup();
     });
 
     // Request the data from the blob
     prepareCache()
+        .then(prepareInputsOutputs)
         .then(() => Q.all(inputPaths.map(ipath => getData(ipath, inputData[ipath]))))
         .then(() => {
-            // Run 'th init.lua' and merge the stdout, stderr
-            job = spawn('th', ['init.lua'], {detached: true});
+            // Run 'python main.py' and merge the stdout, stderr
+            job = spawn('python', ['main.py'], {detached: true});
             job.stdout.on('data', onStdout);
             job.stderr.on('data', onStderr);
             job.on('close', code => {

src/seeds/nn/version.txt

@@ -1 +1 @@
-0.5.0
+1.0.3

src/seeds/pipeline/version.txt

@@ -1 +1 @@
-0.6.0
+0.7.1

src/visualizers/panels/LayerEditor/LayerEditorControl.js

@@ -15,7 +15,7 @@ define([
 
     'use strict';
 
-    var NO_CODE_MESSAGE = '-- <%= name %> is not an editable layer!',
+    var NO_CODE_MESSAGE = '<%= name %> is not an editable layer!',
         LayerEditorControl;
 
     LayerEditorControl = function (options) {
@@ -45,10 +45,10 @@ define([
                 // Retrieve the template from the mixin
                 template = node.getMixinPaths()
                     .map(id => this._client.getNode(id).getAttribute('code'))
-                    .find(code => !!code) || NO_CODE_MESSAGE;
+                    .find(code => !!code) || this.comment(NO_CODE_MESSAGE);
             }
         } else {
-            template = NO_CODE_MESSAGE;
+            template = this.comment(NO_CODE_MESSAGE);
         }
 
         if (template) {

src/visualizers/panels/OperationCodeEditor/OperationCodeEditorControl.js

@@ -3,13 +3,17 @@
 
 define([
     'panels/TextEditor/TextEditorControl',
+    'text!./boilerplate.ejs',
     'deepforge/viz/OperationControl',
+    'deepforge/OperationCode',
     'deepforge/viz/Execute',
     'deepforge/Constants',
     'underscore'
 ], function (
     TextEditorControl,
+    CodeTemplate,
     OperationControl,
+    OperationCode,
     Execute,
     CONSTANTS,
     _
@@ -18,6 +22,7 @@ define([
     'use strict';
 
     var OperationCodeEditorControl;
+    var GenerateBoilerplate = _.template(CodeTemplate);
 
     OperationCodeEditorControl = function (options) {
         options.attributeName = 'code';
@@ -48,6 +53,11 @@ define([
         desc.inputs = this.getOperationInputs(node).map(id => this.formatIO(id));
         desc.outputs = this.getOperationOutputs(node).map(id => this.formatIO(id));
         desc.references = node.getPointerNames().filter(name => name !== 'base');
+
+        // Create the boilerplate operation code, if applicable
+        if (!desc.ownText) {
+            desc.text = GenerateBoilerplate(desc);
+        }
         return desc;
     };
 
@@ -68,6 +78,51 @@ define([
         }
     };
 
+    OperationCodeEditorControl.prototype.saveTextFor = function (id, code) {
+        try {
+            // Parse the operation implementation and detect change in inputs/outputs
+            // TODO: Update this to use the code object
+            var operation = new OperationCode(code),
+                oldInputs = this.getDataNames(this._currentNodeId, true),
+                currentInputs = operation.getInputs().map(input => input.name),
+                name = this._client.getNode(this._currentNodeId).getAttribute('name'),
+                newInputs,
+                rmInputs,
+                oldOutputs = this.getDataNames(this._currentNodeId),
+                currentOutputs = operation.getOutputs().map(input => input.name),
+                newOutputs,
+                rmOutputs;
+
+            // Check for input nodes to remove
+            if (currentInputs[0] === 'self') currentInputs.shift();
+            newInputs = _.difference(currentInputs, oldInputs);
+            rmInputs = _.difference(oldInputs, currentInputs);
+            newOutputs = _.difference(currentOutputs, oldOutputs);
+            rmOutputs = _.difference(oldOutputs, currentOutputs);
+
+            if (rmInputs.length || newInputs.length || rmOutputs.length || newOutputs.length) {
+                var msg = `Updating operation implementation for ${name}`;
+
+                this._client.startTransaction(msg);
+                TextEditorControl.prototype.saveTextFor.call(this, id, code, true);
+
+                // update the inputs
+                rmInputs.forEach(input => this.removeInputData(this._currentNodeId, input));
+                newInputs.map(input => this.addInputData(this._currentNodeId, input));
+
+                // update the outputs
+                rmOutputs.forEach(output => this.removeOutputData(this._currentNodeId, output));
+                newOutputs.map(output => this.addOutputData(this._currentNodeId, output));
+                this._client.completeTransaction();
+            } else {
+                return TextEditorControl.prototype.saveTextFor.call(this, id, code);
+            }
+        } catch (e) {
+            this._logger.debug(`failed parsing operation: ${e}`);
+            return TextEditorControl.prototype.saveTextFor.call(this, id, code);
+        }
+    };
+
     OperationCodeEditorControl.prototype.getOperationAttributes = function () {
         var node = this._client.getNode(this._currentNodeId),
             attrs = node.getValidAttributeNames(),

src/visualizers/panels/OperationCodeEditor/boilerplate.ejs

@@ -0,0 +1,4 @@
+class <%= name %>Operation():
+
+    def execute(self):
+        # Execute your operation here!

src/visualizers/panels/OperationEditor/OperationEditorPanel.js

@@ -71,6 +71,33 @@ define([
         this.setTitle(name || '');
     };
 
+    OperationEditorPanel.prototype.editTitle = function () {
+        this.$panelHeaderTitle.editInPlace({
+            css: {
+                'z-index': 1000
+            },
+            onChange: (oldValue, newValue) => {
+                var nodeId = this.currentNodeId(),
+                    type = this.currentBaseName(),
+                    words = newValue.split(' '),
+                    msg;
+
+                if (words.length > 1) {
+                    newValue = words.map(word => word[0].toUpperCase() + word.substring(1)).join('');
+                }
+
+                newValue = newValue.replace(/Operation$/, '');
+                msg = `Renamed ${type}: ${oldValue} -> ${newValue}`;
+
+                if (!/^\s*$/.test(newValue)) {
+                    this._client.startTransaction(msg);
+                    this._client.setAttribute(nodeId, 'name', newValue);
+                    this._client.completeTransaction();
+                }
+            }
+        });
+    };
+
     OperationEditorPanel.prototype.getPanels = function () {
         return [InterfaceEditor, CodeEditor];
     };

src/visualizers/panels/OperationInterfaceEditor/OperationInterfaceEditorControl.EventHandlers.js

@@ -1,7 +1,11 @@
 /*globals define*/
 define([
+    'panels/EasyDAG/EasyDAGControl.WidgetEventHandlers',
+    'deepforge/OperationCode',
     './Colors'
 ], function(
+    EasyDAGControlEventHandlers,
+    OperationCode,
     COLORS
 ) {
     'use strict';
@@ -66,36 +70,14 @@ define([
             .filter(node => !node.isAbstract());
     };
 
-    OperationInterfaceEditorEvents.prototype.getValidSuccessors = function(nodeId, isInput) {
-        var dataTypeIds;
-
+    OperationInterfaceEditorEvents.prototype.getValidSuccessors = function(nodeId) {
         if (nodeId !== this._currentNodeId) {
             return [];
         }
 
-        // Return all data types in the meta
-        // If input, include abstract types
-        dataTypeIds = this.allDataTypeIds(isInput);
-
-        return dataTypeIds.map(id => {
-            return {
-                node: this._getObjectDescriptor(id)
-            };
-        });
-    };
-
-    OperationInterfaceEditorEvents.prototype._getDataName = function(cntrId, typeId) {
-        var otherIds = this._client.getNode(cntrId).getChildrenIds(),
-            otherNames = otherIds.map(id => this._client.getNode(id).getAttribute('name')),
-            baseName = this._client.getNode(typeId).getAttribute('name').toLowerCase(),
-            name = baseName,
-            i = 1;
-
-        while (otherNames.indexOf(name) !== -1) {
-            i++;
-            name = baseName + '_' + i;
-        }
-        return name;
+        return [{
+            node: this._getObjectDescriptor(this.getDataTypeId())
+        }];
     };
 
     OperationInterfaceEditorEvents.prototype.getRefName = function(node, basename) {
@@ -193,26 +175,92 @@ define([
         this._client.completeTransaction();
     };
 
-    OperationInterfaceEditorEvents.prototype._createConnectedNode = function(typeId, isInput) {
+    OperationInterfaceEditorEvents.prototype._createConnectedNode = function(typeId, isInput, baseName) {
         var node = this._client.getNode(this._currentNodeId),
             name = node.getAttribute('name'),
-            cntrs = node.getChildrenIds(),
-            cntrType = isInput ? 'Inputs' : 'Outputs',
-            cntrId = cntrs.find(id => this.hasMetaName(id, cntrType)),
-            dataName = this._getDataName(cntrId, typeId),
+            msg = `Updating the interface of ${name}`,
+            code = node.getAttribute('code'),
+            id,
+            operation,
+            dataName;
+
+        // Update the source code if the inputs/outputs changed
+        // we know that we are adding a node, so we don't need to do 
+        // the comparing and diffing current vs new
+
+        this._client.startTransaction(msg);
+        id = this.createIONode(this._currentNodeId, typeId, isInput, baseName, true);
+        dataName = this._client.getNode(id).getAttribute('name');
+
+        try {
+            operation = new OperationCode(code);
+            if (isInput) {
+                operation.addInput(dataName);
+            } else {
+                operation.addOutput(dataName);
+            }
+
+            this._client.setAttribute(this._currentNodeId, 'code', operation.getCode());
+        } catch(e) {
+            this.logger.debug(`could not update the code - invalid python!: ${e}`);
+        }
+        this._client.completeTransaction();
+
+        return id;
+    };
+
+    OperationInterfaceEditorEvents.prototype._deleteNode = function(nodeId) {
+        var dataName = this._client.getNode(nodeId).getAttribute('name'),
+            node = this._client.getNode(this._currentNodeId),
+            name = node.getAttribute('name'),
+            isInput = this.isInputData(nodeId),
+            msg = `Updating the interface of ${name}`,
+            code = node.getAttribute('code'),
+            operation = new OperationCode(code);
+
+        // If the input name is used in the code, maybe just comment it out in the args
+        this._client.startTransaction(msg);
+        try {
+            if (isInput) {
+                operation.removeInput(dataName);
+            } else {
+                operation.removeOutput(dataName);
+            }
+            this._client.setAttribute(this._currentNodeId, 'code', operation.getCode());
+        } catch(e) {
+            this.logger.debug(`could not update the code - invalid python!: ${e}`);
+        }
+        this._client.deleteNode(nodeId);
+        //EasyDAGControlEventHandlers.prototype._deleteNode.apply(this, nodeId, true);
+        this._client.completeTransaction();
+    };
+
+    OperationInterfaceEditorEvents.prototype._saveAttributeForNode = function(nodeId, attr, value) {
+        // If nodeId is an input data node, rename the input
+        // If nodeId is an output data node, rename the output
+        var isDataNode = nodeId.indexOf(this._currentNodeId) === 0,
+            node = this._client.getNode(this._currentNodeId),
+            code = node.getAttribute('code'),
             msg;
 
-        msg = `Adding ${isInput ? 'input' : 'output'} "${dataName}" to ${name} interface`;
-        this._client.startTransaction(msg);
-        var id = this._client.createNode({
-            parentId: cntrId,
-            baseId: typeId
-        });
+        if (isDataNode && attr === 'name') {  // rename input/output
+            var operation = new OperationCode(code),
+                dataNode = this._client.getNode(nodeId),
+                oldName = dataNode.getAttribute(attr);
 
-        // Set the name of the new input
-        this._client.setAttribute(id, 'name', dataName);
+            operation.rename(oldName, value);
 
-        this._client.completeTransaction();
+            msg = `Renaming ${oldName}->${value} in ${name}`;
+            this._client.startTransaction(msg);
+            EasyDAGControlEventHandlers.prototype._saveAttributeForNode.apply(this, arguments);
+            this._client.setAttribute(this._currentNodeId, 'code', operation.getCode());
+            this._client.completeTransaction();
+        } else if (nodeId === this._currentNodeId) {  // edit operation attributes
+            // TODO: rename operation
+            // TODO: set operation attribute default
+            console.log('setting attr', arguments);
+            EasyDAGControlEventHandlers.prototype._saveAttributeForNode.apply(this, arguments);
+        }
     };
 
     return OperationInterfaceEditorEvents;

src/visualizers/panels/OperationInterfaceEditor/OperationInterfaceEditorControl.js

@@ -10,7 +10,6 @@ define([
     'panels/EasyDAG/EasyDAGControl',
     'js/Constants',
     'deepforge/Constants',
-    'deepforge/lua',
     'deepforge/viz/OperationControl',
     './OperationInterfaceEditorControl.EventHandlers',
     './Colors',
@@ -19,7 +18,6 @@ define([
     EasyDAGControl,
     GME_CONSTANTS,
     CONSTANTS,
-    luajs,
     OperationControl,
     OperationInterfaceEditorControlEvents,
     COLORS,
@@ -229,20 +227,13 @@ define([
 
             code = this._client.getNode(this._currentNodeId).getAttribute('code');
             try {
-                ast = luajs.parser.parse(code);
-                for (var i = variableIds.length; i--;) {
-                    wasUsed = this._usage[variableIds[i]];
-                    name = this._client.getNode(variableIds[i]).getAttribute('name');
-
-                    isUsed = this._inputs[variableIds[i]] ?
-                        this.isUsedInput(name, ast) :
-                        this.isUsedOutput(name, ast);
-                    if (isUsed !== wasUsed) {
-                        this._onUpdate(variableIds[i]);
-                    }
-                }
+                // Parse the operation implementation for visual cues
+                // TODO
+                // Parse the operation implementation and detect change in inputs/outputs
+                //var schema = OperationParser.parse(code);
+                //console.log(schema);
             } catch (e) {
-                this._logger.debug(`failed parsing lua: ${e}`);
+                this._logger.debug(`failed parsing operation: ${e}`);
             }
 
         } else if (this.containedInCurrent(gmeId) && this.hasMetaName(gmeId, 'Data')) {
@@ -375,11 +366,13 @@ define([
 
     ////////////////////// Unused input checking //////////////////////
     OperationInterfaceEditorControl.prototype.isUsedInput = function(name, ast) {
-        return this._isUsed(name, true, ast);
+        return true;
+        //return this._isUsed(name, true, ast);
     };
 
     OperationInterfaceEditorControl.prototype.isUsedOutput = function(name, ast) {
-        return this._isUsed(name, false, ast);
+        return true;
+        //return this._isUsed(name, false, ast);
     };
 
     OperationInterfaceEditorControl.prototype._isUsed = function(name, isInput, ast) {
@@ -390,8 +383,9 @@ define([
         // verify that it is not used only in the left side of an assignment
         if (hasText) {
             try {
-                ast = ast || luajs.parser.parse(code);
-                return isInput ? this.isUsedVariable(name, ast) : this.isReturnValue(name, ast);
+                return true;
+                //ast = ast || luajs.parser.parse(code);
+                //return isInput ? this.isUsedVariable(name, ast) : this.isReturnValue(name, ast);
             } catch(e) {
                 this._logger.debug(`failed parsing lua: ${e}`);
                 return null;
@@ -406,19 +400,20 @@ define([
         var isUsed = false,
             checker;
 
-        checker = luajs.codegen.traverse((curr, parent) => {
-            if (curr.type === 'variable' && curr.val === name) {
-                // Ignore if it is being assigned...
-                if (parent.type === 'stat.assignment') {
-                    isUsed = isUsed || parent.right.indexOf(curr) !== -1;
-                } else {
-                    isUsed = true;
-                }
-            }
-            return curr;
-        });
+        return true;
+        //checker = luajs.codegen.traverse((curr, parent) => {
+            //if (curr.type === 'variable' && curr.val === name) {
+                //// Ignore if it is being assigned...
+                //if (parent.type === 'stat.assignment') {
+                    //isUsed = isUsed || parent.right.indexOf(curr) !== -1;
+                //} else {
+                    //isUsed = true;
+                //}
+            //}
+            //return curr;
+        //});
 
-        checker(node);
+        //checker(node);
         return isUsed;
     };
 

src/visualizers/panels/OperationInterfaceEditor/OperationInterfaceEditorPanel.js

@@ -4,14 +4,17 @@
  * Generated by VisualizerGenerator 1.7.0 from webgme on Tue May 31 2016 09:16:24 GMT-0500 (CDT).
  */
 
-define(['js/PanelBase/PanelBaseWithHeader',
+define([
+    'js/PanelBase/PanelBaseWithHeader',
     'js/PanelManager/IActivePanel',
     'widgets/OperationInterfaceEditor/OperationInterfaceEditorWidget',
     './OperationInterfaceEditorControl'
-], function (PanelBaseWithHeader,
-             IActivePanel,
-             OperationInterfaceEditorWidget,
-             OperationInterfaceEditorControl) {
+], function (
+    PanelBaseWithHeader,
+    IActivePanel,
+    OperationInterfaceEditorWidget,
+    OperationInterfaceEditorControl
+) {
     'use strict';
 
     var OperationInterfaceEditorPanel;

src/visualizers/widgets/ClassCodeEditor/ClassCodeEditorWidget.js

@@ -19,9 +19,7 @@ define([
     _.extend(ClassCodeEditorWidget.prototype, TextEditorWidget.prototype);
 
     ClassCodeEditorWidget.prototype.getHeader = function(desc) {
-        return [
-            `-- The class definition for ${desc.name}`
-        ].join('\n');
+        return this.comment(`The class definition for ${desc.name}`);
     };
 
     ClassCodeEditorWidget.prototype.updateNode = function() {

src/visualizers/widgets/DeserializeEditor/DeserializeEditorWidget.js

@@ -26,13 +26,13 @@ define([
 
     DeserializeEditorWidget.prototype.getHeader = function(desc) {
         this._name = desc.name;
-        return [
-            `-- The deserialization function for ${desc.name}`,
-            '-- Globals:',
-            '--   `path` - target filename to load',
-            '--',
-            `-- return the loaded ${desc.name}`
-        ].join('\n');
+        return this.comment([
+            `The deserialization function for ${desc.name}`,
+            'Globals:',
+            '  `path` - target filename to load',
+            '',
+            `return the loaded ${desc.name}`
+        ].join('\n'));
     };
 
     DeserializeEditorWidget.prototype.getNameRegex = function() {

src/visualizers/widgets/OperationCodeEditor/OperationCodeEditorWidget.js

@@ -32,24 +32,24 @@ define([
 
     OperationCodeEditorWidget.prototype.getHeader = function (desc) {
         // Add comment about the inputs, attributes and references
-        var inputs = desc.inputs.map(pair => `-- ${pair[0]} (${pair[1]})`).join('\n'),
-            refs = desc.references.map(name => `-- ${name}`).join('\n'),
+        var inputs = desc.inputs.map(pair => `${pair[0]} (${pair[1]})`).join('\n'),
+            refs = desc.references.map(name => `${name}`).join('\n'),
             header = [
-                `-- Editing "${desc.name}" Implementation`
+                `Editing "${desc.name}" Implementation`
             ];
 
         if (inputs.length) {
-            header.push('--');
-            header.push('-- Defined variables:');
+            header.push('');
+            header.push('Defined variables:');
             header.push(inputs);
         }
         if (refs) {
             header.push(refs);
         }
-        header.push('--');
-        header.push('-- The following will be executed when the operation is run:');
+        header.push('');
+        header.push('The \'execute\' method will be called when the operation is run');
 
-        return header.join('\n');
+        return this.comment(header.join('\n'));
     };
 
     OperationCodeEditorWidget.prototype.canAddReturnTmpl = function (desc) {

src/visualizers/widgets/OperationInterfaceEditor/OperationInterfaceEditorWidget.js

@@ -61,31 +61,8 @@ define([
     };
 
     OperationInterfaceEditorWidget.prototype.onAddButtonClicked = function(item, isInput) {
-        var successorPairs = this.getValidSuccessors(item.id, isInput),
-            newClass = this.getCreationNode('Complex', NEW_CLASS_ID),
-            newPrim = this.getCreationNode('Primitive', NEW_PRIM_ID),
-            opts = {};
-
-        // Add the 'Create Class' node
-        successorPairs.push(newClass);
-        successorPairs.push(newPrim);
-
-        // Add tabs
-        opts.tabs = ['Primitive', 'Classes'];
-        opts.tabFilter = (tab, pair) => {
-            return pair.node.isPrimitive === (tab === 'Primitive');
-        };
-
-        AddNodeDialog.prompt(successorPairs, opts)
-            .then(selected => {
-                if (selected.node.id === NEW_CLASS_ID) {
-                    DeepForge.create.Complex();
-                } else if (selected.node.id === NEW_PRIM_ID) {
-                    DeepForge.create.Primitive();
-                } else {
-                    this.onAddItemSelected(selected, isInput);
-                }
-            });
+        var successorPairs = this.getValidSuccessors(item.id, isInput);
+        return this.onAddItemSelected(successorPairs[0], isInput);
     };
 
     OperationInterfaceEditorWidget.prototype.onDeactivate = function() {
@@ -144,8 +121,7 @@ define([
 
     // Hover buttons
     OperationInterfaceEditorWidget.prototype.showHoverButtons = function(item) {
-        var dataNodes = this.allDataTypeIds(),
-            refNodes = this.allValidReferences(),
+        var refNodes = this.allValidReferences(),
             height = item.height,
             cx = item.width/2;
 
@@ -161,7 +137,6 @@ define([
             new Buttons.AddOutput({  // Add output data
                 context: this,
                 $pEl: this.$hoverBtns,
-                disabled: dataNodes.length === 0,
                 item: item,
                 x: cx,
                 y: height
@@ -170,7 +145,6 @@ define([
             new Buttons.AddInput({  // Add input data
                 context: this,
                 $pEl: this.$hoverBtns,
-                disabled: dataNodes.length === 0,
                 item: item,
                 x: item.width/3,
                 y: 0

src/visualizers/widgets/OperationInterfaceEditor/SelectionManager.js

@@ -1,4 +1,4 @@
-/*globals define*/
+/*globals define, $*/
 
 define([
     'widgets/EasyDAG/SelectionManager',
@@ -25,12 +25,10 @@ define([
 
     SelectionManager.prototype.createActionButtons = function(width, height) {
         var selectedType = this.selectedItem.desc.baseName,
-            dataNodes,
             refNodes,
             cx = width/2;
 
         if (selectedType === 'Operation') {
-            dataNodes = this._widget.allDataTypeIds();
             refNodes = this._widget.allValidReferences();
 
             new Buttons.AddOutput({  // Add output data
@@ -38,15 +36,13 @@ define([
                 $pEl: this.$selection,
                 item: this.selectedItem,
                 x: cx,
-                y: height,
-                disabled: dataNodes.length === 0
+                y: height
             });
 
             new Buttons.AddInput({  // Add input data
                 context: this._widget,
                 $pEl: this.$selection,
                 item: this.selectedItem,
-                disabled: dataNodes.length === 0,
                 x: width/3,
                 y: 0
             });

src/visualizers/widgets/SerializeEditor/SerializeEditorWidget.js

@@ -23,12 +23,12 @@ define([
 
     SerializeEditorWidget.prototype.getHeader = function(desc) {
         this._name = desc.name;
-        return [
-            `-- The serialization function for ${desc.name}`,
-            '-- Globals:',
-            '--   `path` - target filename',
-            `--   \`data\` - the ${desc.name} to store`
-        ].join('\n');
+        return this.comment([
+            `The serialization function for ${desc.name}`,
+            'Globals:',
+            '  `path` - target filename',
+            `  \`data\` - the ${desc.name} to store`
+        ].join('\n'));
     };
 
     SerializeEditorWidget.prototype.getNameRegex = function () {

src/visualizers/widgets/TextEditor/TextEditorWidget.js

@@ -18,6 +18,10 @@ define([
 
     var TextEditorWidget,
         WIDGET_CLASS = 'text-editor',
+        LINE_COMMENT = {
+            python: '#',
+            lua: '--'
+        },
         DEFAULT_SETTINGS = {
             keybindings: 'default',
             theme: 'solarized_dark',
@@ -27,6 +31,7 @@ define([
     TextEditorWidget = function (logger, container) {
         this._logger = logger.fork('Widget');
 
+        this.language = this.language || 'python';
         this._el = container;
         this._el.css({height: '100%'});
         this.$editor = $('<div/>');
@@ -86,8 +91,8 @@ define([
 
     TextEditorWidget.prototype.getSessionOptions = function () {
         return {
-            mode: 'ace/mode/lua',
-            tabSize: 3,
+            mode: 'ace/mode/' + this.language,
+            tabSize: 4,
             useSoftTabs: true
         };
     };
@@ -220,18 +225,33 @@ define([
     };
 
     // Adding/Removing/Updating items
+    TextEditorWidget.prototype.comment = function (text) {
+        var prefix = LINE_COMMENT[this.language] + ' ';
+        return text.replace(
+            new RegExp('^(' + LINE_COMMENT[this.language] + ')?','mg'),
+            prefix
+        );
+    };
+
     TextEditorWidget.prototype.getHeader = function (desc) {
-        return `-- Editing "${desc.name}"`;
+        return this.comment(`Editing "${desc.name}"`);
     };
 
     TextEditorWidget.prototype.addNode = function (desc) {
         // Set the current text based on the given
         // Create the header
-        var header = this.getHeader(desc);
+        var header = this.getHeader(desc),
+            content,
+            newContent = header + '\n' + desc.text,
+            //patches = diff.diffChars(content, newContent),
+            cursorPos;
 
+        // TODO: if we are updating the value, we should make sure the cursor position
+        // remains in the same spot (ie, diff the text and update the positions
+        // based on the size of the patches
         this.activeNode = desc.id;
         this.silent = true;
-        this.editor.setValue(header + '\n' + desc.text, 2);
+        this.editor.setValue(newContent, 2);
         this.silent = false;
         this.currentHeader = header;
     };

test/common/OperationCode.spec.js

@@ -0,0 +1,353 @@
+describe.only('OperationCode', function() {
+    var fs = require('fs');
+    var path = require('path');
+    var assert = require('assert');
+    var OperationCode = require('../../src/common/OperationCode');
+    var operation;
+
+    describe('example', function() {
+        var code;
+
+        before(function() {
+            // load the example
+            var filePath = path.join(__dirname, '..', 'test-cases', 'operations', 'example.py');
+            code = fs.readFileSync(filePath, 'utf8');
+        });
+
+        describe('removeInput', function() {
+            before(function() {
+                operation = new OperationCode(code);
+                operation.removeInput('world');
+            });
+
+            it('should have 2 remaining inputs', function() {
+                assert.equal(operation.getInputs().length, 2);
+            });
+
+        });
+
+        describe('attributes', function() {
+            describe('add', function() {
+                beforeEach(function() {
+                    operation = new OperationCode(code);
+                });
+
+                it('should add argument to __init__ method', function() {
+                    operation.addAttribute('number');
+                    var attrs = operation.getAttributes();
+                    // TODO
+                });
+
+                it('should set the default value', function() {
+                    // TODO
+                });
+            });
+
+            // TODO: add attribute
+            // TODO: remove attribute
+            // TODO: rename attribute?
+        });
+
+        describe('rename', function() {
+            before(function() {
+                operation = new OperationCode(code);
+                operation.rename('hello', 'goodbye');
+            });
+
+            it('should rename input arg', function() {
+                var inputs = operation.getInputs();
+                var oldInput = inputs.find(input => input.name === 'hello');
+                var newInput = inputs.find(input => input.name === 'goodbye');
+
+                assert(!oldInput);
+                assert(newInput);
+            });
+
+            it('should rename occurrences in the fn', function() {
+                assert(!operation.getCode().includes('hello'));
+            });
+        });
+
+        describe('parsing', function() {
+            before(function() {
+                operation = new OperationCode(code);
+            });
+
+            it('should parse the correct name', function() {
+                assert.equal(operation.getName(), 'ExampleOperation');
+            });
+
+            it.skip('should parse the correct base', function() {
+                assert.equal(operation.getBase(), 'Operation');
+            });
+
+            it('should parse the input names', function() {
+                const names = ['hello', 'world', 'count'];
+                assert.deepEqual(operation.getInputs().map(input => input.name), names);
+            });
+
+            it.skip('should parse the input types', function() {
+                const types = ['str', 'str', 'int'];
+                assert.deepEqual(operation.getInputs().map(input => input.type), types);
+            });
+
+            it('should parse the output names', function() {
+                const names = ['concat', 'count'];
+                assert.deepEqual(operation.getOutputs().map(output => output.name), names);
+            });
+
+            it.skip('should parse the output types', function() {
+                const types = ['str', 'int'];
+                assert.deepEqual(operation.getOutputs().map(output => output.type), types);
+            });
+        });
+    });
+
+    describe('multi-anon-results', function() {
+        before(function() {
+            var filePath = path.join(__dirname, '..', 'test-cases', 'operations', 'multi-anon-results.py');
+            var example = fs.readFileSync(filePath, 'utf8');
+            operation = new OperationCode(example);
+        });
+
+        it('should parse multiple return values', function() {
+            assert.equal(operation.getOutputs().length, 2);
+        });
+
+        it('should create unique names for each', function() {
+            var [first, second] = operation.getOutputs();
+            assert.notEqual(first.name, second.name);
+        });
+    });
+
+    describe('no-inputs/outputs', function() {
+        var code;
+
+        before(function() {
+            var filePath = path.join(__dirname, '..', 'test-cases', 'operations', 'no-inputs.py');
+            code = fs.readFileSync(filePath, 'utf8');
+        });
+
+        describe('parsing', function() {
+            beforeEach(function() {
+                operation = new OperationCode(code);
+            });
+
+            it('should not require base class', function() {
+                assert.equal(operation.getBase(), null);
+            });
+
+            it('should detect zero output', function() {
+                assert.equal(operation.getOutputs().length, 0);
+            });
+
+            it('should detect zero inputs', function() {
+                assert.equal(operation.getInputs().length, 0);
+            });
+        });
+
+        describe('addInput', function() {
+            var operation;
+
+            before(function() {
+                operation = new OperationCode(code);
+                operation.addInput('first');
+            });
+
+            it('should clear schema', function() {
+                assert(!operation._schema);
+            });
+
+            it('should add input to `execute` fn', function() {
+                var code = operation.getCode();
+                assert(code.includes('first'));
+            });
+
+            it('should have an additional input arg', function() {
+                var inputs = operation.getInputs();
+                assert.equal(inputs.length, 1);
+            });
+        });
+
+        describe('addOutput', function() {
+            var operation;
+
+            describe('lone return', function() {
+                before(function() {
+                    operation = new OperationCode(code);
+                    operation.addOutput('myNewOutput');
+                });
+
+                it('should clear schema', function() {
+                    assert(!operation._schema);
+                });
+
+                it('should add input to `execute` fn', function() {
+                    var code = operation.getCode();
+                    assert(code.includes('myNewOutput'));
+                });
+
+                it('should have an additional input arg', function() {
+                    var inputs = operation.getOutputs();
+                    assert.equal(inputs.length, 1);
+                });
+            });
+
+            describe('no return', function() {
+                before(function() {
+                    operation = new OperationCode(code);
+                    operation.addReturnValue('no_return', 'myNewOutput');
+                });
+
+                it('should clear schema', function() {
+                    assert(!operation._schema);
+                });
+
+                it('should add input to `execute` fn', function() {
+                    var code = operation.getCode();
+                    assert(code.includes('myNewOutput'));
+                });
+
+                it('should have an additional input arg', function() {
+                    var outputs = operation.getReturnValues('no_return');
+                    assert.equal(outputs.length, 1);
+                });
+            });
+        });
+
+    });
+
+    describe('simple', function() {
+        var code;
+
+        before(function() {
+            var filePath = path.join(__dirname, '..', 'test-cases', 'operations', 'simple.py');
+            code = fs.readFileSync(filePath, 'utf8');
+        });
+
+        describe('parsing', function() {
+            beforeEach(function() {
+                operation = new OperationCode(code);
+            });
+
+            it('should not require base class', function() {
+                assert.equal(operation.getBase(), null);
+            });
+
+            it('should detect one output', function() {
+                assert.equal(operation.getOutputs().length, 1);
+            });
+
+            it('should provide the value', function() {
+                assert.equal(operation.getOutputs()[0].value, '20');
+            });
+
+            it('should detect two inputs', function() {
+                assert.equal(operation.getInputs().length, 2);
+            });
+        });
+
+        describe('addInput', function() {
+            var operation;
+
+            before(function() {
+                operation = new OperationCode(code);
+                operation.addInput('myNewInput');
+            });
+
+            it('should clear schema', function() {
+                assert(!operation._schema);
+            });
+
+            it('should add input to `execute` fn', function() {
+                var code = operation.getCode();
+                assert(code.includes('myNewInput'));
+            });
+
+            it('should have an additional input arg', function() {
+                var inputs = operation.getInputs();
+                assert.equal(inputs.length, 3);
+            });
+        });
+
+        describe('addOutput', function() {
+            var operation;
+
+            before(function() {
+                operation = new OperationCode(code);
+                operation.addOutput('myNewOutput');
+            });
+
+            it('should clear schema', function() {
+                assert(!operation._schema);
+            });
+
+            it('should add input to `execute` fn', function() {
+                var code = operation.getCode();
+                assert(code.includes('myNewOutput'));
+            });
+
+            it('should have an additional input arg', function() {
+                var inputs = operation.getOutputs();
+                assert.equal(inputs.length, 2);
+            });
+        });
+
+        describe('removeInput', function() {
+            var operation,
+                result;
+
+            beforeEach(function() {
+                operation = new OperationCode(code);
+                result = operation.removeInput('number');
+            });
+
+            it('should return removed arg', function() {
+                assert.equal(result.name, 'number');
+            });
+
+            it('should only have one remaining argument', function() {
+                assert.equal(operation.getInputs().length, 1);
+            });
+
+            it('should only not have removed argument', function() {
+                assert(!operation.getCode().includes('number'));
+            });
+
+            it('should return null if arg doesn\'t exist', function() {
+                var result = operation.removeInput('numdasfber');
+                assert.equal(result, null);
+            });
+
+        });
+
+        describe('removeOutput', function() {
+            var operation,
+                result;
+
+            beforeEach(function() {
+                operation = new OperationCode(code);
+                result = operation.removeOutput('result');
+            });
+
+            it('should return removed arg', function() {
+                assert.equal(result.name, 'result');
+            });
+
+            it('should have no remaining results', function() {
+                assert.equal(operation.getOutputs().length, 0);
+            });
+
+            it('should only not have removed argument', function() {
+                assert(!operation.getCode().includes('20'));
+            });
+
+            it('should return null if arg doesn\'t exist', function() {
+                var result = operation.removeOutput('numdasfber');
+                assert.equal(result, null);
+            });
+
+        });
+    });
+
+});

test/plugins/GenerateJob/GenerateJob.spec.js

@@ -118,7 +118,7 @@ describe('GenerateJob', function () {
 
     describe('exec files', function() {
         describe('attribute file', function() {
-            var boolString = /['"](true|false)['"]/g;
+            var boolString = /['"](True|False)['"]/g;
 
             beforeEach(preparePlugin);
 
@@ -129,11 +129,38 @@ describe('GenerateJob', function () {
 
                 plugin.setAttribute(node, 'debug', 'true');
                 plugin.createAttributeFile(node, files);
-                content = files['attributes.lua'];
+                content = files['attributes.py'];
                 matches = content.match(boolString);
                 expect(matches).to.equal(null);
             });
 
+            describe('boolean to python boolean', function() {
+
+                it('should convert true to True', function() {
+                    var files = {},
+                        content,
+                        matches;
+
+                    plugin.setAttribute(node, 'debug', 'true');
+                    plugin.createAttributeFile(node, files);
+                    content = files['attributes.py'];
+                    matches = content.includes('True');
+                    expect(matches).to.equal(true);
+                });
+
+                it('should convert false to False', function() {
+                    var files = {},
+                        content,
+                        matches;
+
+                    plugin.setAttribute(node, 'debug', 'False');
+                    plugin.createAttributeFile(node, files);
+                    content = files['attributes.py'];
+                    matches = content.includes('False');
+                    expect(matches).to.equal(true);
+                });
+            });
+
             it('should not quote true boolean values', function() {
                 var files = {},
                     content,
@@ -141,7 +168,7 @@ describe('GenerateJob', function () {
 
                 plugin.setAttribute(node, 'debug', true);
                 plugin.createAttributeFile(node, files);
-                content = files['attributes.lua'];
+                content = files['attributes.py'];
                 matches = content.match(boolString);
                 expect(matches).to.equal(null);
             });
@@ -153,7 +180,7 @@ describe('GenerateJob', function () {
 
                 plugin.setAttribute(node, 'debug', 'false');
                 plugin.createAttributeFile(node, files);
-                content = files['attributes.lua'];
+                content = files['attributes.py'];
                 matches = content.match(boolString);
                 expect(matches).to.equal(null);
             });
@@ -165,11 +192,14 @@ describe('GenerateJob', function () {
 
                 plugin.setAttribute(node, 'debug', false);
                 plugin.createAttributeFile(node, files);
-                content = files['attributes.lua'];
+                content = files['attributes.py'];
                 matches = content.match(boolString);
                 expect(matches).to.equal(null);
             });
         });
     });
 
+    // TODO: What else should I test?
+    // run a hello world example (use golem?)
+ 
 });

test/test-cases/operations/example.py

@@ -0,0 +1,14 @@
+from operations import Operation
+from typing import Tuple
+
+class ExampleOperation(Operation):
+
+    # TODO: add the type hints
+    def execute(hello, world, count):
+        # Doing things
+        concat = hello + world
+        return concat, count
+
+    def other_method(a, b, c):
+        # Doing things
+        return a+b/c

test/test-cases/operations/multi-anon-results.py

@@ -0,0 +1,10 @@
+from operations import Operation
+from typing import Tuple
+
+class ExampleOperation(Operation):
+
+    # TODO: add the type hints
+    def execute(hello, world, count):
+        # Doing things
+        concat = hello + world
+        return concat+1, count-9

test/test-cases/operations/no-inputs.py

@@ -0,0 +1,6 @@
+class ExampleOperation():
+    def execute():
+        return
+
+    def no_return():
+        print('hello')

test/test-cases/operations/simple.py

@@ -0,0 +1,5 @@
+class TrainOperation():
+
+    # TODO: add type hints
+    def execute(self, number):
+        return 20

utils/categories.json

@@ -1,142 +1,53 @@
 {
-   "RNN": [
-      "BiSequencer",
-      "BiSequencerLM",
-      "GRU",
-      "MaskZero",
-      "MaskZeroCriterion",
-      "Recurrence",
-      "Recurrent",
-      "RecurrentAttention",
-      "Recursor",
-      "Repeater",
-      "RepeaterCriterion",
-      "Sequencer",
-      "SequencerCriterion",
-      "TrimZero",
+    "Transfer": [
+        "PReLU",
+        "Softshrink",
+        "Softplus",
+        "LeakyReLU",
+        "Hardshrink",
+        "ELU",
+        "ReLU6",
+        "Hardtanh",
+        "RReLU",
+        "ReLU",
+        "Threshold"
+    ],
 
-      "CopyGrad",
-      "FastLSTM",
-      "LSTM",
-      "LookupTableMaskZero",
-      "NormStabilizer",
-      "SAdd",
-      "SeqBRNN",
-      "SeqGRU",
-      "SeqLSTM",
-      "SeqLSTMP",
-      "SeqReverseSequence"
-   ],
-   "Convolution": [
-      "TemporalConvolution",
-      "TemporalMaxPooling",
-      "TemporalSubSampling",
-      "LookupTable",
-      "SpatialConvolutionMM",
-      "SpatialConvolution",
-      "SpatialConvolutionMap",
-      "SpatialFullConvolutionMap",
-      "SpatialLPPooling",
-      "SpatialMaxPooling",
-      "SpatialAveragePooling",
-      "SpatialAdaptiveMaxPooling",
-      "SpatialSubSampling",
-      "SpatialUpSamplingNearest",
-      "SpatialZeroPadding",
-      "SpatialReflectionPadding",
-      "SpatialReplicationPadding",
-      "SpatialSubtractiveNormalization",
-      "SpatialCrossMapLRN",
-      "SpatialConvolutionLocal",
-      "SpatialDropout",
-      "SpatialDilatedConvolution",
-      "SpatialFractionalMaxPooling",
-      "SpatialDivisiveNormalization",
-      "SpatialContrastiveNormalization",
-      "SpatialBatchNormalization",
-      "SpatialFullConvolution",
-      "SpatialMaxUnpooling",
-      "VolumetricConvolution",
-      "VolumetricMaxPooling",
-      "VolumetricAveragePooling",
-      "VolumetricBatchNormalization",
-      "VolumetricDropout",
+    "Convolution": [
+        "ConvTranspose3d",
+        "Conv3d",
+        "ConvTranspose2d",
+        "Conv2d",
+        "Conv1d",
 
-      "VolumetricFullConvolution",
-      "VolumetricMaxUnpooling"
-   ],
-   "Criterion": [
-      "BCECriterion",
-      "WeightedMSECriterion",
-      "SmoothL1Criterion",
-      "MSECriterion",
-      "AbsCriterion",
-      "MultiCriterion",
-      "DistKLDivCriterion",
-      "HingeEmbeddingCriterion",
-      "CriterionTable",
-      "MultiMarginCriterion",
-      "MultiLabelMarginCriterion",
-      "L1HingeEmbeddingCriterion",
-      "CosineEmbeddingCriterion",
-      "MarginRankingCriterion",
-      "CrossEntropyCriterion",
-      "MarginCriterion",
-      "ClassNLLCriterion",
-      "ParallelCriterion",
-      "SpatialClassNLLCriterion",
-      "SoftMarginCriterion",
-      "MultiLabelSoftMarginCriterion"
-   ],
-   "Simple": [
-      "Linear",
-      "LinearNoBias",
-      "SparseLinear",
-      "Dropout",
-      "Concat",
-      "Abs",
-      "Add",
-      "Mul",
-      "CMul",
-      "Max",
-      "Min",
-      "Mean",
-      "Sum",
-      "Euclidean",
-      "WeightedEuclidean",
-      "Identity",
-      "Copy",
-      "Narrow",
-      "Replicate",
-      "Reshape",
-      "View",
-      "Select",
-      "Exp",
-      "Square",
-      "Sqrt",
-      "Power",
-      "MM",
-      "AddConstant",
-      "MulConstant"
-   ],
-   "Transfer": [
-      "Threshold",
-      "HardTanh",
-      "HardShrink",
-      "SoftShrink",
-      "SoftMax",
-      "SpatialSoftMax",
-      "SoftMin",
-      "SoftPlus",
-      "SoftSign",
-      "LogSigmoid",
-      "LogSoftMax",
-      "Sigmoid",
-      "Tanh",
-      "ReLU",
-      "ReLU6",
-      "PReLU",
-      "RReLU",
-      "LeakyReLU"
-   ]
+        "FractionalMaxPool2d",
+        "LPPool2d",
+        "MaxUnpool3d",
+        "AvgPool3d",
+        "MaxPool3d",
+        "AvgPool2d",
+        "MaxUnpool2d",
+        "MaxPool2d",
+        "MaxPool1d",
+
+        "ReplicationPad3d",
+        "ReplicationPad2d",
+        "ReflectionPad2d"
+    ],
+
+    "Simple": [
+        "Dropout3d",
+        "Dropout2d",
+        "Dropout",
+        "Linear",
+
+        "Embedding"
+    ],
+
+    "Criterion": [
+        "MultiMarginLoss",
+        "MarginRankingLoss",
+        "CosineEmbeddingLoss",
+        "CrossMapLRN2d"
+    ]
 }

utils/nn-parser.js

@@ -28,7 +28,10 @@ if (exists.sync(configPath)) {  // Check the deepforge config
     config = JSON.parse(fs.readFileSync(configPath, 'utf8'));
     torchPath = (config.torch && config.torch.dir) || (configDir + 'torch');
 }
-torchPath += `/install/share/lua/5.1/${outputName}/`;
+// FIXME: Get the pytorch root path...
+torchPath = process.env.HOME + '/projects/pytorch';
+// check 'modules', 'parallel'
+torchPath += '/torch/nn/';
 
 console.log(`parsing ${outputName} from ${torchPath}`);
 
@@ -51,14 +54,18 @@ var lookupType = function(layer){
     return layerType || 'Misc';
 };
 
-fs.readdir(torchPath, function(err,files){
-    if(err) throw err;
-    var layers,
+var parseLayerFiles = function(layerDir) {
+    var files = fs.readdirSync(layerDir),
+        layers,
         layerByName = {};
 
-    layers = files.filter(filename => path.extname(filename) === '.lua')
-        .map(filename => fs.readFileSync(torchPath + filename, 'utf8'))
+    console.log('parsing', layerDir);
+    layers = files.filter(filename => path.extname(filename) === '.py' &&
+            filename[0] !== '_')
+        .map(filename => fs.readFileSync(layerDir + filename, 'utf8'))
         .map(code => LayerParser.parse(code))
+        .filter(list => list !== null)
+        .reduce((l1, l2) => l1.concat(l2), [])
         .filter(layer => !!layer && layer.name);
 
     layers.forEach(layer => {
@@ -104,17 +111,27 @@ fs.readdir(torchPath, function(err,files){
         }
     });
     layers = layers.filter(layer => !SKIP_LAYERS[layer.name]);
+    return layers;
+};
 
-    outputDst += outputName + '.json';
-    // eslint-disable-next-line no-console
-    console.log('Saved nn interface to ' + outputDst);
-    fs.writeFileSync(outputDst, JSON.stringify(layers, null, 2));
+var layers = ['modules']
+    .map(dir => torchPath + dir + '/')
+    .map(path => parseLayerFiles(path))
+    .reduce((l1, l2) => l1.concat(l2))
+    .filter(layer => layer.name[0] !== '_');  // skip hidden/abstract layers
 
-    // Update the CreateTorchMeta index
-    var updateSchemas = `${__dirname}/../src/plugins/CreateTorchMeta/update-schemas.js`,
-        job = require('child_process').fork(updateSchemas);
 
-    job.on('close', code => {
-        process.exit(code);
-    });
+// eslint-disable-next-line no-console
+console.log('discovered', layers.length, 'layers');
+outputDst += outputName + '.json';
+// eslint-disable-next-line no-console
+console.log('Saved nn interface to ' + outputDst);
+fs.writeFileSync(outputDst, JSON.stringify(layers, null, 2));
+
+// Update the CreateTorchMeta index
+var updateSchemas = `${__dirname}/../src/plugins/CreateTorchMeta/update-schemas.js`,
+    job = require('child_process').fork(updateSchemas);
+
+job.on('close', code => {
+    process.exit(code);
 });

utils/pyparser.js

@@ -0,0 +1,104 @@
+var brython = require('./node-brython'),
+    fs = require('fs'),
+    assert = require('assert'),
+    src = fs.readFileSync(process.env.HOME + '/projects/pytorch/torch/nn/modules/conv.py', 'utf8'),
+    root = build_ast(src);
+
+function build_ast(src) {
+  brython.$py_module_path['__main__']='./'
+  return brython.py2js(src,'__main__', '__main__', '__builtins__')
+}
+
+// The provided tree gives us contexts which can have associated 'C'
+function traverse (node, fn) {
+    var i;
+    if (node.children) {
+        for (i = node.children.length; i--;) {
+            traverse(node.children[i], fn);
+            fn(node.children[i]);
+        }
+    }
+    if (node.C && node.C.tree) {
+        for (i = node.C.tree.length; i--;) {
+            traverse(node.C.tree[i], fn);
+            fn(node.C.tree[i]);
+        }
+    }
+}
+
+var types = {},
+    layers = [],
+    pCtx,
+    classNode,
+    params;
+
+function isClass(node) {
+    return node.type === 'class';
+}
+
+function isInitFn(node) {
+    return node.type === 'def' && node.name === '__init__';
+}
+
+function getBaseClass(node) {
+    assert(node.type === 'class');
+    return node.args.tree[0].tree[0].tree[0].value;
+}
+
+var defaults = {},
+    defTypes,
+    args,
+    def;
+
+traverse(root, node => {
+    if (node.type) types[node.type] = true;
+
+    // Get the class for the given function
+    if (isInitFn(node)) {
+        pCtx = node.parent.node.parent;
+        classNode = pCtx.C.tree[0];
+
+        if (isClass(classNode)) {
+            // remove the 'self' variable
+            // TODO: May need to update this for kwargs
+            // (use positional_list)
+            args = node.tree[1].tree;
+            defaults = {};
+            params = node.args.slice(1);
+            defTypes = {};
+            for (var i = args.length; i--;) {
+                if (args[i].tree[0]) {
+                    def = args[i].tree[0].tree[0];
+                    console.log('setting type of ', params[i-1], 'to', def.type);
+                    defTypes[params[i-1]] = def.type;
+                    if (def.type === 'int') {
+                        defaults[params[i-1]] = parseInt.apply(null, def.value.reverse());
+                    } else {
+                        defaults[params[i-1]] = def.value;
+                    }
+                }
+            }
+            layers.push({
+                name: classNode.name,
+                baseType: getBaseClass(classNode),
+                //doc: classNode.doc_string || '',
+                defaults: defaults,
+                types: defTypes,
+                params: params
+            });
+        }
+    }
+    // TODO: What if there is no constructor? Is this a potential problem?
+});
+
+console.log('layers:', layers);
+fs.writeFileSync('./testPyTorchLayers.json', JSON.stringify(layers, null, 2));
+//console.log('layers:', layers.map(l => l.name));
+// Try to find the class definitions...
+//
+//  Need to create:
+//
+//    setters: (I don't think these are used in pytorch!
+//    types: 
+//    defaults: 
+//    type: 

webgme-setup.json

@@ -327,6 +327,9 @@
       },
       "devProject": {
         "src": "src/seeds/devProject"
+      },
+      "minimal": {
+        "src": "src/seeds/minimal"
       }
     },
     "routers": {