v1.4.1

* WIP #1031 added container dependency info

* WIP #1031 updated quickstart

.gitignore

@@ -35,3 +35,6 @@ test-tmp/
 blob-local-storage/
 src/seeds/nn/hash.txt
 src/seeds/pipeline/hash.txt
+
+notes/
+src/worker

.npmignore

@@ -38,3 +38,6 @@ src/seeds/pipeline/hash.txt
 
 # docs
 images/
+
+notes/
+src/worker

README.md

@@ -21,23 +21,13 @@ Additional features include:
 - Facilitates defining custom layers 
 
 ## Quick Start
-Simply run the following command to install deepforge with its dependencies:
-
+The easiest way to start deepforge is using [docker-compose](https://docs.docker.com/compose/). Using docker-compose, deepforge can be started with
 ```
-curl -o- https://raw.githubusercontent.com/dfst/deepforge/master/install.sh | bash
+wget https://raw.githubusercontent.com/deepforge-dev/deepforge/master/docker-compose.yml
+docker-compose up
 ```
 
-Or, if you already have NodeJS (v6) installed, simply run
-
-```
-npm install -g deepforge
-```
-
-Finally, start deepforge with `deepforge start`and navigate to [http://localhost:8888](http://localhost:8888) to start using DeepForge! For more, detailed instructions, check out the [wiki](https://github.com/dfst/deepforge/wiki/Installation-Guide).
-
-**Note**: running deepforge w/ `deepforge start` will also require [MongoDB](https://www.mongodb.com/download-center?jmp=nav#community) to be installed locally.
-
-Also, be sure to check out the other available features of the `deepforge` cli; it can be used to update, manage your torch installation, uninstall deepforge and run individual components!
+Finally, navigate to [http://localhost:8888](http://localhost:8888) to start using DeepForge! For more detailed instructions and other installation options, check out the [docs](http://deepforge.readthedocs.io/en/latest/deployment/overview.html).
 
 ## Additional Resources
 - [Intro to DeepForge Slides](https://docs.google.com/presentation/d/10_y5O3gHXSATfjHVLJg7dOdrz-tAXNWjlxhJ5SlA0ic/edit?usp=sharing)

bin/deepforge

@@ -185,7 +185,6 @@ var startMongo = function(args, port, silent) {
 };
 
 var hasTorch = function() {
-    return true;
     var result = childProcess.spawnSync('th', ['--help']);
     return !result.error;
 };

bin/start-worker.js

@@ -39,13 +39,13 @@ try {
 }
 
 // Check torch support
-//var result = childProcess.spawnSync('th', ['--help']);
-//if (result.error) {
-    //console.error('Checking Torch7 dependency failed. Do you have Torch7 installed ' + 
-        //'and in your PATH?\n\nFor Torch7 installation instructions, check out ' +
-        //'http://torch.ch/docs/getting-started.html');
-    //process.exit(1);
-//}
+var result = childProcess.spawnSync('th', ['--help']);
+if (result.error) {
+    console.error('Checking Torch7 dependency failed. Do you have Torch7 installed ' + 
+        'and in your PATH?\n\nFor Torch7 installation instructions, check out ' +
+        'http://torch.ch/docs/getting-started.html');
+    process.exit(1);
+}
 
 var cleanUp = function() {
     console.log('removing worker directory ', workerPath);

config/config.test.js

@@ -9,8 +9,7 @@ var config = require('./config.default'),
     path = require('path');
 
 config.server.port = 9001;
-config.mongo.uri = process.env.MONGO_URI || 'mongodb://127.0.0.1:27017';
-config.mongo.uri = config.mongo.uri.replace(/\/[a-zA-Z_\-]*$/,  '') + '/deepforge_tests';
+config.mongo.uri = 'mongodb://127.0.0.1:27017/webgme_tests';
 config.blob.fsDir = path.join(__dirname, '..', 'test-tmp', 'blob');
 
 module.exports = config;

config/config.webgme.js

@@ -22,7 +22,6 @@ config.seedProjects.basePaths.push(__dirname + '/../src/seeds/project');
 config.seedProjects.basePaths.push(__dirname + '/../src/seeds/cifar10');
 config.seedProjects.basePaths.push(__dirname + '/../src/seeds/xor');
 config.seedProjects.basePaths.push(__dirname + '/../src/seeds/devProject');
-config.seedProjects.basePaths.push(__dirname + '/../src/seeds/minimal');
 
 
 

docker-compose.yml

@@ -0,0 +1,22 @@
+--- 
+services: 
+  mongo: 
+    image: mongo
+    volumes: 
+      - "$HOME/.deepforge/data:/data/db"
+  server: 
+    environment: 
+      - "MONGO_URI=mongodb://mongo:27017/deepforge"
+    image: deepforge/server
+    ports: 
+      - "8888:8888"
+    volumes: 
+      - "$HOME/.deepforge/blob:/data/blob"
+    depends_on:
+      - mongo
+  worker: 
+    command: "http://server:8888"
+    image: deepforge/worker
+    depends_on:
+      - server
+version: "2"

docs/deployment/native.rst

@@ -1,6 +1,30 @@
 Native Installation
 ===================
 
+Dependencies
+------------
+First, install `NodeJS <https://nodejs.org/en/>`_ (v6) and `MongoDB <https://www.mongodb.org/>`_. You may also need to install git if you haven't already.
+
+Next, you can install DeepForge using npm:
+
+.. code-block:: bash
+
+    npm install -g deepforge
+
+Now, you can check that it installed correctly:
+
+.. code-block:: bash
+
+    deepforge --version
+
+DeepForge can now be started with:
+
+.. code-block:: bash
+
+    deepforge start
+
+However, the first time DeepForge is started, it will make sure that the deep learning framework is installed (if it isn't found on the host system). This may require you to start DeepForge a couple times; the first time it starts it will install Torch7 and require a terminal restart to update a couple environment variables (like `PATH`). The second time it starts it will install additional torch packages but will not require a terminal restart. Finally, DeepForge will start with all the required dependencies.
+
 Database
 ~~~~~~~~
 Download and install MongoDB from the `website <https://www.mongodb.org/>`_. If you are planning on running MongoDB locally on the same machine as DeepForge, simply start `mongod` and continue to setting up DeepForge.

docs/getting_started/hello_cifar.rst

@@ -28,7 +28,7 @@ In this example, we have three main pipelines: :code:`download-normalize`, :code
 
 First, we will have to retrieve and prepare the data by running the :code:`download-normalize` pipeline. This can be done by opening the given pipeline then selecting the `Execute Pipeline` option from the action button in the lower right. As soon as that pipeline finishes, we can now use this data to train a neural network.
 
-Next, we can open the :code:`train` pipeline. Before we execute the pipeline we have to set the input trainning data that we will be using. This is done by selecting the :code:`Input` operation then clicking the value for the :code:`artifact` field. This will provide all the possible options for the input data; for this example, we will want to select the "trainingdata" artifact. After setting the input, we can click on the :code:`train` operation to inspect the hyperparameters we are using and the architecture we are training. Selecting the :code:`Output` operation will allow you to change the name of the resulting artifact of this operation (in this case, a trained model). Finally, we can execute this pipeline like before to train the model.
+Next, we can open the :code:`train` pipeline. Before we execute the pipeline we have to set the input training data that we will be using. This is done by selecting the :code:`Input` operation then clicking the value for the :code:`artifact` field. This will provide all the possible options for the input data; for this example, we will want to select the "trainingdata" artifact. After setting the input, we can click on the :code:`train` operation to inspect the hyperparameters we are using and the architecture we are training. Selecting the :code:`Output` operation will allow you to change the name of the resulting artifact of this operation (in this case, a trained model). Finally, we can execute this pipeline like before to train the model.
 
 .. figure:: select_train_data.png
     :align: center

docs/getting_started/quick_start.rst

@@ -1,29 +1,19 @@
 Quick Start
 ===========
-Before we can start with the examples, we will first install DeepForge locally.
+The easiest way to get started quickly with DeepForge is using docker-compose. First, install `docker <https://docs.docker.com/engine/installation/>`_ and `docker-compose <https://docs.docker.com/compose/install/>`_.
 
-Dependencies
-------------
-First, install `NodeJS <https://nodejs.org/en/>`_ (v6) and `MongoDB <https://www.mongodb.org/>`_. You may also need to install git if you haven't already.
-
-Next, you can install DeepForge using npm:
+Next, download the docker-compose file for DeepForge:
 
 .. code-block:: bash
 
-    npm install -g deepforge
+    wget https://raw.githubusercontent.com/deepforge-dev/deepforge/master/docker-compose.yml
 
-Now, you can check that it installed correctly:
+Then start DeepForge using docker-compose:
 
 .. code-block:: bash
 
-    deepforge --version
+    docker-compose up
 
-DeepForge can now be started with:
-
-.. code-block:: bash
-
-    deepforge start
-
-However, the first time DeepForge is started, it will make sure that the deep learning framework is installed (if it isn't found on the host system). This may require you to start DeepForge a couple times; the first time it starts it will install Torch7 and require a terminal restart to update a couple environment variables (like `PATH`). The second time it starts it will install additional torch packages but will not require a terminal restart. Finally, DeepForge will start with all the required dependencies.
+and now DeepForge can be used by opening a browser to `http://localhost:8888 <http://localhost:8888>`_!
 
 For detailed instructions about deployment installations, check out our `deployment installation instructions <getting_started/configuration.rst>`_

package.json

@@ -17,7 +17,7 @@
     "watch-test": "nodemon --exec 'mocha --recursive test'",
     "build-nn": "node ./utils/nn-parser.js"
   },
-  "version": "1.3.0",
+  "version": "1.4.1",
   "dependencies": {
     "commander": "^2.9.0",
     "dotenv": "^2.0.0",
@@ -43,7 +43,6 @@
     "webgme-simple-nodes": "^2.1.0"
   },
   "devDependencies": {
-    "brython": "^3.2.7",
     "chai": "^3.0.0",
     "jszip": "^2.5.0",
     "mocha": "^2.2.5",

src/common/LayerParser.js

@@ -1,127 +1,378 @@
 /* globals define*/
 (function(root, factory){
     if(typeof define === 'function' && define.amd) {
-        // TODO: Load the brython script
-        define(['./lua'], function(brython){
-            return (root.LayerParser = factory(brython, console.assert));
+        define(['./lua'], function(luajs){
+            return (root.LayerParser = factory(luajs));
         });
     } else if(typeof module === 'object' && module.exports) {
-        var brython = require('./node-brython'),
-            assert = require('assert');
-
-        module.exports = (root.LayerParser = factory(brython, assert));
+        var luajs = require('./lua');
+        module.exports = (root.LayerParser = factory(luajs));
     }
-}(this, function(brython, assert) {
+}(this, function(luajs) {
     var LayerParser = {};
 
-    function build_ast(src) {
-        brython.$py_module_path['__main__']='./'
-        return brython.py2js(src,'__main__', '__main__', '__builtins__')
-    }
+    //////////////////////// Setters //////////////////////// 
+    var returnsSelf = function(fnNode){
+        var stats = fnNode.block.stats,
+            last = stats[stats.length-1];
 
-    // The provided tree gives us contexts which can have associated 'C'
-    function traverse(node, fn) {
-        var i;
-        if (node.children) {
-            for (i = node.children.length; i--;) {
-                traverse(node.children[i], fn);
-                fn(node.children[i]);
+        if (last.type === 'stat.return') {
+            return last.nret[0].type === 'variable' && last.nret[0].val === 'self';
+        }
+        return false;
+    };
+
+    var isAttrSetter = function(node){
+        if (node.type === 'stat.assignment' && node.lefts.length === 1) {
+            var left = node.lefts[0];
+            return left.type === 'expr.index' && left.self.val === 'self';
+        }
+        return false;
+    };
+
+    var getSettingAttrName = function(node){
+        if (isAttrSetter(node)) {
+            var left = node.lefts[0];
+            return left.key.val;
+        }
+        return null;
+    };
+
+    var getSettingAttrValue = function(node){
+        if (isAttrSetter(node)) {
+            return node.right;
+        }
+        return null;
+    };
+
+    var isSetterMethod = function(curr, parent, className){
+        if (parent && parent.type === 'stat.method') {
+            // is it a fn w/ two statements (stats)
+            if (parent.self.val === className && curr.type === 'function' &&
+                curr.block.stats.length === 2) {
+                // Is the first statement setting a value?
+                return returnsSelf(curr) && getSettingAttrName(curr.block.stats[0]);  // does it return itself?
             }
         }
-        if (node.C && node.C.tree) {
-            for (i = node.C.tree.length; i--;) {
-                traverse(node.C.tree[i], fn);
-                fn(node.C.tree[i]);
+        return false;
+    };
+
+    var isFnArg = function(method, name) {
+        return method.args.indexOf(name) !== -1;
+    };
+
+    var getSetterSchema = function(node, method) {
+        var setterType,
+            setterFn,
+            value = getSettingAttrValue(node);
+
+        if (value[0].type === 'variable' && isFnArg(method.func, value[0].val)) {
+            setterType = 'arg';
+            setterFn = method.key.val;
+        } else {
+            setterType = 'const';
+            setterFn = {};
+            setterFn[value[0].val] = method.key.val;
+        }
+
+        return {
+            setterType,
+            setterFn
+        };
+    };
+
+    //////////////////////// Setters END //////////////////////// 
+
+    var isInitFn = function(node, className) {
+        if (node.type === 'stat.method' && node.self.val === className) {
+            return node.key.val === '__init';
+        }
+        return false;
+    };
+
+    var getClassAttrDefs = function(method) {
+        var fn = method.func,
+            dict = {},
+            attr,
+            right,
+            value;
+
+        luajs.codegen.traverse(curr => {
+            if (isAttrSetter(curr)) {
+                // Store the value if it is set to a constant
+                attr = curr.lefts[0].key.val;
+                right = curr.right[0];
+                if (right.type.indexOf('const.') !== -1) {
+                    value = right.val;
+
+                    if (right.type === 'const.nil') {
+                        value = null;
+                    }
+
+                    dict[attr] = value;
+                }
+            }
+        })(fn);
+
+        return dict;
+    };
+
+    var getAttrsAndVals = function(method) {
+        // Given a method, get the 'self' attributes and the default values
+        var fn = method.func,
+            dict = {},
+            varName,
+            value,
+            varUsageCnt = {};
+
+        // Get the variables that are used only once (or updating themselves)
+        luajs.codegen.traverse(curr => {
+            if (curr.type === 'variable') {
+                varUsageCnt[curr.val] = varUsageCnt[curr.val] ?
+                    varUsageCnt[curr.val] + 1 : 1;
+            }
+        })(method);
+
+        luajs.codegen.traverse(curr => {
+            // If the variable is only used once and is 'or'-ed w/ a constant
+            // during this use, we can infer that this is the default value
+            if (curr.type === 'expr.op' && curr.op === 'op.or' &&
+                curr.left.type === 'variable' && curr.right.type.indexOf('const') !== -1) {
+                varName = curr.left.val;
+                if (varUsageCnt[varName] === 1) {
+                    value = curr.right.type === 'const.nil' ? null : curr.right;
+                    dict[varName] = value;
+                }
+            }
+        })(fn);
+
+        return dict;
+    };
+
+    var copyNodeValues = function(attrs, from, to) {
+        var value;
+        for (var i = attrs.length; i--;) {
+            value = from[attrs[i]] || null;
+            if (value) {
+                value = (value && value.hasOwnProperty('val')) ? value.val : value;
+                to[attrs[i]] = value;
             }
         }
-    }
+        return to;
+    };
 
-    var types = {},
-        layers = [],
-        pCtx,
-        classNode,
-        params;
+    var getTypeCheckInfo = function(cond) {
+        var caller,
+            method,
+            target,
+            expType;
 
-    function isClass(node) {
-        return node.type === 'class';
-    }
+        // Check for torch.isTypeOf:
+        if (cond.type === 'expr.call' && cond.func.type === 'expr.index') {
+            caller = cond.func.self.val;
+            method = cond.func.key.val;
 
-    function isInitFn(node) {
-        return node.type === 'def' && node.name === '__init__';
-    }
+            if (cond.type === 'expr.call' && caller === 'torch') {
+                target = cond.args[0].val;
+                if (method === 'isTypeOf' && target) {
+                    expType = cond.args[1].val;
+                    return {
+                        target,
+                        type: expType
+                    };
+                }
+            }
+        } else if (cond.type === 'expr.op') {  // torch.type() === ''
+            // Check right side, too!
+            var sides = [cond.left, cond.right],
+                side,
+                otherSide;
 
-    function getBaseClass(node) {
-        assert(node.type === 'class');
-        return node.args.tree[0].tree[0].tree[0].value;
-    }
+            for (var i = sides.length; i--;) {
+                side = sides[i];
+                otherSide = sides[(i+1)%2];
+                if (side.type === 'expr.call' && side.func.type === 'expr.index') {
+                    // Is it torch?
+                    caller = side.func.self.val;
+                    method = side.func.key.val;
+                    if (caller === 'torch' && method === 'type') {
+                        if (side.args[0].type === 'variable') {
+                            target = side.args[0].val;
+                            if (otherSide.type === 'const.string') {
+                                expType = otherSide.val;
 
-    function findTorchLayers(root) {
-        var defaults = {},
-            layers = [],
-            defTypes,
-            args,
-            def;
-
-        traverse(root, node => {
-            // Get the class for the given function
-            if (isInitFn(node)) {
-            // TODO: What if there is no constructor? Is this a potential problem?
-                pCtx = node.parent.node.parent;
-                classNode = pCtx.C.tree[0];
-
-                if (isClass(classNode)) {
-                    // remove the 'self' variable
-                    // TODO: May need to update this for kwargs
-                    // (use positional_list)
-                    args = node.tree[1].tree;
-                    defaults = {};
-                    params = node.args.slice(1);
-                    defTypes = {};
-                    for (var i = args.length; i--;) {
-                        if (args[i].tree[0]) {
-                            def = args[i].tree[0].tree[0];
-                            if (def.type === 'int') {
-                                defaults[params[i-1]] = parseInt.apply(null, def.value.reverse());
-                            } else {
-                                defaults[params[i-1]] = def.value;
-                            }
-                            if (/^(True|False)$/.test(defaults[params[i-1]])) {
-                                defTypes[params[i-1]] = 'boolean';
-                            } else {
-                                defTypes[params[i-1]] = def.type;
+                                return {
+                                    target: target,
+                                    type: expType
+                                };
                             }
                         }
+                        
                     }
-                    layers.push({
-                        name: classNode.name,
-                        baseType: getBaseClass(classNode),
-                        //doc: classNode.doc_string || '',
-                        defaults: defaults,
-                        types: defTypes,
-                        setters: {},
-                        params: params
-                    });
                 }
             }
+            return null;
+        }
+    };
+
+    var isError = function(stat) {
+        var fn;
+        if (stat.type === 'stat.expr' && stat.expr.type === 'expr.call') {
+            fn = stat.expr.func.val;
+            return fn === 'error';
+        }
+        return false;
+    };
+
+    var inferParamTypes = function(node, paramDefs) {
+        var types = {},
+            check,
+            cond;
+
+        // Infer from assertions
+        luajs.codegen.traverse(curr => {
+            // check for 'assert's that check type
+            if (curr.type === 'expr.call' && curr.func.val === 'assert') {
+                cond = curr.args[0];
+                check = getTypeCheckInfo(cond);
+                if (check) {
+                    types[check.target] = check.type;
+                }
+            } else if (curr.type === 'stat.if' && curr.cond.op === 'uop.not') {
+                // if statements throwing errors on type mismatch
+                cond = curr.cond.operand;  // non-negated version
+                // Check that it throws an error on true
+                if (curr.tblock.stats.some(isError)) {
+                    check = getTypeCheckInfo(cond);
+                    if (check) {
+                        types[check.target] = check.type;
+                    }
+                }
+            }
+        })(node);
+
+        // Infer from defaults
+        Object.keys(paramDefs).forEach(param => {
+            var val = paramDefs[param];
+            if (val) {  // initialized to 'null' doesn't help us...
+                types[param] = val.type.replace('const.', '');
+            }
         });
 
-        return layers;
-    }
+        return types;
+    };
 
-    // Try to find the class definitions...
-    //
-    //  Need to create:
-    //
-    //    setters: (I don't think these are used in pytorch!
-    //    types: 
-    //    type: 
-    //////////////////////// Setters //////////////////////// 
-    LayerParser.parse = function(src) {
+    var findTorchClass = function(ast){ 
+        var torchClassArgs,  // args for `torch.class(...)`
+            name = '',
+            alias,
+            baseType,
+            params,
+            setters = {},
+            defaults = {},
+            paramDefs,
+            attrDefs;
+
+        if(ast.type == 'function'){
+            ast.block.stats.forEach(function(func){
+                if(func.type == 'stat.local' && func.right && func.right[0] &&
+                  func.right[0].func && func.right[0].func.self &&
+                  func.right[0].func.self.val == 'torch' &&
+                  func.right[0].func.key.val == 'class'){
+
+                    torchClassArgs = func.right[0].args.map(arg => arg.val);
+                    name = torchClassArgs[0];
+                    if(name !== ''){
+                        name = name.replace('nn.', '');
+                        alias = func.names[0] || name;
+                        if (torchClassArgs.length > 1) {
+                            baseType = torchClassArgs[1].replace('nn.', '');
+                        }
+                    }
+                }
+            });
+        }
+
+        // Get the setters, defaults and type info (inferred)
+        var setterNames,
+            schema,
+            types,
+            values;
+
+        luajs.codegen.traverse((curr, parent) => {
+            var firstLine,
+                attrName;
+
+            // Record the setter functions
+            if (isSetterMethod(curr, parent, alias)) {
+                firstLine = curr.block.stats[0];
+                // just use the attribute attrName for now...
+                attrName = getSettingAttrName(firstLine);
+
+                // merge schemas
+                schema = getSetterSchema(firstLine, parent);
+                if (setters[attrName] && setters[attrName].setterType === 'const') {  // merge
+                    for (var val in schema.setterFn) {
+                        setters[attrName].setterFn[val] = schema.setterFn[val];
+                    }
+                } else {
+                    setters[attrName] = schema;
+                }
+            } else if (isInitFn(curr, alias)) {  // Record the defaults
+                paramDefs = getAttrsAndVals(curr);
+                attrDefs = getClassAttrDefs(curr);
+                types = inferParamTypes(curr, paramDefs);
+
+                // get ctor args
+                params = curr.func.args;
+                if(params.length === 0 && curr.func.varargs){
+                    params.push('params');
+                }
+            }
+
+        })(ast);
+
+        // Get the defaults for the params from defs
+        if (paramDefs && params) {
+            copyNodeValues(params, paramDefs, defaults);
+        }
+
+        // Get the defaults for the setters from attrDefs
+        if (attrDefs) {
+            setterNames = Object.keys(setters);
+            copyNodeValues(setterNames, attrDefs, defaults);
+        }
+
+        // Remove any const setters w/ only one value and no default
+        setterNames = Object.keys(setters);
+        for (var i = setterNames.length; i--;) {
+            schema = setters[setterNames[i]];
+            if (schema.setterType === 'const') {
+                values = Object.keys(schema.setterFn);
+                if (values.length === 1 &&
+                    // boolean setters can have the default value inferred
+                    values[0] !== 'true' && values[0] !== 'false' &&
+                    !defaults[setterNames[i]]) {
+
+                    delete setters[setterNames[i]];
+                }
+            }
+        }
+
+        return {
+            name,
+            baseType,
+            params,
+            setters,
+            types,
+            defaults
+        };
+    };
+
+    LayerParser.parse = function(text) {
         try {
-            brython.$py_module_path['__main__']='./';
-            var ast = brython.py2js(src,'__main__', '__main__', '__builtins__');
-            var layers = findTorchLayers(ast);
-            return layers;
+            var ast = luajs.parser.parse(text);
+            return findTorchClass(ast);
         } catch (e) {
             return null;
         }

src/common/OperationCode.js

@@ -1,331 +0,0 @@
-/*globals Sk, define*/
-var isNodeJs = typeof module === 'object' && module.exports;
-(function(root, factory){
-    if(typeof define === 'function' && define.amd) {
-        define(['./skulpt.min'], function(){
-            return (root.OperationParser = factory(Sk));
-        });
-    } else if(isNodeJs) {
-        require('./skulpt.min');
-
-        module.exports = (root.OperationParser = factory(Sk));
-    }
-}(this, function(Sk) {
-    var MAIN_FN = 'execute';
-    var CTOR_FN = '__init__';
-    var OperationCode = function(code, filename) {
-        this._lines = code.split('\n');
-        this.filename = filename;
-    };
-
-    OperationCode.prototype.getName = function() {
-        if (!this._schema) this.updateSchema();
-
-        return this._schema.name;
-    };
-
-    OperationCode.prototype.getBase = function() {
-        if (!this._schema) this.updateSchema();
-
-        return this._schema.base;
-    };
-
-    OperationCode.prototype.getArguments = function(method) {
-        if (!this._schema) this.updateSchema();
-        if (!this._schema.methods[method]) return null;
-
-        return this._schema.methods[method].inputs.slice();
-    };
-
-    OperationCode.prototype.getReturnValues = function(method) {
-        if (!this._schema) this.updateSchema();
-        if (!this._schema.methods[method]) return null;
-
-        return this._schema.methods[method].outputs.slice();
-    };
-
-    OperationCode.prototype.getOutputs = function() {
-        return this.getReturnValues(MAIN_FN);
-    };
-
-    OperationCode.prototype.getInputs = function() {
-        return this.getArguments(MAIN_FN);
-    };
-
-    OperationCode.prototype.removeInput = function(name) {
-        return this._removeIOCode(this.getInputs(), name);
-    };
-
-    OperationCode.prototype.removeOutput = function(name) {
-        return this._removeIOCode(this.getOutputs(), name);
-    };
-
-    OperationCode.prototype._removeIOCode = function(ios, name) {
-        var match,
-            prev,
-            line,
-            startIndex,
-            endIndex;
-
-        for (var i = 0; i < ios.length; i++) {
-            match = ios[i];
-            prev = ios[i-1];
-
-            if (match.name === name) {
-                line = this._lines[match.pos.line-1];
-
-                startIndex = prev ? prev.pos.col + prev.value.toString().length : match.pos.col;
-                // only remove the following ',' if first input/output
-                endIndex = i === 0 && i < ios.length-1 ? ios[i+1].pos.col :
-                    match.pos.col + match.value.toString().length;
-                this._lines[match.pos.line-1] = line.substring(0, startIndex) +
-                    line.substring(endIndex);
-
-                this.clearSchema();
-                return match;
-            }
-        }
-        return null;
-    };
-
-    OperationCode.prototype.addInput = function(name) {
-        return this.addArgument(MAIN_FN, name);
-    };
-
-    OperationCode.prototype.addOutput = function(name) {
-        return this.addReturnValue(MAIN_FN, name);
-    };
-
-    OperationCode.prototype.addArgument = function(method, name) {
-        return this._addIOCode(method, name, true);
-    };
-
-    OperationCode.prototype.addReturnValue = function(method, name) {
-        return this._addIOCode(method, name, false);
-    };
-
-    OperationCode.prototype.addMethod = function(method) {
-        // TODO: get the position at the top of the class def
-        var line = this._schema.body.pos.line - 1,
-            indentSize = this._schema.body.pos.col,
-            indent = new Array(indentSize+1).join(' '),
-            snippet = indent + `def ${method}():`,
-            body = new Array(indentSize+5).join(' ') + 'return';
-
-        this._lines.splice(line-1, 0, '');
-        this._lines.splice(line-1, 0, snippet);
-        this._lines.splice(line, 0, body);
-
-        this.clearSchema();
-    };
-
-    OperationCode.prototype.hasMethod = function(method) {
-        if (!this._schema) this.updateSchema();
-        return this._schema.methods[method];
-    };
-
-    OperationCode.prototype._addIOCode = function(method, name, isInput) {
-        if (!this.hasMethod(method)) this.addMethod(method);
-
-        this.updateSchema();
-
-        var ios = this._schema.methods[method][isInput ? 'inputs' : 'outputs'].slice(),
-            node = this._schema.methods[method].node,
-            body = node.body,
-            content = name,
-            line,
-            startIndex,
-            endIndex,
-            lineIndex;
-
-        if (ios.length) {
-            var pos = ios[ios.length-1].pos;
-            var argLen = ios[ios.length-1].name.length;
-
-            line = this._lines[pos.line-1];
-            startIndex = pos.col + argLen;
-            endIndex = pos.col + argLen;
-            content = ', ' + name;
-            lineIndex = pos.line - 1;
-        } else if (isInput) {
-            var first = body[0];
-            lineIndex = first.lineno - 2;
-            line = this._lines[lineIndex];
-            this._lines[lineIndex] = line.replace(/\).*?:/, name + '):');
-
-            return this.clearSchema();
-        } else {
-            var ret = body.find(node => this._isNodeType(node, 'Return_'));
-            if (ret) {
-                lineIndex = ret.lineno-1;
-                startIndex = endIndex = ret.col_offset + 6;
-                content = ' ' + content;
-            } else {  // add to the end of the body (no return statement)
-                var lastNode = body[body.length-1];
-                var indent = new Array(lastNode.col_offset+1).join(' ');
-
-                lineIndex = lastNode.lineno;
-                this._lines.splice(lineIndex, 0, '');
-                startIndex = endIndex = 0;
-                content = indent + 'return ' + content;
-            }
-        }
-
-        line = this._lines[lineIndex];
-        this._lines[lineIndex] = line.substring(0, startIndex) + content +
-            line.substring(endIndex);
-
-        this.clearSchema();
-    };
-
-    OperationCode.prototype.rename = function(oldName, name) {
-        if (!this.hasMethod(MAIN_FN)) return;
-
-        var fnSchema = this._schema.methods[MAIN_FN];
-        var startLine = fnSchema.bounds.start.line - 1;
-        var endLine = fnSchema.bounds.end ? fnSchema.bounds.end.line - 1 : this._lines.length;
-        var pattern = new RegExp('\\b' + oldName + '\\b');
-
-        for (var i = startLine; i < endLine; i++) {
-            this._lines[i] = this._lines[i].replace(pattern, name);
-        }
-        this.clearSchema();
-    };
-
-    OperationCode.prototype.getCode = function() {
-        return this._lines.join('\n');
-    };
-
-    OperationCode.prototype.getAst = function () {
-        if (this._ast) return this._ast;
-
-        var filename = this.filename || 'operation.py';
-        var cst = Sk.parse(filename, this.getCode()).cst;
-        var ast = Sk.astFromParse(cst, filename);
-        return this._ast = ast;
-    };
-
-    OperationCode.prototype._isNodeType = function (node, name) {
-        return node.constructor.name === name;
-    };
-
-    OperationCode.prototype._parseFn = function (node, schema, next) {
-        var name = node.name.v;
-
-        schema.methods[name] = {};
-        // add inputs
-        schema.methods[name].inputs = node.args.args.map(arg => {
-            return {
-                name: arg.id.v,
-                value: arg.id.v,
-                pos: {
-                    line: arg.lineno,
-                    col: arg.col_offset
-                }
-            };
-        });
-
-        // add outputs
-        var ret = node.body.find(node => this._isNodeType(node, 'Return_'));
-        var retVals = [];
-        if (ret) {
-            retVals = ret.value && this._isNodeType(ret.value, 'Tuple') ?
-                ret.value.elts : [ret.value];
-        }
-
-        schema.methods[name].outputs = retVals
-            .filter(node => !!node)
-            .map((arg, index) => {
-                var isNameNode = this._isNodeType(arg, 'Name');
-                var name = isNameNode ? arg.id.v : 'result';
-                if (!isNameNode && index > 0) {
-                    name += '_' + index;
-                }
-
-                var value = this._isNodeType(arg, 'Num') ? arg.n.v : name;
-
-                return {
-                    name: name,
-                    value: value,
-                    pos: {
-                        line: arg.lineno,
-                        col: arg.col_offset
-                    }
-                };
-            });
-
-        // Get the function location
-        schema.methods[name].bounds = {};
-        schema.methods[name].bounds.start = {
-            line: node.lineno,
-            col: node.col_offset
-        };
-
-        if (next) {
-            schema.methods[name].bounds.end = {
-                line: next.lineno,
-                col: next.col_offset
-            };
-        }
-
-        schema.methods[name].node = node;
-    };
-
-    OperationCode.prototype.updateSchema = function () {
-        if (!this._schema) this._schema = this.getSchema();
-    };
-
-    OperationCode.prototype.clearSchema = function () {
-        this._ast = null;
-        this._schema = null;
-    };
-
-    OperationCode.prototype.getSchema = function () {
-        var schema = {
-            name: null,
-            base: null,
-            methods: {}
-        };
-        var ast = this.getAst();
-
-        // Find the class definition
-        var classDef = ast.body.find(node => this._isNodeType(node, 'ClassDef'));
-        if (classDef) {
-            schema.name = classDef.name.v;
-
-            // TODO: what if fn is inherited?
-            var nodes = classDef.body;
-            for (var i = 0; i < nodes.length; i++) {
-                if (this._isNodeType(nodes[i], 'FunctionDef')) {
-                    this._parseFn(nodes[i], schema, nodes[i+1]);
-                }
-            }
-            schema.body = {
-                pos: {
-                    line: nodes[0].lineno,
-                    col: nodes[0].col_offset,
-                }
-            };
-
-        }
-
-        schema.ast = ast;
-
-        return schema;
-    };
-
-    /////////////////////// Attributes /////////////////////// 
-    OperationCode.prototype.addAttribute = function(name, value) {
-        return this._addIOCode(CTOR_FN, name, true);
-    };
-
-    OperationCode.prototype.removeAttribute = function(name) {
-        // TODO
-    };
-
-    OperationCode.prototype.getAttributes = function() {
-        return this.getArguments(CTOR_FN);
-    };
-
-    return OperationCode;
-}));

src/common/OperationParser.js

@@ -1,126 +0,0 @@
-/* globals define, Sk*/
-var isNodeJs = typeof module === 'object' && module.exports;
-(function(root, factory){
-    if(typeof define === 'function' && define.amd) {
-        define(['./skulpt.min'], function(){
-            return (root.OperationParser = factory(Sk));
-        });
-    } else if(isNodeJs) {
-        require('./skulpt.min');
-
-        module.exports = (root.OperationParser = factory(Sk));
-    }
-}(this, function(Sk) {
-    Sk.python3 = true;
-    var OperationParser = {};
-
-    // The provided tree gives us contexts which can have associated 'C'
-    function traverse(node, fn) {
-        var i;
-        if (node.children) {
-            for (i = node.children.length; i--;) {
-                traverse(node.children[i], fn);
-                fn(node.children[i]);
-            }
-        }
-        if (node.C && node.C.tree) {
-            for (i = node.C.tree.length; i--;) {
-                traverse(node.C.tree[i], fn);
-                fn(node.C.tree[i]);
-            }
-        }
-    }
-
-    function isNodeType(node, name) {
-        return node.constructor.name === name;
-    }
-
-    function parseFn(node, schema) {
-        var name = node.name.v;
-
-        schema.methods[name] = {};
-        // add inputs
-        schema.methods[name].inputs = node.args.args.map(arg => {
-            return {
-                name: arg.id.v,
-                value: arg.id.v,
-                pos: {
-                    line: arg.lineno,
-                    col: arg.col_offset
-                }
-            };
-        });
-
-        // add outputs
-        var ret = node.body.find(node => isNodeType(node, 'Return_'));
-        var retVals = [];
-        if (ret) {
-            retVals = ret.value && isNodeType(ret.value, 'Tuple') ?
-                ret.value.elts : [ret.value];
-        }
-
-        schema.methods[name].outputs = retVals.map((arg, index) => {
-            var isNameNode = isNodeType(arg, 'Name');
-            var name = isNameNode ? arg.id.v : 'result';
-            if (!isNameNode && index > 0) {
-                name + '_' + index;
-            }
-
-            var value = isNodeType(arg, 'Num') ? arg.n.v : name;
-
-            return {
-                name: name,
-                value: value,
-                pos: {
-                    line: arg.lineno,
-                    col: arg.col_offset
-                }
-            };
-        });
-    }
-
-    function parseOperationAst(ast) {
-        var schema = {
-            name: null,
-            base: null,
-            methods: {}
-        };
-
-        // Find the class definition
-        var classDef = ast.body.find(node => isNodeType(node, 'ClassDef'));
-        if (classDef) {
-            schema.name = classDef.name.v;
-
-            // TODO: what if fn is inherited?
-            classDef.body
-                .filter(node => isNodeType(node, 'FunctionDef'))
-                .forEach(node => parseFn(node, schema));
-
-        }
-
-        schema.inputs = schema.methods.execute.inputs;
-        schema.outputs = schema.methods.execute.outputs;
-        schema.ast = ast;
-        return schema;
-    }
-
-    OperationParser._traverse = traverse;
-    OperationParser._getAst = function(src, filename) {
-        filename = filename || 'operation.py';
-        var cst = Sk.parse(filename, src).cst;
-        var ast = Sk.astFromParse(cst, filename);
-        return ast;
-    };
-
-    OperationParser.parse = function(src, filename) {
-        //try {
-            var ast = this._getAst(src, filename);
-            return  parseOperationAst(ast);
-        //} catch (e) {
-            //console.error('operation parsing failed:', e);
-            //return null;
-        //}
-    };
-
-    return OperationParser;
-}));

src/common/globals.js

@@ -59,14 +59,8 @@ define([
     var createNamedNode = function(baseId, parentId, isMeta) {
         var newId = client.createNode({parentId, baseId}),
             baseNode = client.getNode(baseId),
-            basename,
-            newName;
-
-        basename = 'New';
-        if (baseNode.getAttribute('name') !== 'Operation') {
-            basename += baseNode.getAttribute('name');
-        }
-        newName = getUniqueName(parentId, basename);
+            basename = 'New' + baseNode.getAttribute('name'),
+            newName = getUniqueName(parentId, basename);
 
         // If instance, make the first char lowercase
         if (!isMeta) {

src/common/lua.js

@@ -3234,7 +3234,7 @@ function runScripts(){
 
 if (window.addEventListener){
     window.addEventListener("DOMContentLoaded", runScripts, false);
-} else if (window.attachEvent) {
+} else {
     window.attachEvent("onload", runScripts);
 }
 

src/common/node-brython.js

@@ -1,177 +0,0 @@
-/*
-Author: Billy Earney
-Date: 04/19/2013
-License: MIT
-
-Description: This file can work as a "bridge" between nodejs and brython
- so that client side brython code can be executed on the server side.
-Will brython replace Cython one day?  Only time will tell.
-:)
-
-*/
-
-var fs = require('fs'),
-    path = require('path'),
-    //brythonSrcPath = path.join(__dirname, '..', '..', 'node_modules', 'brython', 'www', 'src', 'brython.js');
-    brythonSrcPath = path.join(__dirname, 'brython.js');
-
-document={};
-document.getElementsByTagName = () => [{src: ''}];
-window={};
-window.location = {href: ''};
-window.navigator={}
-window.confirm = () => true;
-window.console = console;
-document.$py_src = {}
-document.$debug = 0
-
-self={};
-__BRYTHON__={}
-__BRYTHON__.$py_module_path = {}
-__BRYTHON__.$py_module_alias = {}
-__BRYTHON__.$py_next_hash = -Math.pow(2,53)
-__BRYTHON__.exception_stack = []
-__BRYTHON__.scope = {}
-__BRYTHON__.modules = {}
-
-// Read and eval library
-jscode = fs.readFileSync(brythonSrcPath, 'utf8');
-eval(jscode);
-
-//function node_import(module,alias,names) {
-function $import_single(module) {
-  var search_path=['../src/libs', '../src/Lib'];
-  var ext=['.js', '.py'];
-  var mods=[module, module+'/__init__'];
-
-  for(var i=0, _len_i = search_path.length; i < _len_i; i++) {
-     for (var j=0, _len_j = ext.length; j < _len_j; j++) {
-         for (var k=0, _len_k = mods.length; k < _len_k; k++) {
-             var path=search_path[i]+'/'+mods[k]+ext[j]
-
-             //console.log("searching for " + path);
-             var module_contents;
-             try {
-               module_contents=fs.readFileSync(path, 'utf8')
-             } catch(err) {}
-             if (module_contents !== undefined) {
-                console.log("imported " + module)
-                //console.log(module_contents);
-                if (ext[j] == '.js') {
-                   return $import_js_module(module,alias,names,path,module_contents)
-                }
-                return $import_py_module(module,alias,names,path,module_contents)
-             }
-         }
-     }
-  }
-  console.log("error time!");
-  res = Error()
-  res.name = 'NotFoundError'
-  res.message = "No module named '"+module+"'"
-  throw res
-}
-
-$compile_python=function(module_contents,module) {
-    var root = __BRYTHON__.py2js(module_contents,module)
-    var body = root.children
-    root.children = []
-    // use the module pattern : module name returns the results of an anonymous function
-    var mod_node = new $Node('expression')
-    //if(names!==undefined){alias='$module'}
-    new $NodeJSCtx(mod_node,'$module=(function()')
-    root.insert(0,mod_node)
-    mod_node.children = body
-    // search for module-level names : functions, classes and variables
-    var mod_names = []
-    for(var i=0, _len_i = mod_node.children.length; i < _len_i;i++){
-        var node = mod_node.children[i]
-        // use function get_ctx() 
-        // because attribute 'context' is renamed by make_dist...
-        var ctx = node.get_ctx().tree[0]
-        if(ctx.type==='def'||ctx.type==='class'){
-            if(mod_names.indexOf(ctx.name)===-1){mod_names.push(ctx.name)}
-        } else if(ctx.type==='from') {
-            for (var j=0, _len_j = ctx.names.length; j < _len_j; j++) {
-                var name=ctx.names[j];
-                if (name === '*') {
-                   // just pass, we don't want to include '*'
-                } else if (ctx.aliases[name] !== undefined) {
-                   if (mod_names.indexOf(ctx.aliases[name])===-1){
-                      mod_names.push(ctx.aliases[name])
-                   }
-                } else {
-                   if (mod_names.indexOf(ctx.names[j])===-1){
-                     mod_names.push(ctx.names[j])
-                   }
-                }
-            }
-        }else if(ctx.type==='assign'){
-            var left = ctx.tree[0]
-            if(left.type==='expr'&&left.tree[0].type==='id'&&left.tree[0].tree.length===0){
-                var id_name = left.tree[0].value
-                if(mod_names.indexOf(id_name)===-1){mod_names.push(id_name)}
-            }
-        }
-    }
-    // create the object that will be returned when the anonymous function is run
-    var ret_code = 'return {'
-    for(var i=0, _len_i = mod_names.length; i < _len_i;i++){
-        ret_code += mod_names[i]+':'+mod_names[i]+','
-    }
-    ret_code += '__getattr__:function(attr){return this[attr]},'
-    ret_code += '__setattr__:function(attr,value){this[attr]=value}'
-    ret_code += '}'
-    var ret_node = new $Node('expression')
-    new $NodeJSCtx(ret_node,ret_code)
-    mod_node.add(ret_node)
-    // add parenthesis for anonymous function execution
-    
-    var ex_node = new $Node('expression')
-    new $NodeJSCtx(ex_node,')()')
-    root.add(ex_node)
-    
-    try{
-        var js = root.to_js()
-        return js;
-    }catch(err){
-        eval('throw '+err.name+'(err.message)')
-    }
-    return undefined;
-}
-
-function build_ast(src) {
-  __BRYTHON__.$py_module_path['__main__']='./'
-  return __BRYTHON__.py2js(src,'__main__', '__main__', '__builtins__')
-}
-
-function execute_python_script(filename) {
-  _py_src=fs.readFileSync(filename, 'utf8')
-  var root = build_ast(_py_src)
-  var js = root.to_js()
-  //eval(js);
-}
-
-//console.log("try to execute compile script");
-
-__BRYTHON__.$py_module_path = __BRYTHON__.$py_module_path || {}
-__BRYTHON__.$py_module_alias = __BRYTHON__.$py_module_alias || {}
-__BRYTHON__.exception_stack = __BRYTHON__.exception_stack || []
-__BRYTHON__.scope = __BRYTHON__.scope || {}
-__BRYTHON__.imported = __BRYTHON__.imported || {}
-__BRYTHON__.modules = __BRYTHON__.modules || {}
-__BRYTHON__.compile_python=$compile_python
-
-__BRYTHON__.debug = 0
-__BRYTHON__.$options = {}
-__BRYTHON__.$options.debug = 0
-
-// other import algs don't work in node
-//import_funcs=[node_import]
-
-if (!module.parent) {
-    var filename=process.argv[2];
-    execute_python_script(filename)
-}
-
-module.exports = __BRYTHON__;

src/common/viz/OperationControl.js

@@ -38,103 +38,5 @@ define([
         return typeId ? this._client.getNode(typeId).getChildrenIds() : [];
     };
 
-    OperationControl.prototype.createIONode = function(opId, typeId, isInput, baseName, silent) {
-        var cntrId = this.getDataContainerId(opId, isInput),
-            name = this._client.getNode(opId).getAttribute('name'),
-            dataName,
-            msg;
-
-        baseName = baseName || this._client.getNode(typeId).getAttribute('name').toLowerCase();
-        dataName = this._getDataName(cntrId, baseName);
-
-        msg = `Adding ${isInput ? 'input' : 'output'} "${dataName}" to ${name} interface`;
-        if (!silent) {
-            this._client.startTransaction(msg);
-        }
-
-        var id = this._client.createNode({
-            parentId: cntrId,
-            baseId: typeId
-        });
-
-        // Set the name of the new input
-        this._client.setAttribute(id, 'name', dataName);
-
-        if (!silent) {
-            this._client.completeTransaction();
-        }
-        return id;
-    };
-
-    OperationControl.prototype._getDataName = function(cntrId, baseName) {
-        var otherNames = this._getDataNames(cntrId),
-            name = baseName,
-            i = 1;
-
-        while (otherNames.indexOf(name) !== -1) {
-            i++;
-            name = baseName + '_' + i;
-        }
-        return name;
-    };
-
-    OperationControl.prototype._getDataNames = function(cntrId) {
-        var otherIds = this._client.getNode(cntrId).getChildrenIds();
-
-        return otherIds.map(id => this._client.getNode(id).getAttribute('name'));
-    };
-
-    OperationControl.prototype.getDataNames = function(opId, isInput) {
-        return this._getDataNames(this.getDataContainerId(opId, isInput));
-    };
-
-    OperationControl.prototype.getDataContainerId = function(opId, isInput) {
-        var node = this._client.getNode(opId),
-            cntrs = node.getChildrenIds(),
-            cntrType = isInput ? 'Inputs' : 'Outputs';
-
-        return cntrs.find(id => this.hasMetaName(id, cntrType));
-    };
-
-    OperationControl.prototype.getDataTypeId = function() {
-        var dataNode = this._client.getAllMetaNodes()
-            .find(node => node.getAttribute('name') === 'Data');
-
-        return dataNode.getId();
-    };
-
-    OperationControl.prototype.addInputData = function(opId, name) {
-        return this.createIONode(opId, this.getDataTypeId(), true, name, true);
-    };
-
-    OperationControl.prototype.removeInputData = function(opId, name) {
-        var cntrId = this.getDataContainerId(opId, true),
-            otherIds = this._client.getNode(cntrId).getChildrenIds(),
-            dataId = otherIds.find(id => this._client.getNode(id).getAttribute('name') === name);
-
-        if (dataId) {  // ow, data not found
-            this._client.deleteNode(dataId);
-        }
-    };
-
-    OperationControl.prototype.addOutputData = function(opId, name) {
-        return this.createIONode(opId, this.getDataTypeId(), false, name, true);
-    };
-
-    OperationControl.prototype.removeOutputData = function(opId, name) {
-        var cntrId = this.getDataContainerId(opId),
-            otherIds = this._client.getNode(cntrId).getChildrenIds(),
-            dataId = otherIds.find(id => this._client.getNode(id).getAttribute('name') === name);
-
-        if (dataId) {  // ow, data not found
-            this._client.deleteNode(dataId);
-        }
-    };
-
-    OperationControl.prototype.isInputData = function(nodeId) {
-        var node = this._client.getNode(nodeId);
-        return this.hasMetaName(node.getParentId(), 'Inputs');
-    };
-
     return OperationControl;
 });

src/decorators/ContainerLayerDecorator/EasyDAG/NestedLayer.js

@@ -42,8 +42,8 @@ define([
             embedded: true,
             widget: this.widget
         });
-        this.control._onUnload = () => {
-            ArchEditor.prototype._onUnload.apply(this.control, arguments);
+        this.control._onUnload = id => {
+            ArchEditor.prototype._onUnload.call(this.control, id);
             // If it was the last node, remove it
             var node = this.control._client.getNode(this.id);
             if (node.getChildrenIds().length === 0) {

src/decorators/OpIntDecorator/EasyDAG/OpIntDecorator.EasyDAGWidget.js

@@ -103,7 +103,6 @@ define([
             () => this.deleteAttribute(name));
     };
 
-    // TODO: implement this in the widget controller (so we can update the op code)
     OpIntDecorator.prototype.deleteAttribute = function(name) {
         var opName = this._node.attributes.name.value,
             msg = `Deleting "${name}" attribute from "${opName}" operation`;
@@ -114,7 +113,6 @@ define([
         this.client.completeTransaction();
     };
 
-    // TODO: implement this in the widget controller (so we can update the op code)
     OpIntDecorator.prototype.setAttributeMeta = function(name, desc) {
         var schema,
             opName = this._node.attributes.name.value,

src/plugins/CreateTorchMeta/CreateTorchMeta.js

@@ -205,10 +205,9 @@ define([
     };
 
     // Some helper methods w/ attribute handling
-    var PYTHON_TO_GME = {
+    var LUA_TO_GME = {
         boolean: 'boolean',
-        float: 'float',
-        int: 'integer',
+        number: 'float',
         string: 'string'
     };
 
@@ -302,7 +301,7 @@ define([
             attrs.forEach(name => {
                 desc = {};
                 defVal = defaults.hasOwnProperty(name) ? defaults[name] : '';
-                type = PYTHON_TO_GME[types[name]];
+                type = LUA_TO_GME[types[name]];
                 if (type) {
                     desc.type = type;
                 }
@@ -377,11 +376,6 @@ define([
             // Set the min, max
             schema.max = +schema.max;
         }
-        // Add the enum for booleans so we use python style True/False
-        if (schema.type === 'boolean') {
-            schema.enum = ['True', 'False'];
-            schema.type = 'string';
-        }
 
         // Create the attribute and set the schema
         this.core.setAttributeMeta(node, name, schema);

src/plugins/CreateTorchMeta/metadata.json

@@ -17,6 +17,7 @@
       "value": "all",
       "valueItems": [
         "nn",
+        "rnn",
         "all"
       ],
       "valueType": "string",

src/plugins/CreateTorchMeta/schemas/index.js

@@ -1,10 +1,13 @@
 /*globals define*/
 define([
-    'text!./nn.json'
+    'text!./nn.json',
+    'text!./rnn.json'
 ], function(
-    nn
+    nn,
+    rnn
 ) {
     return {
-        nn: nn
+        nn: nn,
+        rnn: rnn
     };
 });

src/plugins/CreateTorchMeta/schemas/rnn.json

@@ -0,0 +1,178 @@
+[
+  {
+    "name": "CopyGrad",
+    "baseType": "Identity",
+    "setters": {},
+    "defaults": {},
+    "type": "RNN"
+  },
+  {
+    "name": "FastLSTM",
+    "baseType": "LSTM",
+    "params": [
+      "inputSize",
+      "outputSize",
+      "rho",
+      "eps",
+      "momentum",
+      "affine"
+    ],
+    "setters": {},
+    "types": {
+      "eps": "number",
+      "momentum": "number"
+    },
+    "defaults": {
+      "momentum": 0.1,
+      "eps": 0.1
+    },
+    "type": "RNN"
+  },
+  {
+    "name": "LSTM",
+    "baseType": "AbstractRecurrent",
+    "params": [
+      "inputSize",
+      "outputSize",
+      "rho",
+      "cell2gate"
+    ],
+    "setters": {},
+    "types": {
+      "rho": "number"
+    },
+    "defaults": {
+      "rho": 9999
+    },
+    "type": "RNN"
+  },
+  {
+    "name": "LinearNoBias",
+    "baseType": "Linear",
+    "params": [
+      "inputSize",
+      "outputSize"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "Simple"
+  },
+  {
+    "name": "LookupTableMaskZero",
+    "baseType": "LookupTable",
+    "params": [
+      "nIndex",
+      "nOutput"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "RNN"
+  },
+  {
+    "name": "NormStabilizer",
+    "baseType": "AbstractRecurrent",
+    "params": [
+      "beta"
+    ],
+    "setters": {},
+    "defaults": {},
+    "type": "RNN"
+  },
+  {
+    "name": "Recurrent",
+    "baseType": "AbstractRecurrent",
+    "params": [
+      "start",
+      "input",
+      "feedback",
+      "transfer",
+      "rho",
+      "merge"
+    ],
+    "setters": {},
+    "types": {
+      "start": "nn.Module",
+      "transfer": "nn.Module",
+      "feedback": "nn.Module",
+      "input": "nn.Module"
+    },
+    "defaults": {},
+    "type": "RNN"
+  },
+  {
+    "name": "SAdd",
+    "baseType": "Module",
+    "params": [
+      "addend",
+      "negate"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "RNN"
+  },
+  {
+    "name": "SeqBRNN",
+    "baseType": "Container",
+    "params": [
+      "inputDim",
+      "hiddenDim",
+      "batchFirst"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "RNN"
+  },
+  {
+    "name": "SeqGRU",
+    "baseType": "Module",
+    "params": [
+      "inputSize",
+      "outputSize"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "RNN"
+  },
+  {
+    "name": "SeqLSTM",
+    "baseType": "Module",
+    "params": [
+      "inputsize",
+      "hiddensize",
+      "outputsize"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "RNN"
+  },
+  {
+    "name": "SeqLSTMP",
+    "baseType": "SeqLSTM",
+    "params": [
+      "inputsize",
+      "hiddensize",
+      "outputsize"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "RNN"
+  },
+  {
+    "name": "SeqReverseSequence",
+    "baseType": "Module",
+    "params": [
+      "dim"
+    ],
+    "setters": {},
+    "types": {},
+    "defaults": {},
+    "type": "RNN"
+  }
+]

src/plugins/GenerateArchitecture/GenerateArchitecture.js

@@ -49,6 +49,10 @@ define([
         this.LayerDict = createLayerDict(this.core, this.META);
         this.uniqueId = 2;
         this.varnames = {net: true};
+        this.definitions = [
+            'require \'nn\'',
+            'require \'rnn\''
+        ];
 
         return PluginBase.prototype.main.apply(this, arguments);
     };
@@ -72,11 +76,6 @@ define([
             result = {},
             code = '';
 
-        this.definitions = [
-            'import torch',
-            'import torch.nn as nn'
-        ];
-
         // Add an index to each layer
         layers.forEach((l, index) => l[INDEX] = index);
 
@@ -86,7 +85,6 @@ define([
             code += this.genLayerDefinitions(layers);
         }
 
-        // TODO: Define the network w/ 'class ARCHITECTURE_NAME'
         this.logger.debug('Generating architecture code...');
         code += this.genArchCode(layers);
         this.logger.debug('Prepending hoisted code...');

src/plugins/GenerateJob/GenerateJob.js

@@ -107,24 +107,16 @@ define([
             .then(mds => {
                 // Record the large files
                 var inputData = {},
-                    runsh = [
-                        '# Bash script to download data files and run job',
-                        'if [ -z "$DEEPFORGE_URL" ]; then',
-                        '  echo "Please set DEEPFORGE_URL and re-run:"',
-                        '  echo ""',
-                        '  echo "  DEEPFORGE_URL=http://my.deepforge.server.com:8080 bash run.sh"',
-                        '  echo ""',
-                        '  exit 1',
-                        'fi',
-                        'mkdir outputs\n'
-                    ].join('\n');
+                    runsh = '# Bash script to download data files and run job\n' +
+                    'if [ -z "$DEEPFORGE_URL" ]; then\n  echo "Please set DEEPFORGE_URL and' +
+                    ' re-run:"\n  echo ""  \n  echo "  DEEPFORGE_URL=http://my.' +
+                    'deepforge.server.com:8080 bash run.sh"\n  echo ""\n exit 1\nfi\n';
 
                 mds.forEach((metadata, i) => {
                     // add the hashes for each input
                     var input = inputs[i], 
                         hash = files.inputAssets[input],
-                        dataDir = 'inputs/' + input + '/',
-                        dataPath = dataDir + 'data',
+                        dataPath = 'inputs/' + input + '/data',
                         url = this.blobClient.getRelativeDownloadURL(hash);
 
                     inputData[dataPath] = {
@@ -133,13 +125,12 @@ define([
                     };
 
                     // Add to the run.sh file
-                    runsh += `mkdir -p ${dataDir}\n`;
                     runsh += `wget $DEEPFORGE_URL${url} -O ${dataPath}\n`;
                 });
 
                 delete files.inputAssets;
                 files['input-data.json'] = JSON.stringify(inputData, null, 2);
-                runsh += 'python main.py';
+                runsh += 'th init.lua';
                 files['run.sh'] = runsh;
 
                 // Add pointer assets
@@ -218,21 +209,23 @@ define([
     GenerateJob.prototype.createOperationFiles = function (node) {
         var files = {};
         // For each operation, generate the output files:
-        //   inputs/<arg-name>/init.py  (respective data deserializer)
-        //   pointers/<name>/init.py  (result of running the main plugin on pointer target - may need a rename)
+        //   inputs/<arg-name>/init.lua  (respective data deserializer)
+        //   pointers/<name>/init.lua  (result of running the main plugin on pointer target - may need a rename)
         //   outputs/<name>/  (make dirs for each of the outputs)
-        //   outputs/init.py  (serializers for data outputs)
+        //   outputs/init.lua  (serializers for data outputs)
         //
-        //   attributes.py (returns py table of operation attributes)
-        //   init.py (main file -> calls main and serializes outputs)
-        //   <name>.py (entry point -> calls main operation code)
+        //   attributes.lua (returns lua table of operation attributes)
+        //   init.lua (main file -> calls main and serializes outputs)
+        //   <name>.lua (entry point -> calls main operation code)
 
         // add the given files
-        this.logger.info('About to generate operation execution files');
+        this.logger.info('About to create dist execution files');
+        files['start.js'] = _.template(Templates.START)(CONSTANTS);
         return this.createEntryFile(node, files)
             .then(() => this.createClasses(node, files))
             .then(() => this.createCustomLayers(node, files))
             .then(() => this.createInputs(node, files))
+            .then(() => this.createOutputs(node, files))
             .then(() => this.createMainFile(node, files))
             .then(() => {
                 this.createAttributeFile(node, files);
@@ -245,8 +238,7 @@ define([
     };
 
     GenerateJob.prototype.createEntryFile = function (node, files) {
-        this.logger.info('Creating deepforge.py file...');
-        files['deepforge.py'] = _.template(Templates.DEEPFORGE)(CONSTANTS);
+        this.logger.info('Creating entry files...');
         return this.getOutputs(node)
             .then(outputs => {
                 var name = this.getAttribute(node, 'name'),
@@ -254,9 +246,12 @@ define([
 
                 // inputs and outputs
                 content.name = name;
-                content.outputs = outputs.map(output => output[0]);
+                content.outputs = outputs;
+
+                files['init.lua'] = _.template(Templates.ENTRY)(content);
 
                 // Create the deepforge file
+                files['deepforge.lua'] = _.template(Templates.DEEPFORGE)(CONSTANTS);
             });
     };
 
@@ -299,18 +294,17 @@ define([
 
             return inheritanceLvl[aId] > inheritanceLvl[bId];
         }).map(node =>
-            // FIXME: update this
-            `require './${this.getAttribute(node, 'name')}.py'`
+            `require './${this.getAttribute(node, 'name')}.lua'`
         ).join('\n');
 
         // Create the class files
         classNodes.forEach(node => {
             var name = this.getAttribute(node, 'name');
-            files[`classes/${name}.py`] = this.getAttribute(node, 'code');
+            files[`classes/${name}.lua`] = this.getAttribute(node, 'code');
         });
 
         // Create the custom layers file
-        files['classes/init.py'] = code;
+        files['classes/init.lua'] = code;
     };
 
     GenerateJob.prototype.getTypeDictFor = function (name, metanodes) {
@@ -324,7 +318,6 @@ define([
         return isType;
     };
 
-    // TODO: update this to nn modules
     GenerateJob.prototype.createCustomLayers = function (node, files) {
         var metaDict = this.core.getAllMetaNodes(this.rootNode),
             isCustomLayer,
@@ -344,7 +337,7 @@ define([
             .map(node => this.getAttribute(node, 'code')).join('\n');
 
         // Create the custom layers file
-        files['custom-layers.py'] = code;
+        files['custom-layers.lua'] = code;
     };
 
     GenerateJob.prototype.getConnectionContainer = function () {
@@ -381,15 +374,11 @@ define([
                 //
                 // For each input,
                 //  - create the deserializer
-                //  - put it in inputs/<name>/init.py
-                //  - copy the data asset to /inputs/<name>/init.py
+                //  - put it in inputs/<name>/init.lua
+                //  - copy the data asset to /inputs/<name>/init.lua
                 inputs = allInputs
                     .filter(pair => !!this.getAttribute(pair[2], 'data'));  // remove empty inputs
 
-                files['start.js'] = _.template(Templates.START)({
-                    CONSTANTS,
-                    inputs: inputs.map(pair => pair[0])
-                });
                 files.inputAssets = {};  // data assets
                 return Q.all(inputs.map(pair => {
                     var name = pair[0],
@@ -425,25 +414,67 @@ define([
             })
             .then(_tplContents => {
                 tplContents = _tplContents;
-                inputs.forEach(pair => {
+                var hashes = inputs.map(pair => {
                     var hash = this.getAttribute(pair[2], 'data');
                     files.inputAssets[pair[0]] = hash;
+                    return {
+                        hash: hash,
+                        name: pair[0]
+                    };
                 });
 
+                return Q.all(hashes.map(pair => 
+                    this.blobClient.getMetadata(pair.hash)
+                        .fail(() => {
+                            throw Error(`BLOB_FETCH_FAILED:${pair.name}`);
+                        })));
+            })
+            .then(metadatas => {
+                // Create the deserializer
+                tplContents.forEach((ctnt, i) => {
+                    // Get the name of the given asset
+                    ctnt.filename = metadatas[i].name;
+                    files['inputs/' + ctnt.name + '/init.lua'] = _.template(Templates.DESERIALIZE)(ctnt);
+                });
                 return files;
             });
     };
 
+    GenerateJob.prototype.createOutputs = function (node, files) {
+        // For each of the output types, grab their serialization functions and
+        // create the `outputs/init.lua` file
+        this.logger.info('Creating outputs/init.lua...');
+        return this.getOutputs(node)
+            .then(outputs => {
+                var outputTypes = outputs
+                // Get the serialize functions for each
+                    .map(tuple => {
+                        var node = tuple[2],
+                            serFn = this.getAttribute(node, 'serialize');
+
+                        if (this.isMetaTypeOf(node, this.META.Complex)) {
+                            // Complex objects are expected to define their own
+                            // serialize methods
+                            serFn = 'if data ~= nil then data:serialize(path) end';
+                        }
+
+                        return [tuple[1], serFn];
+                    });
+
+                files['outputs/init.lua'] = _.template(Templates.SERIALIZE)({types: outputTypes});
+            });
+    };
+
     GenerateJob.prototype.createMainFile = function (node, files) {
         this.logger.info('Creating main file...');
-        var content = {};
         return this.getInputs(node)
             .then(inputs => {
                 var name = this.getAttribute(node, 'name'),
                     code = this.getAttribute(node, 'code'),
-                    pointers = this.core.getPointerNames(node).filter(ptr => ptr !== 'base');
-
-                content.name = name;
+                    pointers = this.core.getPointerNames(node).filter(ptr => ptr !== 'base'),
+                    content = {
+                        name: name
+                    };
 
                 // Get input data arguments
                 content.inputs = inputs
@@ -455,16 +486,11 @@ define([
 
                 // Add remaining code
                 content.code = code;
-                return this.getOutputs(node);
-            })
-            .then(outputs => {
-                content.outputs = outputs.map(output => output[0]);
 
-                files['main.py'] = _.template(Templates.MAIN)(content);
-                files['operations.py'] = content.code;
+                files['main.lua'] = _.template(Templates.MAIN)(content);
 
                 // Set the line offset
-                var lineOffset = 0;
+                var lineOffset = this.getLineOffset(files['main.lua'], code);
                 this.setAttribute(node, CONSTANTS.LINE_OFFSET, lineOffset);
             });
     };
@@ -478,7 +504,6 @@ define([
 
     GenerateJob.prototype.createAttributeFile = function (node, files) {
         var numOrBool = /^(-?\d+\.?\d*((e|e-)\d+)?|(true|false))$/,
-            isBool = /^(true|false)$/,
             table;
 
         this.logger.info('Creating attributes file...');
@@ -489,16 +514,12 @@ define([
                 if (!numOrBool.test(value)) {
                     value = `"${value}"`;
                 }
-                if (isBool.test(value)) {  // Convert to python bool
-                    value = value.toString();
-                    value = value[0].toUpperCase() + value.slice(1);
-                }
-                return [`'${name}'`, value];
+                return [`['${name}']`, value];
             })
-            .map(pair => pair.join(': '))
-            .join(',\n    ') + '\n}';
+            .map(pair => pair.join(' = '))
+            .join(',\n\t') + '\n}';
 
-        files['attributes.py'] = `# attributes of ${this.getAttribute(node, 'name')}\nattributes = ${table}`;
+        files['attributes.lua'] = `-- attributes of ${this.getAttribute(node, 'name')}\nreturn ${table}`;
     };
 
     GenerateJob.prototype.createPointers = function (node, files, cb) {
@@ -519,7 +540,7 @@ define([
             var name = this.getAttribute(node, 'name');
             this.logger.info(`Pointer generation for ${name} FINISHED!`);
             resultHashes.forEach((hash, index) => {
-                files.ptrAssets[`pointers/${pointers[index]}/init.py`] = hash;
+                files.ptrAssets[`pointers/${pointers[index]}/init.lua`] = hash;
             });
             return cb(null, files);
         })

src/plugins/GenerateJob/templates/deepforge.ejs

@@ -1,105 +1,105 @@
-# Instantiate the deepforge object
+-- Instantiate the deepforge object
+deepforge = {}
 
-#class deepforge()
-# function deepforge.initialize()
-    # require 'nn'
-    # require 'rnn'
-    # require './classes/init'
-    # require './custom-layers'
-# end
+function deepforge.initialize()
+    require 'nn'
+    require 'rnn'
+    require './classes/init'
+    require './custom-layers'
+end
 
-# function deepforge.id()
-    # if __deepforge_id == nil then
-        # __deepforge_id = 0
-    # end
-    # __deepforge_id = __deepforge_id + 1
-    # return __deepforge_id
-# end
-# 
-# function deepforge._cmd(...)
-    # local cmd = '<%= START_CMD %>'
-    # local arg = {...}
-    # local n = #arg
-    # for i=1,n do
-        # cmd = cmd .. ' ' .. tostring(arg[i])
-    # end
-    # print(cmd .. ' ')  # guarantee ends w/ space
-# end
-# 
-# # Graph support
-# Graph = torch.class('deepforge.Graph')
-# 
-# function Graph:__init(name)
-    # self.id = deepforge.id()
-    # self.name = name
-    # deepforge._cmd('<%= GRAPH_CREATE %>', self.id, name)
-# end
-# 
-# _Line = torch.class('deepforge._Line')
-# 
-# function _Line:__init(graphId, name, opts)
-    # self.id = deepforge.id()
-    # self.name = name
-    # deepforge._cmd('<%= GRAPH_CREATE_LINE %>', graphId, self.id, name)
-# end
-# 
-# function _Line:add(x, y)
-    # assert(type(x) == "number" and type(y) == "number", "adding point (" .. tostring(x) .. ", " .. tostring(y) .. ") to " .. self.name .. " failed: expected (number, number)")
-    # deepforge._cmd('<%= GRAPH_PLOT %>', self.id, x, y)
-# end
-# 
-# function Graph:line(name, opts)
-    # return deepforge._Line(self.id, name, opts)
-# end
-# 
-# function Graph:xlabel(name)
-    # deepforge._cmd('<%= GRAPH_LABEL_AXIS.X %>', self.id, name)
-# end
-# 
-# function Graph:ylabel(name)
-    # deepforge._cmd('<%= GRAPH_LABEL_AXIS.Y %>', self.id, name)
-# end
-# 
-# # Image support
-# local function saveImage(name, tensor)
-    # require 'image'
-    # require 'paths'
-# 
-    # # save it in the tmp directory
-    # local filename = name .. '.png'
-    # local path = paths.concat('metadata', filename)
-# 
-    # if paths.dir('metadata') == nil then
-       # paths.mkdir('metadata')
-    # end
-# 
-    # image.save(path, tensor)
-# end
-# 
-# function deepforge.image(name, tensor)
-    # saveImage(name, tensor)
-    # deepforge._cmd("<%= IMAGE.BASIC %>", deepforge.id(), name)
-# end
-# 
-# Image = torch.class('deepforge.Image')
-# function Image:__init(name, tensor)
-    # self.id = deepforge.id()
-    # self.name = name
-# 
-    # if tensor ~= nil then
-        # saveImage(name, tensor)
-        # deepforge._cmd('<%= IMAGE.CREATE %>', self.id, self.name)
-    # end
-# end
-# 
-# function Image:update(tensor)
-    # saveImage(self.name, tensor)
-    # deepforge._cmd('<%= IMAGE.UPDATE %>', self.id, self.name)
-# end
-# 
-# function Image:title(name)
-    # self.name = name
-    # deepforge._cmd('<%= IMAGE.NAME %>', self.id, self.name)
-# end
-# 
-# return deepforge
+function deepforge.id()
+    if __deepforge_id == nil then
+        __deepforge_id = 0
+    end
+    __deepforge_id = __deepforge_id + 1
+    return __deepforge_id
+end
+
+function deepforge._cmd(...)
+    local cmd = '<%= START_CMD %>'
+    local arg = {...}
+    local n = #arg
+    for i=1,n do
+        cmd = cmd .. ' ' .. tostring(arg[i])
+    end
+    print(cmd .. ' ')  -- guarantee ends w/ space
+end
+
+-- Graph support
+Graph = torch.class('deepforge.Graph')
+
+function Graph:__init(name)
+    self.id = deepforge.id()
+    self.name = name
+    deepforge._cmd('<%= GRAPH_CREATE %>', self.id, name)
+end
+
+_Line = torch.class('deepforge._Line')
+
+function _Line:__init(graphId, name, opts)
+    self.id = deepforge.id()
+    self.name = name
+    deepforge._cmd('<%= GRAPH_CREATE_LINE %>', graphId, self.id, name)
+end
+
+function _Line:add(x, y)
+    assert(type(x) == "number" and type(y) == "number", "adding point (" .. tostring(x) .. ", " .. tostring(y) .. ") to " .. self.name .. " failed: expected (number, number)")
+    deepforge._cmd('<%= GRAPH_PLOT %>', self.id, x, y)
+end
+
+function Graph:line(name, opts)
+    return deepforge._Line(self.id, name, opts)
+end
+
+function Graph:xlabel(name)
+    deepforge._cmd('<%= GRAPH_LABEL_AXIS.X %>', self.id, name)
+end
+
+function Graph:ylabel(name)
+    deepforge._cmd('<%= GRAPH_LABEL_AXIS.Y %>', self.id, name)
+end
+
+-- Image support
+local function saveImage(name, tensor)
+    require 'image'
+    require 'paths'
+
+    -- save it in the tmp directory
+    local filename = name .. '.png'
+    local path = paths.concat('metadata', filename)
+
+    if paths.dir('metadata') == nil then
+       paths.mkdir('metadata')
+    end
+
+    image.save(path, tensor)
+end
+
+function deepforge.image(name, tensor)
+    saveImage(name, tensor)
+    deepforge._cmd("<%= IMAGE.BASIC %>", deepforge.id(), name)
+end
+
+Image = torch.class('deepforge.Image')
+function Image:__init(name, tensor)
+    self.id = deepforge.id()
+    self.name = name
+
+    if tensor ~= nil then
+        saveImage(name, tensor)
+        deepforge._cmd('<%= IMAGE.CREATE %>', self.id, self.name)
+    end
+end
+
+function Image:update(tensor)
+    saveImage(self.name, tensor)
+    deepforge._cmd('<%= IMAGE.UPDATE %>', self.id, self.name)
+end
+
+function Image:title(name)
+    self.name = name
+    deepforge._cmd('<%= IMAGE.NAME %>', self.id, self.name)
+end
+
+return deepforge

src/plugins/GenerateJob/templates/entry.ejs

@@ -1,7 +1,16 @@
-import main
-import pickle
+-- Instantiate the deepforge object
+require './deepforge'
 
-# run the <%= name %> and serialize the results
+-- run the <%= name %> and serialize the results
+print('\n############### Running "<%= name.replace(/'/g, '\\\'') %>" Operation ############### ')
+results = require './main'
+print('############### "<%= name.replace(/'/g, '\\\'') %>" Operation Complete! ###############')
 
-# serialize by type
-import outputs
+-- serialize by type
+outputs = require './outputs'
+<% outputs.forEach(pair => { 
+    var name = pair[0],
+        type = pair[1];
+%>
+outputs.<%= type %>('<%= name %>', results.<%= name %>)
+<% }); %>

src/plugins/GenerateJob/templates/index.js

@@ -1,12 +1,14 @@
 /*globals define*/
 define([
     'text!./start.ejs',
+    'text!./entry.ejs',
     'text!./main.ejs',
     'text!./deepforge.ejs',
     'text!./serialize.ejs',
     'text!./deserialize.ejs'
 ], function(
     START,
+    ENTRY,
     MAIN,
     DEEPFORGE,
     SERIALIZE,
@@ -15,6 +17,7 @@ define([
 
     return {
         START,
+        ENTRY,
         MAIN,
         SERIALIZE,
         DEEPFORGE,

src/plugins/GenerateJob/templates/main.ejs

@@ -1,29 +1,12 @@
-import pickle
-# load custom layers and classes
-# from deepforge import deepforge
+-- load custom layers and classes
+deepforge.initialize()
 
-# deepforge.initialize()
+-- input data<% inputs.forEach(function(pair) { var input = pair[0], isNil = pair[1];%>
+local <%= input %> = <% if (isNil) { %>nil<% } else { %>require './inputs/<%= input %>'<%}}); %>
 
-# input data<% inputs.forEach(function(pair) { var input = pair[0], isNil = pair[1];%>
-<%= input %> = <% if (isNil) { %>None<% } else { %>pickle.load(open('./inputs/<%= input %>/data', 'rb')) <%}}); %>
+-- load references<% pointers.forEach(function(pair) { var pointer = pair[0], isNil = pair[1];%>
+local <%= pointer %> = <% if (isNil) { %>nil<% } else { %>require './pointers/<%= pointer %>'<%}}); %>
+local attributes = require './attributes'
 
-# load references<% pointers.forEach(function(pair) { var pointer = pair[0], isNil = pair[1];%>
-from pointers import <%= pointer %>
-<%}); %>
-
-from operations import <%= name %>Operation
-from attributes import attributes
-
-# main operation code for <%= name %>
-operation = <%= name %>Operation()
-print('\n############### Running "<%= name.replace(/'/g, '\\\'') %>" Operation ############### ')
-<%= outputs.length ? outputs.join(', ') : 'result' %> = operation.execute(<%= inputs.map(function(pair) {
-    var name = pair[0],
-        isNone = pair[1];
-    return isNone ? 'None' : name;
-})%>)
-print('############### "<%= name.replace(/'/g, '\\\'') %>" Operation Complete! ###############')
-
-<% outputs.forEach(name => { %>
-pickle.dump(<%= name %>, open('outputs/<%= name %>', 'wb'))
-<% }); %>
+-- main operation code for <%= name %>
+<%= code %>

src/plugins/GenerateJob/templates/serialize.ejs

@@ -1,17 +1,16 @@
-# Serialization functions for: <%= types.map(function(type) {return type[0];}).join('\n--   ') %>
+-- Serialization functions for: <%= types.map(function(type) {return type[0];}).join('\n--   ') %>
+require 'paths'
 
-serializer = {}
+local serializer = {}
 <% types.forEach(function(pair) {
 var type = pair[0],
-    fn = pair[1],
-    safeType = type.replace(/[^a-zA-Z\d_]/g, '_');
-
+    fn = pair[1];
 %>
-
-def <%= safeType %> (name, data):
-    path = 'outputs/' .. name
+function serializer.<%= type %> (name, data)
+    local path = 'outputs/' .. name
+    local abs_path = paths.concat('outputs', name)
     <%= fn.replace('\n', '\n    ') %>
-
-serializer['<%= type %>'] = <%= safeType %>
-
+end
 <% }); %>
+
+return serializer

src/plugins/GenerateJob/templates/start.ejs

@@ -11,8 +11,8 @@ var spawn = require('child_process').spawn,
 ['error', 'warn', 'info', 'log', 'debug'].forEach(method => logger[method] = log);
 
 // Get the BlobClient...
-var COMMAND_PREFIX = '<%= CONSTANTS.START_CMD %>',
-    IMAGE = '<%= CONSTANTS.IMAGE.PREFIX %>',
+var COMMAND_PREFIX = '<%= START_CMD %>',
+    IMAGE = '<%= IMAGE.PREFIX %>',
     requirejs = require('webgme').requirejs,
     remainingImageCount = 0,
     exitCode = null;
@@ -35,23 +35,14 @@ requirejs([
 
     // Create CACHE_DIR if it doesn't exist
     var prepareCache = function() {
-        var dirs = CACHE_DIR.replace(/\/$/, '').split('/'),
-            cacheParent;
+    var dirs = CACHE_DIR.replace(/\/$/, '').split('/'),
+        cacheParent;
 
         dirs.pop();
         cacheParent = dirs.join('/');
         return makeIfNeeded(cacheParent).then(() => makeIfNeeded(CACHE_DIR));
     };
 
-    var prepareInputsOutputs = function() {
-        var dirs = ['inputs', <% inputs.forEach(function(input) { %>
-            'inputs/<%= input %>',
-            <% }) %>
-            'outputs'];
-        return Q.all(dirs.map(dir => makeIfNeeded(dir)));
-    };
-
-
     var makeIfNeeded = function(dir) {
         var deferred = Q.defer(),
             job;
@@ -118,7 +109,7 @@ requirejs([
     var onStderr = function(data) {
         var text = data.toString();
         // Filter out directory label from stack traces
-        process.stdout.write(text.replace(/\.\.\.\/.*\/(main|deepforge|init).py/g, '$1'));
+        process.stdout.write(text.replace(/\.\.\.\/.*\/(main|deepforge|init).lua/g, '$1'));
     };
 
     var onStdout = function(data) {
@@ -238,19 +229,17 @@ requirejs([
         cleanup();
         process.exit(130);
     });
-    process.on('uncaughtException', err => {
+    process.on('uncaughtException', () => {
         log('received "uncaughtException" event')
-        log(err);
         cleanup();
     });
 
     // Request the data from the blob
     prepareCache()
-        .then(prepareInputsOutputs)
         .then(() => Q.all(inputPaths.map(ipath => getData(ipath, inputData[ipath]))))
         .then(() => {
-            // Run 'python main.py' and merge the stdout, stderr
-            job = spawn('python', ['main.py'], {detached: true});
+            // Run 'th init.lua' and merge the stdout, stderr
+            job = spawn('th', ['init.lua'], {detached: true});
             job.stdout.on('data', onStdout);
             job.stderr.on('data', onStderr);
             job.on('close', code => {

src/seeds/nn/version.txt

@@ -1 +1 @@
-1.0.3
+0.5.0

src/seeds/pipeline/version.txt

@@ -1 +1 @@
-0.7.1
+0.6.0

src/visualizers/panels/LayerEditor/LayerEditorControl.js

@@ -15,7 +15,7 @@ define([
 
     'use strict';
 
-    var NO_CODE_MESSAGE = '<%= name %> is not an editable layer!',
+    var NO_CODE_MESSAGE = '-- <%= name %> is not an editable layer!',
         LayerEditorControl;
 
     LayerEditorControl = function (options) {
@@ -45,10 +45,10 @@ define([
                 // Retrieve the template from the mixin
                 template = node.getMixinPaths()
                     .map(id => this._client.getNode(id).getAttribute('code'))
-                    .find(code => !!code) || this.comment(NO_CODE_MESSAGE);
+                    .find(code => !!code) || NO_CODE_MESSAGE;
             }
         } else {
-            template = this.comment(NO_CODE_MESSAGE);
+            template = NO_CODE_MESSAGE;
         }
 
         if (template) {

src/visualizers/panels/OperationCodeEditor/OperationCodeEditorControl.js

@@ -3,17 +3,13 @@
 
 define([
     'panels/TextEditor/TextEditorControl',
-    'text!./boilerplate.ejs',
     'deepforge/viz/OperationControl',
-    'deepforge/OperationCode',
     'deepforge/viz/Execute',
     'deepforge/Constants',
     'underscore'
 ], function (
     TextEditorControl,
-    CodeTemplate,
     OperationControl,
-    OperationCode,
     Execute,
     CONSTANTS,
     _
@@ -22,7 +18,6 @@ define([
     'use strict';
 
     var OperationCodeEditorControl;
-    var GenerateBoilerplate = _.template(CodeTemplate);
 
     OperationCodeEditorControl = function (options) {
         options.attributeName = 'code';
@@ -53,11 +48,6 @@ define([
         desc.inputs = this.getOperationInputs(node).map(id => this.formatIO(id));
         desc.outputs = this.getOperationOutputs(node).map(id => this.formatIO(id));
         desc.references = node.getPointerNames().filter(name => name !== 'base');
-
-        // Create the boilerplate operation code, if applicable
-        if (!desc.ownText) {
-            desc.text = GenerateBoilerplate(desc);
-        }
         return desc;
     };
 
@@ -78,51 +68,6 @@ define([
         }
     };
 
-    OperationCodeEditorControl.prototype.saveTextFor = function (id, code) {
-        try {
-            // Parse the operation implementation and detect change in inputs/outputs
-            // TODO: Update this to use the code object
-            var operation = new OperationCode(code),
-                oldInputs = this.getDataNames(this._currentNodeId, true),
-                currentInputs = operation.getInputs().map(input => input.name),
-                name = this._client.getNode(this._currentNodeId).getAttribute('name'),
-                newInputs,
-                rmInputs,
-                oldOutputs = this.getDataNames(this._currentNodeId),
-                currentOutputs = operation.getOutputs().map(input => input.name),
-                newOutputs,
-                rmOutputs;
-
-            // Check for input nodes to remove
-            if (currentInputs[0] === 'self') currentInputs.shift();
-            newInputs = _.difference(currentInputs, oldInputs);
-            rmInputs = _.difference(oldInputs, currentInputs);
-            newOutputs = _.difference(currentOutputs, oldOutputs);
-            rmOutputs = _.difference(oldOutputs, currentOutputs);
-
-            if (rmInputs.length || newInputs.length || rmOutputs.length || newOutputs.length) {
-                var msg = `Updating operation implementation for ${name}`;
-
-                this._client.startTransaction(msg);
-                TextEditorControl.prototype.saveTextFor.call(this, id, code, true);
-
-                // update the inputs
-                rmInputs.forEach(input => this.removeInputData(this._currentNodeId, input));
-                newInputs.map(input => this.addInputData(this._currentNodeId, input));
-
-                // update the outputs
-                rmOutputs.forEach(output => this.removeOutputData(this._currentNodeId, output));
-                newOutputs.map(output => this.addOutputData(this._currentNodeId, output));
-                this._client.completeTransaction();
-            } else {
-                return TextEditorControl.prototype.saveTextFor.call(this, id, code);
-            }
-        } catch (e) {
-            this._logger.debug(`failed parsing operation: ${e}`);
-            return TextEditorControl.prototype.saveTextFor.call(this, id, code);
-        }
-    };
-
     OperationCodeEditorControl.prototype.getOperationAttributes = function () {
         var node = this._client.getNode(this._currentNodeId),
             attrs = node.getValidAttributeNames(),

src/visualizers/panels/OperationCodeEditor/boilerplate.ejs

@@ -1,4 +0,0 @@
-class <%= name %>Operation():
-
-    def execute(self):
-        # Execute your operation here!

src/visualizers/panels/OperationEditor/OperationEditorPanel.js

@@ -71,33 +71,6 @@ define([
         this.setTitle(name || '');
     };
 
-    OperationEditorPanel.prototype.editTitle = function () {
-        this.$panelHeaderTitle.editInPlace({
-            css: {
-                'z-index': 1000
-            },
-            onChange: (oldValue, newValue) => {
-                var nodeId = this.currentNodeId(),
-                    type = this.currentBaseName(),
-                    words = newValue.split(' '),
-                    msg;
-
-                if (words.length > 1) {
-                    newValue = words.map(word => word[0].toUpperCase() + word.substring(1)).join('');
-                }
-
-                newValue = newValue.replace(/Operation$/, '');
-                msg = `Renamed ${type}: ${oldValue} -> ${newValue}`;
-
-                if (!/^\s*$/.test(newValue)) {
-                    this._client.startTransaction(msg);
-                    this._client.setAttribute(nodeId, 'name', newValue);
-                    this._client.completeTransaction();
-                }
-            }
-        });
-    };
-
     OperationEditorPanel.prototype.getPanels = function () {
         return [InterfaceEditor, CodeEditor];
     };

src/visualizers/panels/OperationInterfaceEditor/OperationInterfaceEditorControl.EventHandlers.js

@@ -1,11 +1,7 @@
 /*globals define*/
 define([
-    'panels/EasyDAG/EasyDAGControl.WidgetEventHandlers',
-    'deepforge/OperationCode',
     './Colors'
 ], function(
-    EasyDAGControlEventHandlers,
-    OperationCode,
     COLORS
 ) {
     'use strict';
@@ -70,14 +66,36 @@ define([
             .filter(node => !node.isAbstract());
     };
 
-    OperationInterfaceEditorEvents.prototype.getValidSuccessors = function(nodeId) {
+    OperationInterfaceEditorEvents.prototype.getValidSuccessors = function(nodeId, isInput) {
+        var dataTypeIds;
+
         if (nodeId !== this._currentNodeId) {
             return [];
         }
 
-        return [{
-            node: this._getObjectDescriptor(this.getDataTypeId())
-        }];
+        // Return all data types in the meta
+        // If input, include abstract types
+        dataTypeIds = this.allDataTypeIds(isInput);
+
+        return dataTypeIds.map(id => {
+            return {
+                node: this._getObjectDescriptor(id)
+            };
+        });
+    };
+
+    OperationInterfaceEditorEvents.prototype._getDataName = function(cntrId, typeId) {
+        var otherIds = this._client.getNode(cntrId).getChildrenIds(),
+            otherNames = otherIds.map(id => this._client.getNode(id).getAttribute('name')),
+            baseName = this._client.getNode(typeId).getAttribute('name').toLowerCase(),
+            name = baseName,
+            i = 1;
+
+        while (otherNames.indexOf(name) !== -1) {
+            i++;
+            name = baseName + '_' + i;
+        }
+        return name;
     };
 
     OperationInterfaceEditorEvents.prototype.getRefName = function(node, basename) {
@@ -175,92 +193,26 @@ define([
         this._client.completeTransaction();
     };
 
-    OperationInterfaceEditorEvents.prototype._createConnectedNode = function(typeId, isInput, baseName) {
+    OperationInterfaceEditorEvents.prototype._createConnectedNode = function(typeId, isInput) {
         var node = this._client.getNode(this._currentNodeId),
             name = node.getAttribute('name'),
-            msg = `Updating the interface of ${name}`,
-            code = node.getAttribute('code'),
-            id,
-            operation,
-            dataName;
-
-        // Update the source code if the inputs/outputs changed
-        // we know that we are adding a node, so we don't need to do 
-        // the comparing and diffing current vs new
-
-        this._client.startTransaction(msg);
-        id = this.createIONode(this._currentNodeId, typeId, isInput, baseName, true);
-        dataName = this._client.getNode(id).getAttribute('name');
-
-        try {
-            operation = new OperationCode(code);
-            if (isInput) {
-                operation.addInput(dataName);
-            } else {
-                operation.addOutput(dataName);
-            }
-
-            this._client.setAttribute(this._currentNodeId, 'code', operation.getCode());
-        } catch(e) {
-            this.logger.debug(`could not update the code - invalid python!: ${e}`);
-        }
-        this._client.completeTransaction();
-
-        return id;
-    };
-
-    OperationInterfaceEditorEvents.prototype._deleteNode = function(nodeId) {
-        var dataName = this._client.getNode(nodeId).getAttribute('name'),
-            node = this._client.getNode(this._currentNodeId),
-            name = node.getAttribute('name'),
-            isInput = this.isInputData(nodeId),
-            msg = `Updating the interface of ${name}`,
-            code = node.getAttribute('code'),
-            operation = new OperationCode(code);
-
-        // If the input name is used in the code, maybe just comment it out in the args
-        this._client.startTransaction(msg);
-        try {
-            if (isInput) {
-                operation.removeInput(dataName);
-            } else {
-                operation.removeOutput(dataName);
-            }
-            this._client.setAttribute(this._currentNodeId, 'code', operation.getCode());
-        } catch(e) {
-            this.logger.debug(`could not update the code - invalid python!: ${e}`);
-        }
-        this._client.deleteNode(nodeId);
-        //EasyDAGControlEventHandlers.prototype._deleteNode.apply(this, nodeId, true);
-        this._client.completeTransaction();
-    };
-
-    OperationInterfaceEditorEvents.prototype._saveAttributeForNode = function(nodeId, attr, value) {
-        // If nodeId is an input data node, rename the input
-        // If nodeId is an output data node, rename the output
-        var isDataNode = nodeId.indexOf(this._currentNodeId) === 0,
-            node = this._client.getNode(this._currentNodeId),
-            code = node.getAttribute('code'),
+            cntrs = node.getChildrenIds(),
+            cntrType = isInput ? 'Inputs' : 'Outputs',
+            cntrId = cntrs.find(id => this.hasMetaName(id, cntrType)),
+            dataName = this._getDataName(cntrId, typeId),
             msg;
 
-        if (isDataNode && attr === 'name') {  // rename input/output
-            var operation = new OperationCode(code),
-                dataNode = this._client.getNode(nodeId),
-                oldName = dataNode.getAttribute(attr);
+        msg = `Adding ${isInput ? 'input' : 'output'} "${dataName}" to ${name} interface`;
+        this._client.startTransaction(msg);
+        var id = this._client.createNode({
+            parentId: cntrId,
+            baseId: typeId
+        });
 
-            operation.rename(oldName, value);
+        // Set the name of the new input
+        this._client.setAttribute(id, 'name', dataName);
 
-            msg = `Renaming ${oldName}->${value} in ${name}`;
-            this._client.startTransaction(msg);
-            EasyDAGControlEventHandlers.prototype._saveAttributeForNode.apply(this, arguments);
-            this._client.setAttribute(this._currentNodeId, 'code', operation.getCode());
-            this._client.completeTransaction();
-        } else if (nodeId === this._currentNodeId) {  // edit operation attributes
-            // TODO: rename operation
-            // TODO: set operation attribute default
-            console.log('setting attr', arguments);
-            EasyDAGControlEventHandlers.prototype._saveAttributeForNode.apply(this, arguments);
-        }
+        this._client.completeTransaction();
     };
 
     return OperationInterfaceEditorEvents;

src/visualizers/panels/OperationInterfaceEditor/OperationInterfaceEditorControl.js

@@ -10,6 +10,7 @@ define([
     'panels/EasyDAG/EasyDAGControl',
     'js/Constants',
     'deepforge/Constants',
+    'deepforge/lua',
     'deepforge/viz/OperationControl',
     './OperationInterfaceEditorControl.EventHandlers',
     './Colors',
@@ -18,6 +19,7 @@ define([
     EasyDAGControl,
     GME_CONSTANTS,
     CONSTANTS,
+    luajs,
     OperationControl,
     OperationInterfaceEditorControlEvents,
     COLORS,
@@ -227,13 +229,20 @@ define([
 
             code = this._client.getNode(this._currentNodeId).getAttribute('code');
             try {
-                // Parse the operation implementation for visual cues
-                // TODO
-                // Parse the operation implementation and detect change in inputs/outputs
-                //var schema = OperationParser.parse(code);
-                //console.log(schema);
+                ast = luajs.parser.parse(code);
+                for (var i = variableIds.length; i--;) {
+                    wasUsed = this._usage[variableIds[i]];
+                    name = this._client.getNode(variableIds[i]).getAttribute('name');
+
+                    isUsed = this._inputs[variableIds[i]] ?
+                        this.isUsedInput(name, ast) :
+                        this.isUsedOutput(name, ast);
+                    if (isUsed !== wasUsed) {
+                        this._onUpdate(variableIds[i]);
+                    }
+                }
             } catch (e) {
-                this._logger.debug(`failed parsing operation: ${e}`);
+                this._logger.debug(`failed parsing lua: ${e}`);
             }
 
         } else if (this.containedInCurrent(gmeId) && this.hasMetaName(gmeId, 'Data')) {
@@ -366,13 +375,11 @@ define([
 
     ////////////////////// Unused input checking //////////////////////
     OperationInterfaceEditorControl.prototype.isUsedInput = function(name, ast) {
-        return true;
-        //return this._isUsed(name, true, ast);
+        return this._isUsed(name, true, ast);
     };
 
     OperationInterfaceEditorControl.prototype.isUsedOutput = function(name, ast) {
-        return true;
-        //return this._isUsed(name, false, ast);
+        return this._isUsed(name, false, ast);
     };
 
     OperationInterfaceEditorControl.prototype._isUsed = function(name, isInput, ast) {
@@ -383,9 +390,8 @@ define([
         // verify that it is not used only in the left side of an assignment
         if (hasText) {
             try {
-                return true;
-                //ast = ast || luajs.parser.parse(code);
-                //return isInput ? this.isUsedVariable(name, ast) : this.isReturnValue(name, ast);
+                ast = ast || luajs.parser.parse(code);
+                return isInput ? this.isUsedVariable(name, ast) : this.isReturnValue(name, ast);
             } catch(e) {
                 this._logger.debug(`failed parsing lua: ${e}`);
                 return null;
@@ -400,20 +406,19 @@ define([
         var isUsed = false,
             checker;
 
-        return true;
-        //checker = luajs.codegen.traverse((curr, parent) => {
-            //if (curr.type === 'variable' && curr.val === name) {
-                //// Ignore if it is being assigned...
-                //if (parent.type === 'stat.assignment') {
-                    //isUsed = isUsed || parent.right.indexOf(curr) !== -1;
-                //} else {
-                    //isUsed = true;
-                //}
-            //}
-            //return curr;
-        //});
+        checker = luajs.codegen.traverse((curr, parent) => {
+            if (curr.type === 'variable' && curr.val === name) {
+                // Ignore if it is being assigned...
+                if (parent.type === 'stat.assignment') {
+                    isUsed = isUsed || parent.right.indexOf(curr) !== -1;
+                } else {
+                    isUsed = true;
+                }
+            }
+            return curr;
+        });
 
-        //checker(node);
+        checker(node);
         return isUsed;
     };
 

src/visualizers/panels/OperationInterfaceEditor/OperationInterfaceEditorPanel.js

@@ -4,17 +4,14 @@
  * Generated by VisualizerGenerator 1.7.0 from webgme on Tue May 31 2016 09:16:24 GMT-0500 (CDT).
  */
 
-define([
-    'js/PanelBase/PanelBaseWithHeader',
+define(['js/PanelBase/PanelBaseWithHeader',
     'js/PanelManager/IActivePanel',
     'widgets/OperationInterfaceEditor/OperationInterfaceEditorWidget',
     './OperationInterfaceEditorControl'
-], function (
-    PanelBaseWithHeader,
-    IActivePanel,
-    OperationInterfaceEditorWidget,
-    OperationInterfaceEditorControl
-) {
+], function (PanelBaseWithHeader,
+             IActivePanel,
+             OperationInterfaceEditorWidget,
+             OperationInterfaceEditorControl) {
     'use strict';
 
     var OperationInterfaceEditorPanel;

src/visualizers/widgets/ClassCodeEditor/ClassCodeEditorWidget.js

@@ -19,7 +19,9 @@ define([
     _.extend(ClassCodeEditorWidget.prototype, TextEditorWidget.prototype);
 
     ClassCodeEditorWidget.prototype.getHeader = function(desc) {
-        return this.comment(`The class definition for ${desc.name}`);
+        return [
+            `-- The class definition for ${desc.name}`
+        ].join('\n');
     };
 
     ClassCodeEditorWidget.prototype.updateNode = function() {

src/visualizers/widgets/DeserializeEditor/DeserializeEditorWidget.js

@@ -26,13 +26,13 @@ define([
 
     DeserializeEditorWidget.prototype.getHeader = function(desc) {
         this._name = desc.name;
-        return this.comment([
-            `The deserialization function for ${desc.name}`,
-            'Globals:',
-            '  `path` - target filename to load',
-            '',
-            `return the loaded ${desc.name}`
-        ].join('\n'));
+        return [
+            `-- The deserialization function for ${desc.name}`,
+            '-- Globals:',
+            '--   `path` - target filename to load',
+            '--',
+            `-- return the loaded ${desc.name}`
+        ].join('\n');
     };
 
     DeserializeEditorWidget.prototype.getNameRegex = function() {

src/visualizers/widgets/OperationCodeEditor/OperationCodeEditorWidget.js

@@ -32,24 +32,24 @@ define([
 
     OperationCodeEditorWidget.prototype.getHeader = function (desc) {
         // Add comment about the inputs, attributes and references
-        var inputs = desc.inputs.map(pair => `${pair[0]} (${pair[1]})`).join('\n'),
-            refs = desc.references.map(name => `${name}`).join('\n'),
+        var inputs = desc.inputs.map(pair => `-- ${pair[0]} (${pair[1]})`).join('\n'),
+            refs = desc.references.map(name => `-- ${name}`).join('\n'),
             header = [
-                `Editing "${desc.name}" Implementation`
+                `-- Editing "${desc.name}" Implementation`
             ];
 
         if (inputs.length) {
-            header.push('');
-            header.push('Defined variables:');
+            header.push('--');
+            header.push('-- Defined variables:');
             header.push(inputs);
         }
         if (refs) {
             header.push(refs);
         }
-        header.push('');
-        header.push('The \'execute\' method will be called when the operation is run');
+        header.push('--');
+        header.push('-- The following will be executed when the operation is run:');
 
-        return this.comment(header.join('\n'));
+        return header.join('\n');
     };
 
     OperationCodeEditorWidget.prototype.canAddReturnTmpl = function (desc) {

src/visualizers/widgets/OperationInterfaceEditor/OperationInterfaceEditorWidget.js

@@ -61,8 +61,31 @@ define([
     };
 
     OperationInterfaceEditorWidget.prototype.onAddButtonClicked = function(item, isInput) {
-        var successorPairs = this.getValidSuccessors(item.id, isInput);
-        return this.onAddItemSelected(successorPairs[0], isInput);
+        var successorPairs = this.getValidSuccessors(item.id, isInput),
+            newClass = this.getCreationNode('Complex', NEW_CLASS_ID),
+            newPrim = this.getCreationNode('Primitive', NEW_PRIM_ID),
+            opts = {};
+
+        // Add the 'Create Class' node
+        successorPairs.push(newClass);
+        successorPairs.push(newPrim);
+
+        // Add tabs
+        opts.tabs = ['Primitive', 'Classes'];
+        opts.tabFilter = (tab, pair) => {
+            return pair.node.isPrimitive === (tab === 'Primitive');
+        };
+
+        AddNodeDialog.prompt(successorPairs, opts)
+            .then(selected => {
+                if (selected.node.id === NEW_CLASS_ID) {
+                    DeepForge.create.Complex();
+                } else if (selected.node.id === NEW_PRIM_ID) {
+                    DeepForge.create.Primitive();
+                } else {
+                    this.onAddItemSelected(selected, isInput);
+                }
+            });
     };
 
     OperationInterfaceEditorWidget.prototype.onDeactivate = function() {
@@ -121,7 +144,8 @@ define([
 
     // Hover buttons
     OperationInterfaceEditorWidget.prototype.showHoverButtons = function(item) {
-        var refNodes = this.allValidReferences(),
+        var dataNodes = this.allDataTypeIds(),
+            refNodes = this.allValidReferences(),
             height = item.height,
             cx = item.width/2;
 
@@ -137,6 +161,7 @@ define([
             new Buttons.AddOutput({  // Add output data
                 context: this,
                 $pEl: this.$hoverBtns,
+                disabled: dataNodes.length === 0,
                 item: item,
                 x: cx,
                 y: height
@@ -145,6 +170,7 @@ define([
             new Buttons.AddInput({  // Add input data
                 context: this,
                 $pEl: this.$hoverBtns,
+                disabled: dataNodes.length === 0,
                 item: item,
                 x: item.width/3,
                 y: 0

src/visualizers/widgets/OperationInterfaceEditor/SelectionManager.js

@@ -1,4 +1,4 @@
-/*globals define, $*/
+/*globals define*/
 
 define([
     'widgets/EasyDAG/SelectionManager',
@@ -25,10 +25,12 @@ define([
 
     SelectionManager.prototype.createActionButtons = function(width, height) {
         var selectedType = this.selectedItem.desc.baseName,
+            dataNodes,
             refNodes,
             cx = width/2;
 
         if (selectedType === 'Operation') {
+            dataNodes = this._widget.allDataTypeIds();
             refNodes = this._widget.allValidReferences();
 
             new Buttons.AddOutput({  // Add output data
@@ -36,13 +38,15 @@ define([
                 $pEl: this.$selection,
                 item: this.selectedItem,
                 x: cx,
-                y: height
+                y: height,
+                disabled: dataNodes.length === 0
             });
 
             new Buttons.AddInput({  // Add input data
                 context: this._widget,
                 $pEl: this.$selection,
                 item: this.selectedItem,
+                disabled: dataNodes.length === 0,
                 x: width/3,
                 y: 0
             });

src/visualizers/widgets/SerializeEditor/SerializeEditorWidget.js

@@ -23,12 +23,12 @@ define([
 
     SerializeEditorWidget.prototype.getHeader = function(desc) {
         this._name = desc.name;
-        return this.comment([
-            `The serialization function for ${desc.name}`,
-            'Globals:',
-            '  `path` - target filename',
-            `  \`data\` - the ${desc.name} to store`
-        ].join('\n'));
+        return [
+            `-- The serialization function for ${desc.name}`,
+            '-- Globals:',
+            '--   `path` - target filename',
+            `--   \`data\` - the ${desc.name} to store`
+        ].join('\n');
     };
 
     SerializeEditorWidget.prototype.getNameRegex = function () {

src/visualizers/widgets/TextEditor/TextEditorWidget.js

@@ -18,10 +18,6 @@ define([
 
     var TextEditorWidget,
         WIDGET_CLASS = 'text-editor',
-        LINE_COMMENT = {
-            python: '#',
-            lua: '--'
-        },
         DEFAULT_SETTINGS = {
             keybindings: 'default',
             theme: 'solarized_dark',
@@ -31,7 +27,6 @@ define([
     TextEditorWidget = function (logger, container) {
         this._logger = logger.fork('Widget');
 
-        this.language = this.language || 'python';
         this._el = container;
         this._el.css({height: '100%'});
         this.$editor = $('<div/>');
@@ -91,8 +86,8 @@ define([
 
     TextEditorWidget.prototype.getSessionOptions = function () {
         return {
-            mode: 'ace/mode/' + this.language,
-            tabSize: 4,
+            mode: 'ace/mode/lua',
+            tabSize: 3,
             useSoftTabs: true
         };
     };
@@ -225,33 +220,18 @@ define([
     };
 
     // Adding/Removing/Updating items
-    TextEditorWidget.prototype.comment = function (text) {
-        var prefix = LINE_COMMENT[this.language] + ' ';
-        return text.replace(
-            new RegExp('^(' + LINE_COMMENT[this.language] + ')?','mg'),
-            prefix
-        );
-    };
-
     TextEditorWidget.prototype.getHeader = function (desc) {
-        return this.comment(`Editing "${desc.name}"`);
+        return `-- Editing "${desc.name}"`;
     };
 
     TextEditorWidget.prototype.addNode = function (desc) {
         // Set the current text based on the given
         // Create the header
-        var header = this.getHeader(desc),
-            content,
-            newContent = header + '\n' + desc.text,
-            //patches = diff.diffChars(content, newContent),
-            cursorPos;
+        var header = this.getHeader(desc);
 
-        // TODO: if we are updating the value, we should make sure the cursor position
-        // remains in the same spot (ie, diff the text and update the positions
-        // based on the size of the patches
         this.activeNode = desc.id;
         this.silent = true;
-        this.editor.setValue(newContent, 2);
+        this.editor.setValue(header + '\n' + desc.text, 2);
         this.silent = false;
         this.currentHeader = header;
     };

test/common/OperationCode.spec.js

@@ -1,353 +0,0 @@
-describe.only('OperationCode', function() {
-    var fs = require('fs');
-    var path = require('path');
-    var assert = require('assert');
-    var OperationCode = require('../../src/common/OperationCode');
-    var operation;
-
-    describe('example', function() {
-        var code;
-
-        before(function() {
-            // load the example
-            var filePath = path.join(__dirname, '..', 'test-cases', 'operations', 'example.py');
-            code = fs.readFileSync(filePath, 'utf8');
-        });
-
-        describe('removeInput', function() {
-            before(function() {
-                operation = new OperationCode(code);
-                operation.removeInput('world');
-            });
-
-            it('should have 2 remaining inputs', function() {
-                assert.equal(operation.getInputs().length, 2);
-            });
-
-        });
-
-        describe('attributes', function() {
-            describe('add', function() {
-                beforeEach(function() {
-                    operation = new OperationCode(code);
-                });
-
-                it('should add argument to __init__ method', function() {
-                    operation.addAttribute('number');
-                    var attrs = operation.getAttributes();
-                    // TODO
-                });
-
-                it('should set the default value', function() {
-                    // TODO
-                });
-            });
-
-            // TODO: add attribute
-            // TODO: remove attribute
-            // TODO: rename attribute?
-        });
-
-        describe('rename', function() {
-            before(function() {
-                operation = new OperationCode(code);
-                operation.rename('hello', 'goodbye');
-            });
-
-            it('should rename input arg', function() {
-                var inputs = operation.getInputs();
-                var oldInput = inputs.find(input => input.name === 'hello');
-                var newInput = inputs.find(input => input.name === 'goodbye');
-
-                assert(!oldInput);
-                assert(newInput);
-            });
-
-            it('should rename occurrences in the fn', function() {
-                assert(!operation.getCode().includes('hello'));
-            });
-        });
-
-        describe('parsing', function() {
-            before(function() {
-                operation = new OperationCode(code);
-            });
-
-            it('should parse the correct name', function() {
-                assert.equal(operation.getName(), 'ExampleOperation');
-            });
-
-            it.skip('should parse the correct base', function() {
-                assert.equal(operation.getBase(), 'Operation');
-            });
-
-            it('should parse the input names', function() {
-                const names = ['hello', 'world', 'count'];
-                assert.deepEqual(operation.getInputs().map(input => input.name), names);
-            });
-
-            it.skip('should parse the input types', function() {
-                const types = ['str', 'str', 'int'];
-                assert.deepEqual(operation.getInputs().map(input => input.type), types);
-            });
-
-            it('should parse the output names', function() {
-                const names = ['concat', 'count'];
-                assert.deepEqual(operation.getOutputs().map(output => output.name), names);
-            });
-
-            it.skip('should parse the output types', function() {
-                const types = ['str', 'int'];
-                assert.deepEqual(operation.getOutputs().map(output => output.type), types);
-            });
-        });
-    });
-
-    describe('multi-anon-results', function() {
-        before(function() {
-            var filePath = path.join(__dirname, '..', 'test-cases', 'operations', 'multi-anon-results.py');
-            var example = fs.readFileSync(filePath, 'utf8');
-            operation = new OperationCode(example);
-        });
-
-        it('should parse multiple return values', function() {
-            assert.equal(operation.getOutputs().length, 2);
-        });
-
-        it('should create unique names for each', function() {
-            var [first, second] = operation.getOutputs();
-            assert.notEqual(first.name, second.name);
-        });
-    });
-
-    describe('no-inputs/outputs', function() {
-        var code;
-
-        before(function() {
-            var filePath = path.join(__dirname, '..', 'test-cases', 'operations', 'no-inputs.py');
-            code = fs.readFileSync(filePath, 'utf8');
-        });
-
-        describe('parsing', function() {
-            beforeEach(function() {
-                operation = new OperationCode(code);
-            });
-
-            it('should not require base class', function() {
-                assert.equal(operation.getBase(), null);
-            });
-
-            it('should detect zero output', function() {
-                assert.equal(operation.getOutputs().length, 0);
-            });
-
-            it('should detect zero inputs', function() {
-                assert.equal(operation.getInputs().length, 0);
-            });
-        });
-
-        describe('addInput', function() {
-            var operation;
-
-            before(function() {
-                operation = new OperationCode(code);
-                operation.addInput('first');
-            });
-
-            it('should clear schema', function() {
-                assert(!operation._schema);
-            });
-
-            it('should add input to `execute` fn', function() {
-                var code = operation.getCode();
-                assert(code.includes('first'));
-            });
-
-            it('should have an additional input arg', function() {
-                var inputs = operation.getInputs();
-                assert.equal(inputs.length, 1);
-            });
-        });
-
-        describe('addOutput', function() {
-            var operation;
-
-            describe('lone return', function() {
-                before(function() {
-                    operation = new OperationCode(code);
-                    operation.addOutput('myNewOutput');
-                });
-
-                it('should clear schema', function() {
-                    assert(!operation._schema);
-                });
-
-                it('should add input to `execute` fn', function() {
-                    var code = operation.getCode();
-                    assert(code.includes('myNewOutput'));
-                });
-
-                it('should have an additional input arg', function() {
-                    var inputs = operation.getOutputs();
-                    assert.equal(inputs.length, 1);
-                });
-            });
-
-            describe('no return', function() {
-                before(function() {
-                    operation = new OperationCode(code);
-                    operation.addReturnValue('no_return', 'myNewOutput');
-                });
-
-                it('should clear schema', function() {
-                    assert(!operation._schema);
-                });
-
-                it('should add input to `execute` fn', function() {
-                    var code = operation.getCode();
-                    assert(code.includes('myNewOutput'));
-                });
-
-                it('should have an additional input arg', function() {
-                    var outputs = operation.getReturnValues('no_return');
-                    assert.equal(outputs.length, 1);
-                });
-            });
-        });
-
-    });
-
-    describe('simple', function() {
-        var code;
-
-        before(function() {
-            var filePath = path.join(__dirname, '..', 'test-cases', 'operations', 'simple.py');
-            code = fs.readFileSync(filePath, 'utf8');
-        });
-
-        describe('parsing', function() {
-            beforeEach(function() {
-                operation = new OperationCode(code);
-            });
-
-            it('should not require base class', function() {
-                assert.equal(operation.getBase(), null);
-            });
-
-            it('should detect one output', function() {
-                assert.equal(operation.getOutputs().length, 1);
-            });
-
-            it('should provide the value', function() {
-                assert.equal(operation.getOutputs()[0].value, '20');
-            });
-
-            it('should detect two inputs', function() {
-                assert.equal(operation.getInputs().length, 2);
-            });
-        });
-
-        describe('addInput', function() {
-            var operation;
-
-            before(function() {
-                operation = new OperationCode(code);
-                operation.addInput('myNewInput');
-            });
-
-            it('should clear schema', function() {
-                assert(!operation._schema);
-            });
-
-            it('should add input to `execute` fn', function() {
-                var code = operation.getCode();
-                assert(code.includes('myNewInput'));
-            });
-
-            it('should have an additional input arg', function() {
-                var inputs = operation.getInputs();
-                assert.equal(inputs.length, 3);
-            });
-        });
-
-        describe('addOutput', function() {
-            var operation;
-
-            before(function() {
-                operation = new OperationCode(code);
-                operation.addOutput('myNewOutput');
-            });
-
-            it('should clear schema', function() {
-                assert(!operation._schema);
-            });
-
-            it('should add input to `execute` fn', function() {
-                var code = operation.getCode();
-                assert(code.includes('myNewOutput'));
-            });
-
-            it('should have an additional input arg', function() {
-                var inputs = operation.getOutputs();
-                assert.equal(inputs.length, 2);
-            });
-        });
-
-        describe('removeInput', function() {
-            var operation,
-                result;
-
-            beforeEach(function() {
-                operation = new OperationCode(code);
-                result = operation.removeInput('number');
-            });
-
-            it('should return removed arg', function() {
-                assert.equal(result.name, 'number');
-            });
-
-            it('should only have one remaining argument', function() {
-                assert.equal(operation.getInputs().length, 1);
-            });
-
-            it('should only not have removed argument', function() {
-                assert(!operation.getCode().includes('number'));
-            });
-
-            it('should return null if arg doesn\'t exist', function() {
-                var result = operation.removeInput('numdasfber');
-                assert.equal(result, null);
-            });
-
-        });
-
-        describe('removeOutput', function() {
-            var operation,
-                result;
-
-            beforeEach(function() {
-                operation = new OperationCode(code);
-                result = operation.removeOutput('result');
-            });
-
-            it('should return removed arg', function() {
-                assert.equal(result.name, 'result');
-            });
-
-            it('should have no remaining results', function() {
-                assert.equal(operation.getOutputs().length, 0);
-            });
-
-            it('should only not have removed argument', function() {
-                assert(!operation.getCode().includes('20'));
-            });
-
-            it('should return null if arg doesn\'t exist', function() {
-                var result = operation.removeOutput('numdasfber');
-                assert.equal(result, null);
-            });
-
-        });
-    });
-
-});

test/plugins/GenerateJob/GenerateJob.spec.js

@@ -118,7 +118,7 @@ describe('GenerateJob', function () {
 
     describe('exec files', function() {
         describe('attribute file', function() {
-            var boolString = /['"](True|False)['"]/g;
+            var boolString = /['"](true|false)['"]/g;
 
             beforeEach(preparePlugin);
 
@@ -129,38 +129,11 @@ describe('GenerateJob', function () {
 
                 plugin.setAttribute(node, 'debug', 'true');
                 plugin.createAttributeFile(node, files);
-                content = files['attributes.py'];
+                content = files['attributes.lua'];
                 matches = content.match(boolString);
                 expect(matches).to.equal(null);
             });
 
-            describe('boolean to python boolean', function() {
-
-                it('should convert true to True', function() {
-                    var files = {},
-                        content,
-                        matches;
-
-                    plugin.setAttribute(node, 'debug', 'true');
-                    plugin.createAttributeFile(node, files);
-                    content = files['attributes.py'];
-                    matches = content.includes('True');
-                    expect(matches).to.equal(true);
-                });
-
-                it('should convert false to False', function() {
-                    var files = {},
-                        content,
-                        matches;
-
-                    plugin.setAttribute(node, 'debug', 'False');
-                    plugin.createAttributeFile(node, files);
-                    content = files['attributes.py'];
-                    matches = content.includes('False');
-                    expect(matches).to.equal(true);
-                });
-            });
-
             it('should not quote true boolean values', function() {
                 var files = {},
                     content,
@@ -168,7 +141,7 @@ describe('GenerateJob', function () {
 
                 plugin.setAttribute(node, 'debug', true);
                 plugin.createAttributeFile(node, files);
-                content = files['attributes.py'];
+                content = files['attributes.lua'];
                 matches = content.match(boolString);
                 expect(matches).to.equal(null);
             });
@@ -180,7 +153,7 @@ describe('GenerateJob', function () {
 
                 plugin.setAttribute(node, 'debug', 'false');
                 plugin.createAttributeFile(node, files);
-                content = files['attributes.py'];
+                content = files['attributes.lua'];
                 matches = content.match(boolString);
                 expect(matches).to.equal(null);
             });
@@ -192,14 +165,11 @@ describe('GenerateJob', function () {
 
                 plugin.setAttribute(node, 'debug', false);
                 plugin.createAttributeFile(node, files);
-                content = files['attributes.py'];
+                content = files['attributes.lua'];
                 matches = content.match(boolString);
                 expect(matches).to.equal(null);
             });
         });
     });
 
-    // TODO: What else should I test?
-    // run a hello world example (use golem?)
- 
 });

test/test-cases/operations/example.py

@@ -1,14 +0,0 @@
-from operations import Operation
-from typing import Tuple
-
-class ExampleOperation(Operation):
-
-    # TODO: add the type hints
-    def execute(hello, world, count):
-        # Doing things
-        concat = hello + world
-        return concat, count
-
-    def other_method(a, b, c):
-        # Doing things
-        return a+b/c

test/test-cases/operations/multi-anon-results.py

@@ -1,10 +0,0 @@
-from operations import Operation
-from typing import Tuple
-
-class ExampleOperation(Operation):
-
-    # TODO: add the type hints
-    def execute(hello, world, count):
-        # Doing things
-        concat = hello + world
-        return concat+1, count-9

test/test-cases/operations/no-inputs.py

@@ -1,6 +0,0 @@
-class ExampleOperation():
-    def execute():
-        return
-
-    def no_return():
-        print('hello')

test/test-cases/operations/simple.py

@@ -1,5 +0,0 @@
-class TrainOperation():
-
-    # TODO: add type hints
-    def execute(self, number):
-        return 20

utils/categories.json

@@ -1,53 +1,142 @@
 {
-    "Transfer": [
-        "PReLU",
-        "Softshrink",
-        "Softplus",
-        "LeakyReLU",
-        "Hardshrink",
-        "ELU",
-        "ReLU6",
-        "Hardtanh",
-        "RReLU",
-        "ReLU",
-        "Threshold"
-    ],
+   "RNN": [
+      "BiSequencer",
+      "BiSequencerLM",
+      "GRU",
+      "MaskZero",
+      "MaskZeroCriterion",
+      "Recurrence",
+      "Recurrent",
+      "RecurrentAttention",
+      "Recursor",
+      "Repeater",
+      "RepeaterCriterion",
+      "Sequencer",
+      "SequencerCriterion",
+      "TrimZero",
 
-    "Convolution": [
-        "ConvTranspose3d",
-        "Conv3d",
-        "ConvTranspose2d",
-        "Conv2d",
-        "Conv1d",
+      "CopyGrad",
+      "FastLSTM",
+      "LSTM",
+      "LookupTableMaskZero",
+      "NormStabilizer",
+      "SAdd",
+      "SeqBRNN",
+      "SeqGRU",
+      "SeqLSTM",
+      "SeqLSTMP",
+      "SeqReverseSequence"
+   ],
+   "Convolution": [
+      "TemporalConvolution",
+      "TemporalMaxPooling",
+      "TemporalSubSampling",
+      "LookupTable",
+      "SpatialConvolutionMM",
+      "SpatialConvolution",
+      "SpatialConvolutionMap",
+      "SpatialFullConvolutionMap",
+      "SpatialLPPooling",
+      "SpatialMaxPooling",
+      "SpatialAveragePooling",
+      "SpatialAdaptiveMaxPooling",
+      "SpatialSubSampling",
+      "SpatialUpSamplingNearest",
+      "SpatialZeroPadding",
+      "SpatialReflectionPadding",
+      "SpatialReplicationPadding",
+      "SpatialSubtractiveNormalization",
+      "SpatialCrossMapLRN",
+      "SpatialConvolutionLocal",
+      "SpatialDropout",
+      "SpatialDilatedConvolution",
+      "SpatialFractionalMaxPooling",
+      "SpatialDivisiveNormalization",
+      "SpatialContrastiveNormalization",
+      "SpatialBatchNormalization",
+      "SpatialFullConvolution",
+      "SpatialMaxUnpooling",
+      "VolumetricConvolution",
+      "VolumetricMaxPooling",
+      "VolumetricAveragePooling",
+      "VolumetricBatchNormalization",
+      "VolumetricDropout",
 
-        "FractionalMaxPool2d",
-        "LPPool2d",
-        "MaxUnpool3d",
-        "AvgPool3d",
-        "MaxPool3d",
-        "AvgPool2d",
-        "MaxUnpool2d",
-        "MaxPool2d",
-        "MaxPool1d",
-
-        "ReplicationPad3d",
-        "ReplicationPad2d",
-        "ReflectionPad2d"
-    ],
-
-    "Simple": [
-        "Dropout3d",
-        "Dropout2d",
-        "Dropout",
-        "Linear",
-
-        "Embedding"
-    ],
-
-    "Criterion": [
-        "MultiMarginLoss",
-        "MarginRankingLoss",
-        "CosineEmbeddingLoss",
-        "CrossMapLRN2d"
-    ]
+      "VolumetricFullConvolution",
+      "VolumetricMaxUnpooling"
+   ],
+   "Criterion": [
+      "BCECriterion",
+      "WeightedMSECriterion",
+      "SmoothL1Criterion",
+      "MSECriterion",
+      "AbsCriterion",
+      "MultiCriterion",
+      "DistKLDivCriterion",
+      "HingeEmbeddingCriterion",
+      "CriterionTable",
+      "MultiMarginCriterion",
+      "MultiLabelMarginCriterion",
+      "L1HingeEmbeddingCriterion",
+      "CosineEmbeddingCriterion",
+      "MarginRankingCriterion",
+      "CrossEntropyCriterion",
+      "MarginCriterion",
+      "ClassNLLCriterion",
+      "ParallelCriterion",
+      "SpatialClassNLLCriterion",
+      "SoftMarginCriterion",
+      "MultiLabelSoftMarginCriterion"
+   ],
+   "Simple": [
+      "Linear",
+      "LinearNoBias",
+      "SparseLinear",
+      "Dropout",
+      "Concat",
+      "Abs",
+      "Add",
+      "Mul",
+      "CMul",
+      "Max",
+      "Min",
+      "Mean",
+      "Sum",
+      "Euclidean",
+      "WeightedEuclidean",
+      "Identity",
+      "Copy",
+      "Narrow",
+      "Replicate",
+      "Reshape",
+      "View",
+      "Select",
+      "Exp",
+      "Square",
+      "Sqrt",
+      "Power",
+      "MM",
+      "AddConstant",
+      "MulConstant"
+   ],
+   "Transfer": [
+      "Threshold",
+      "HardTanh",
+      "HardShrink",
+      "SoftShrink",
+      "SoftMax",
+      "SpatialSoftMax",
+      "SoftMin",
+      "SoftPlus",
+      "SoftSign",
+      "LogSigmoid",
+      "LogSoftMax",
+      "Sigmoid",
+      "Tanh",
+      "ReLU",
+      "ReLU6",
+      "PReLU",
+      "RReLU",
+      "LeakyReLU"
+   ]
 }

utils/nn-parser.js

@@ -28,10 +28,7 @@ if (exists.sync(configPath)) {  // Check the deepforge config
     config = JSON.parse(fs.readFileSync(configPath, 'utf8'));
     torchPath = (config.torch && config.torch.dir) || (configDir + 'torch');
 }
-// FIXME: Get the pytorch root path...
-torchPath = process.env.HOME + '/projects/pytorch';
-// check 'modules', 'parallel'
-torchPath += '/torch/nn/';
+torchPath += `/install/share/lua/5.1/${outputName}/`;
 
 console.log(`parsing ${outputName} from ${torchPath}`);
 
@@ -54,18 +51,14 @@ var lookupType = function(layer){
     return layerType || 'Misc';
 };
 
-var parseLayerFiles = function(layerDir) {
-    var files = fs.readdirSync(layerDir),
-        layers,
+fs.readdir(torchPath, function(err,files){
+    if(err) throw err;
+    var layers,
         layerByName = {};
 
-    console.log('parsing', layerDir);
-    layers = files.filter(filename => path.extname(filename) === '.py' &&
-            filename[0] !== '_')
-        .map(filename => fs.readFileSync(layerDir + filename, 'utf8'))
+    layers = files.filter(filename => path.extname(filename) === '.lua')
+        .map(filename => fs.readFileSync(torchPath + filename, 'utf8'))
         .map(code => LayerParser.parse(code))
-        .filter(list => list !== null)
-        .reduce((l1, l2) => l1.concat(l2), [])
         .filter(layer => !!layer && layer.name);
 
     layers.forEach(layer => {
@@ -111,27 +104,17 @@ var parseLayerFiles = function(layerDir) {
         }
     });
     layers = layers.filter(layer => !SKIP_LAYERS[layer.name]);
-    return layers;
-};
 
-var layers = ['modules']
-    .map(dir => torchPath + dir + '/')
-    .map(path => parseLayerFiles(path))
-    .reduce((l1, l2) => l1.concat(l2))
-    .filter(layer => layer.name[0] !== '_');  // skip hidden/abstract layers
+    outputDst += outputName + '.json';
+    // eslint-disable-next-line no-console
+    console.log('Saved nn interface to ' + outputDst);
+    fs.writeFileSync(outputDst, JSON.stringify(layers, null, 2));
 
+    // Update the CreateTorchMeta index
+    var updateSchemas = `${__dirname}/../src/plugins/CreateTorchMeta/update-schemas.js`,
+        job = require('child_process').fork(updateSchemas);
 
-// eslint-disable-next-line no-console
-console.log('discovered', layers.length, 'layers');
-outputDst += outputName + '.json';
-// eslint-disable-next-line no-console
-console.log('Saved nn interface to ' + outputDst);
-fs.writeFileSync(outputDst, JSON.stringify(layers, null, 2));
-
-// Update the CreateTorchMeta index
-var updateSchemas = `${__dirname}/../src/plugins/CreateTorchMeta/update-schemas.js`,
-    job = require('child_process').fork(updateSchemas);
-
-job.on('close', code => {
-    process.exit(code);
+    job.on('close', code => {
+        process.exit(code);
+    });
 });

utils/pyparser.js

@@ -1,104 +0,0 @@
-var brython = require('./node-brython'),
-    fs = require('fs'),
-    assert = require('assert'),
-    src = fs.readFileSync(process.env.HOME + '/projects/pytorch/torch/nn/modules/conv.py', 'utf8'),
-    root = build_ast(src);
-
-function build_ast(src) {
-  brython.$py_module_path['__main__']='./'
-  return brython.py2js(src,'__main__', '__main__', '__builtins__')
-}
-
-// The provided tree gives us contexts which can have associated 'C'
-function traverse (node, fn) {
-    var i;
-    if (node.children) {
-        for (i = node.children.length; i--;) {
-            traverse(node.children[i], fn);
-            fn(node.children[i]);
-        }
-    }
-    if (node.C && node.C.tree) {
-        for (i = node.C.tree.length; i--;) {
-            traverse(node.C.tree[i], fn);
-            fn(node.C.tree[i]);
-        }
-    }
-}
-
-var types = {},
-    layers = [],
-    pCtx,
-    classNode,
-    params;
-
-function isClass(node) {
-    return node.type === 'class';
-}
-
-function isInitFn(node) {
-    return node.type === 'def' && node.name === '__init__';
-}
-
-function getBaseClass(node) {
-    assert(node.type === 'class');
-    return node.args.tree[0].tree[0].tree[0].value;
-}
-
-var defaults = {},
-    defTypes,
-    args,
-    def;
-
-traverse(root, node => {
-    if (node.type) types[node.type] = true;
-
-    // Get the class for the given function
-    if (isInitFn(node)) {
-        pCtx = node.parent.node.parent;
-        classNode = pCtx.C.tree[0];
-
-        if (isClass(classNode)) {
-            // remove the 'self' variable
-            // TODO: May need to update this for kwargs
-            // (use positional_list)
-            args = node.tree[1].tree;
-            defaults = {};
-            params = node.args.slice(1);
-            defTypes = {};
-            for (var i = args.length; i--;) {
-                if (args[i].tree[0]) {
-                    def = args[i].tree[0].tree[0];
-                    console.log('setting type of ', params[i-1], 'to', def.type);
-                    defTypes[params[i-1]] = def.type;
-                    if (def.type === 'int') {
-                        defaults[params[i-1]] = parseInt.apply(null, def.value.reverse());
-                    } else {
-                        defaults[params[i-1]] = def.value;
-                    }
-                }
-            }
-            layers.push({
-                name: classNode.name,
-                baseType: getBaseClass(classNode),
-                //doc: classNode.doc_string || '',
-                defaults: defaults,
-                types: defTypes,
-                params: params
-            });
-        }
-    }
-    // TODO: What if there is no constructor? Is this a potential problem?
-});
-
-console.log('layers:', layers);
-fs.writeFileSync('./testPyTorchLayers.json', JSON.stringify(layers, null, 2));
-//console.log('layers:', layers.map(l => l.name));
-// Try to find the class definitions...
-//
-//  Need to create:
-//
-//    setters: (I don't think these are used in pytorch!
-//    types: 
-//    defaults: 
-//    type: 

webgme-setup.json

@@ -327,9 +327,6 @@
       },
       "devProject": {
         "src": "src/seeds/devProject"
-      },
-      "minimal": {
-        "src": "src/seeds/minimal"
       }
     },
     "routers": {