WIP #1024 fixed the code update on new data added

and support for return values if no current ones

.dockerignore

@@ -0,0 +1 @@
+/node_modules

Dockerfile

@@ -0,0 +1,24 @@
+# Dockerfile for running the server itself
+FROM node:6.10.1
+MAINTAINER Brian Broll <brian.broll@gmail.com>
+
+RUN echo '{"allow_root": true}' > /root/.bowerrc && mkdir -p /root/.config/configstore/ && \
+    echo '{}' > /root/.config/configstore/bower-github.json
+
+RUN mkdir /deepforge
+ADD . /deepforge
+WORKDIR /deepforge
+
+RUN cd $(npm root -g)/npm \
+    && npm install fs-extra \
+    && sed -i -e s/graceful-fs/fs-extra/ -e s/fs.rename/fs.move/ ./lib/utils/rename.js
+
+RUN ln -s /deepforge/bin/deepforge /usr/local/bin
+
+EXPOSE 8888
+
+# Set up the data storage
+RUN deepforge config blob.dir /data/blob && \
+    deepforge config mongo.dir /data/db
+
+CMD ["deepforge", "start", "--server"]

Dockerfile.worker

@@ -0,0 +1,65 @@
+# This has torch and cuda support
+FROM kaixhin/cuda-torch
+MAINTAINER Brian Broll <brian.broll@gmail.com>
+
+# install nodejs v6
+RUN groupadd --gid 1000 node \
+  && useradd --uid 1000 --gid node --shell /bin/bash --create-home node
+
+# gpg keys listed at https://github.com/nodejs/node#release-team
+RUN set -ex \
+  && for key in \
+    9554F04D7259F04124DE6B476D5A82AC7E37093B \
+    94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \
+    FD3A5288F042B6850C66B31F09FE44734EB7990E \
+    71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \
+    DD8F2338BAE7501E3DD5AC78C273792F7D83545D \
+    B9AE9905FFD7803F25714661B63B535A4C206CA9 \
+    C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \
+    56730D5401028683275BD23C23EFEFE93C4CFFFE \
+  ; do \
+    gpg --keyserver ha.pool.sks-keyservers.net --recv-keys "$key"; \
+  done
+
+ENV NPM_CONFIG_LOGLEVEL info
+ENV NODE_VERSION 6.10.1
+
+RUN curl -SLO "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-x64.tar.xz" \
+  && curl -SLO "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \
+  && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \
+  && grep " node-v$NODE_VERSION-linux-x64.tar.xz\$" SHASUMS256.txt | sha256sum -c - \
+  && tar -xJf "node-v$NODE_VERSION-linux-x64.tar.xz" -C /usr/local --strip-components=1 \
+  && rm "node-v$NODE_VERSION-linux-x64.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \
+  && ln -s /usr/local/bin/node /usr/local/bin/nodejs
+
+# install deepforge
+RUN echo '{"allow_root": true}' > /root/.bowerrc && mkdir -p /root/.config/configstore/ && \
+    echo '{}' > /root/.config/configstore/bower-github.json
+
+RUN mkdir /deepforge
+ADD . /deepforge
+WORKDIR /deepforge
+
+RUN cd $(npm root -g)/npm \
+    && npm install fs-extra \
+    && sed -i -e s/graceful-fs/fs-extra/ -e s/fs.rename/fs.move/ ./lib/utils/rename.js
+
+RUN ln -s /deepforge/bin/deepforge /usr/local/bin
+
+# configure the worker
+RUN deepforge config blob.dir /data/blob && \
+    deepforge config mongo.dir /data/db && \
+    deepforge config worker.cache.useBlob false && \
+    deepforge config worker.cache.dir /deepforge/worker-cache && \
+    deepforge config torch.dir /root/torch/ && \
+    git config --global user.email "deepforge-worker@deepforge.org" && \
+    git config --global user.name "deepforge-worker"
+
+# Update torch
+RUN apt-get update && apt-get install sudo wget && \
+    . /root/torch/install/bin/torch-activate && \
+    cd /root/torch/ && bash /root/torch/update.sh && \
+    deepforge update -t
+
+ENTRYPOINT ["deepforge", "start", "--worker"]
+CMD ["http://172.17.0.1:8888"]

README.md

@@ -4,6 +4,8 @@
 [![Join the chat at https://gitter.im/deepforge-dev/deepforge](https://badges.gitter.im/deepforge-dev/deepforge.svg)](https://gitter.im/deepforge-dev/deepforge?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 [![Stories in Ready](https://badge.waffle.io/deepforge-dev/deepforge.png?label=ready&title=Ready)](https://waffle.io/deepforge-dev/deepforge)
 
+Using DeepForge? [Let us know what you think!](https://goo.gl/forms/2pDdCPXoUvkQhVzQ2)
+
 # DeepForge
 DeepForge is an open-source visual development environment for deep learning providing end-to-end support for creating deep learning models. This is achieved through providing the ability to design **architectures**, create training **pipelines**, and then execute these pipelines over a cluster. Using a notebook-esque api, users can get real-time feedback about the status of any of their **executions** including compare them side-by-side in real-time.
 
@@ -46,7 +48,13 @@ Also, be sure to check out the other available features of the `deepforge` cli;
 - [Datamodel Developer Slides](https://docs.google.com/presentation/d/1hd3IyUlzW_TIPnzCnE-1pdz00Pw8WaIxYiOW_Hyog-M/edit#slide=id.p)
 
 ## Interested in contributing?
-Contributions are welcome! Either fork the project and submit some PR's or shoot me an email about getting more involved! If you have any questions, check out the [wiki](https://github.com/dfst/deepforge/wiki/) or drop me a line on the gitter!
+Contributions are welcome! There are a couple different ways to contribute to DeepForge:
+- Provide user feedback!
+    - on the [documentation](http://deepforge.readthedocs.io)
+    - on deepforge and its future development: https://goo.gl/forms/2pDdCPXoUvkQhVzQ2
+- Contribute to the project directly by submitting some PR's!
+
+If you have any questions, check out the [wiki](https://github.com/dfst/deepforge/wiki/) or drop me a line on the gitter!
 
 
 Sponsored by [Digital Reasoning](http://www.digitalreasoning.com/)

bin/deepforge

@@ -185,6 +185,7 @@ var startMongo = function(args, port, silent) {
 };
 
 var hasTorch = function() {
+    return true;
     var result = childProcess.spawnSync('th', ['--help']);
     return !result.error;
 };

bin/start-worker.js

@@ -36,17 +36,16 @@ try {
 } catch (e) {
     // Create dir
     childProcess.spawnSync('ln', ['-s', `${__dirname}/../node_modules`, modules]);
-    return true;
 }
 
 // Check torch support
-var result = childProcess.spawnSync('th', ['--help']);
-if (result.error) {
-    console.error('Checking Torch7 dependency failed. Do you have Torch7 installed ' + 
-        'and in your PATH?\n\nFor Torch7 installation instructions, check out ' +
-        'http://torch.ch/docs/getting-started.html');
-    process.exit(1);
-}
+//var result = childProcess.spawnSync('th', ['--help']);
+//if (result.error) {
+    //console.error('Checking Torch7 dependency failed. Do you have Torch7 installed ' + 
+        //'and in your PATH?\n\nFor Torch7 installation instructions, check out ' +
+        //'http://torch.ch/docs/getting-started.html');
+    //process.exit(1);
+//}
 
 var cleanUp = function() {
     console.log('removing worker directory ', workerPath);

config/config.base.js

@@ -8,6 +8,7 @@ require('dotenv').load({silent: true});
 
 // Add/overwrite any additional settings here
 config.server.port = +process.env.PORT || config.server.port;
+config.server.timeout = 0;
 config.mongo.uri = process.env.MONGO_URI || config.mongo.uri;
 config.blob.fsDir = process.env.DEEPFORGE_BLOB_DIR || config.blob.fsDir;
 

config/config.test.js

@@ -9,7 +9,8 @@ var config = require('./config.default'),
     path = require('path');
 
 config.server.port = 9001;
-config.mongo.uri = 'mongodb://127.0.0.1:27017/webgme_tests';
+config.mongo.uri = process.env.MONGO_URI || 'mongodb://127.0.0.1:27017';
+config.mongo.uri = config.mongo.uri.replace(/\/[a-zA-Z_\-]*$/,  '') + '/deepforge_tests';
 config.blob.fsDir = path.join(__dirname, '..', 'test-tmp', 'blob');
 
 module.exports = config;

config/config.webgme.js

@@ -22,6 +22,7 @@ config.seedProjects.basePaths.push(__dirname + '/../src/seeds/project');
 config.seedProjects.basePaths.push(__dirname + '/../src/seeds/cifar10');
 config.seedProjects.basePaths.push(__dirname + '/../src/seeds/xor');
 config.seedProjects.basePaths.push(__dirname + '/../src/seeds/devProject');
+config.seedProjects.basePaths.push(__dirname + '/../src/seeds/minimal');
 
 
 

docs/.gitignore

@@ -0,0 +1 @@
+_build

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = deepforge
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/conf.py

@@ -0,0 +1,159 @@
+import sphinx_rtd_theme
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# deepforge documentation build configuration file, created by
+# sphinx-quickstart on Mon Mar 13 18:56:27 2017.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = []
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = 'DeepForge'
+copyright = '2017, Brian Broll'
+author = 'Brian Broll'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = ''
+# The full version, including alpha/beta/rc tags.
+release = ''
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = False
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_rtd_theme'
+html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+
+# -- Options for HTMLHelp output ------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'deepforgedoc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'deepforge.tex', 'deepforge Documentation',
+     'Brian Broll', 'manual'),
+]
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'deepforge', 'deepforge Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'deepforge', 'deepforge Documentation',
+     author, 'deepforge', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+
+

docs/deployment/dockerized.rst

@@ -0,0 +1,45 @@
+Dockerized Installation
+-----------------------
+Each of the components are also available as docker containers. This page outlines the running of each of the main components as docker containers and connecting them as necessary.
+
+Database
+~~~~~~~~
+First, you can start the mongo container using:
+
+.. code-block:: bash
+
+    docker run -d -v /abs/path/to/data:/data/db mongo
+
+where :code:`/abs/path/to/data` is the path to the mongo data location on the host. If running the database in a container, you will need to get the ip address of the given container:
+
+.. code-block:: bash
+
+    docker inspect <container id>  | grep IPAddr
+
+The :code:`<container id>` is the value returned from the original :code:`docker run` command.
+
+When running mongo in a docker container, it is important to mount an external volume (using the :code:`-v` flag) to be used for the actual data (otherwise the data will be lost when the container is stopped).
+
+Server
+~~~~~~
+The DeepForge server can be started with
+
+.. code-block:: bash
+
+    docker run -d -v $HOME/.deepforge/blob:/data/blob \
+    -p 8888:8888 -e MONGO_URI=mongodb://172.17.0.2:27017/deepforge \
+    deepforge/server
+
+where :code:`172.17.0.2` is the ip address of the mongo container and :code:`$HOME/.deepforge/blob` is the path to use for binary DeepForge data on the host. Of course, if the mongo instance is locating at a different location, :code:`MONGO_URI` can be set to this address as well. Also, the first port (:code:`8888`) can be replaced with the desired port to expose on the host.
+
+Worker
+~~~~~~
+As workers may require GPU access, they will need to use the nvidia-docker plugin. Workers can be created using
+
+.. code-block:: bash
+
+    nvidia-docker run -d deepforge/worker http://172.17.0.1:8888
+
+where :code:`http://172.17.0.1:8888` is the location of the DeepForge server to which to connect.
+
+**Note**: The :code:`deepforge/worker` image is packaged with cuda 7.5. Depending upon your hardware and nvidia version, you may need to build your own docker image or run the worker natively.

docs/deployment/native.rst

@@ -0,0 +1,106 @@
+Native Installation
+===================
+
+Database
+~~~~~~~~
+Download and install MongoDB from the `website <https://www.mongodb.org/>`_. If you are planning on running MongoDB locally on the same machine as DeepForge, simply start `mongod` and continue to setting up DeepForge.
+
+If you are planning on running MongoDB remotely, set the environment variable "MONGO_URI" to the URI of the Mongo instance that DeepForge will be using:
+
+.. code-block:: bash
+
+    MONGO_URI="mongodb://pathToMyMongo.com:27017/myCollection" deepforge start
+
+Server
+~~~~~~
+The DeepForge server is included with the deepforge cli and can be started simply with 
+
+.. code-block:: bash
+
+    deepforge start --server
+
+By default, DeepForge will start on `http://localhost:8888`. However, the port can be specified with the `--port` option. For example:
+
+.. code-block:: bash
+
+    deepforge start --server --port 3000
+
+Worker
+~~~~~~
+The DeepForge worker can be started with
+
+.. code-block:: bash
+
+    deepforge start --worker
+
+The worker will install dependencies the first time it is run (including torch, if it is not already installed). 
+
+To connect to a remote deepforge instance, add the url of the DeepForge server:
+
+.. code-block:: bash
+
+    deepforge start --worker http://myaddress.com:1234
+
+Updating
+~~~~~~~~
+DeepForge can be updated with the command line interface rather simply:
+
+.. code-block:: bash
+
+    deepforge update
+
+By default, this will update both DeepForge and the local torch installation. To only update DeepForge, add the `--server` flag:
+
+.. code-block:: bash
+
+    deepforge update --server
+
+For more update options, check out `deepforge update --help`!
+
+Manual Installation (Development)
+---------------------------------
+Installing DeepForge for development is essentially cloning the repository and then using `npm` (node package manager) to run the various start, test, etc, commands (including starting the individual components). The deepforge cli can still be used but must be referenced from `./bin/deepforge`. That is, `deepforge start` becomes `./bin/deepforge start` (from the project root).
+
+DeepForge Server
+~~~~~~~~~~~~~~~~
+First, clone the repository:
+
+.. code-block:: bash
+
+    git clone https://github.com/dfst/deepforge.git
+
+Then install the project dependencies:
+
+.. code-block:: bash
+
+    npm install
+
+To run all components locally start with 
+
+.. code-block:: bash
+
+    ./bin/deepforge start
+
+and navigate to `http://localhost:8888` to start using DeepForge!
+
+Alternatively, if jobs are going to be executed on an external worker, run `./bin/deepforge start -s` locally and navigate to `http://localhost:8888`.
+
+DeepForge Worker
+~~~~~~~~~~~~~~~~
+If you are using `./bin/deepforge start -s` you will need to set up a DeepForge worker (`./bin/deepforge start` starts a local worker for you!). DeepForge workers are slave machines connected to DeepForge which execute the provided jobs. This allows the jobs to access the GPU, etc, and provides a number of benefits over trying to perform deep learning tasks in the browser.
+
+Once DeepForge is installed on the worker, start it with
+
+.. code-block:: bash
+
+    ./bin/deepforge start -w
+
+Note: If you are running the worker on a different machine, put the address of the DeepForge server as an argument to the command. For example:
+
+.. code-block:: bash
+
+    ./bin/deepforge start -w http://myaddress.com:1234
+
+Updating
+~~~~~~~~
+Updating can be done the same as any other git project; that is, by running `git pull` from the project root. Sometimes, the dependencies need to be updated so it is recommended to run `npm install` following `git pull`.

docs/deployment/overview.rst

@@ -0,0 +1,26 @@
+Overview
+========
+
+DeepForge Component Overview
+----------------------------
+DeepForge is composed of four main elements:
+
+- *Server*: Main component hosting all the project information and is connected to by the clients
+- *Database*: MongoDB database containing DeepForge, job queue for the workers, etc
+- *Worker*: Slave machine performing the actual machine learning computation
+- *Client*: The connected browsers working on DeepForge projects.
+
+Of course, only the *Server*, *Database* (MongoDB) and *Worker* need to be installed. If you are not going to execute any machine learning pipelines, installing the *Worker* can be skipped.
+
+Component Dependencies
+----------------------
+The following dependencies are required for each component:
+
+- *Server* (NodeJS v6.2.1)
+- *Database* (MongoDB v3.0.7)
+- *Worker*: NodeJS v6.2.1 (used for job management logic) and `Torch <http://torch.ch/docs/getting-started.html#>`_ (this will be installed automatically by the cli when needed)
+- *Client*: We recommend using Google Chrome and are not supporting other browsers (for now). In other words, other browsers can be used at your own risk.
+
+Configuration
+-------------
+After installing DeepForge, it can be helpful to check out `configuring DeepForge <getting_started/configuration.rst>`_

docs/fundamentals/custom_data_types.rst

@@ -0,0 +1,13 @@
+Custom Data Types
+=================
+
+As operation inputs and outputs are strongly typed, DeepForge supports the creation of custom data types to promote flexibility when designing complex pipelines and operations. DeepForge data types can be either primitive types or custom classes. Custom DeepForge primitive types are relatively straight-forward; they can inherit from other types and must implement a serialization and deserialization methods (which may be as simple as :code:`torch.save` and :code:`torch.load`). Custom classes are also relatively simple to define but actually contain their own methods along with serialization and deserialization functions.
+
+New data types can be defined from the operation editor from the dialog for selecting input or output data for the operation. After defining a new class, this class is available from within any of the operations in the DeepForge project.
+
+.. figure:: model_data_editor.png
+    :align: center
+    :scale: 55 %
+
+    Editing the serialization and deserialization for the "model" type
+

docs/fundamentals/custom_layers.rst

@@ -0,0 +1,87 @@
+Custom Layers
+=============
+
+DeepForge supports the creation of custom neural network layers using Torch7 and the easy usage of these layers in the visual architecture editor. Before creating custom layers, it is recommended to read about `creating custom layers in Torch7 <http://torch.ch/docs/developer-docs.html>`_.
+
+A new custom layer can be created from the "add layer dialog" in the architecture editor. When creating a layer, DeepForge provides a code editor for creating custom neural network layers prepopulated with a basic template for defining the custom layer.
+
+After defining the layer in the layer editor, DeepForge will provide this layer in the architecture editor and expose any configurable attributes for the layer. These attributes are parsed from the layer definition.
+
+Best Practices
+--------------
+Here are a couple best practices to keep in mind when defining custom neural network layers:
+
+-  Use type assertions for layer, boolean attributes
+
+-  Return :code:`self` when defining setter functions
+
+**Type assertions** should be used when defining layer attributes (ie, constructor arguments or arguments to a setter function). For example, consider the following layer definition for :code:`RecurrentAttention` which accepts an :code:`action` layer argument to its constructor.
+
+.. code-block:: lua
+
+    local RecurrentAttention, parent = torch.class("nn.RecurrentAttention", "nn.AbstractSequencer")
+
+    function RecurrentAttention:__init(rnn, action, nStep, hiddenSize)
+       parent.__init(self)
+       assert(torch.isTypeOf(action, 'nn.Module'))
+       assert(torch.type(nStep) == 'number')
+       assert(torch.type(hiddenSize) == 'table')
+       assert(torch.type(hiddenSize[1]) == 'number', "Does not support table hidden layers" )
+       
+       self.rnn = rnn
+       -- we can decorate the module with a Recursor to make it AbstractRecurrent
+       self.rnn = (not torch.isTypeOf(rnn, 'nn.AbstractRecurrent')) and nn.Recursor(rnn) or rnn
+       
+       -- samples an x,y actions for each example
+       self.action =  (not torch.isTypeOf(action, 'nn.AbstractRecurrent')) and nn.Recursor(action) or action 
+       self.hiddenSize = hiddenSize
+       self.nStep = nStep
+       
+       self.modules = {self.rnn, self.action}
+       
+       self.output = {} -- rnn output
+       self.actions = {} -- action output
+       
+       self.forwardActions = false
+       
+       self.gradHidden = {}
+    end
+
+In this example, :code:`assert(torch.isTypeOf(action, 'nn.Module'))` enforces that the :code:`action` variable is another neural network layer. After defining the layer, DeepForge will parse the layer definition and create a visual representation for use in the architecture editor. As this assertion enforces that :code:`action` is a neural network layer, DeepForge will update itself accordingly; in this case, editing the attribute will allow the user to hierarchically create nested neural network architectures to be passed as the :code:`action` argument to the constructor.
+
+.. figure:: recurrent_attention.png
+    :align: center
+    :scale: 85 %
+
+    RecurrentAttention has attributes for each of the constructor arguments
+
+An example of the generated visual model for the :code:`RecurrentAttention` is provided above. This layer has attributes for each of the constructor arguments defined in its definition. Clicking on the :code:`<none>` value for the :code:`action` attribute will then allow the user to provide layer inputs as shown below.
+
+.. figure:: action_layer.png
+    :align: center
+    :scale: 55 %
+
+    Creating layer inputs for the "action" variable
+
+The second best practice is to make sure to **return self in any setter functions**. An example of this can be found in the setters in the :code:`SpatialMaxPooling` layer shown below:
+
+.. code-block:: lua
+
+    function SpatialMaxPooling:ceil()
+      self.ceil_mode = true
+      return self
+    end
+
+    function SpatialMaxPooling:floor()
+      self.ceil_mode = false
+      return self
+    end
+
+Returning :code:`self` in setter functions is a good convention when defining neural network layers in Torch7 as it promotes simple and legible code such as
+
+.. code-block:: lua
+
+    net:add(nn.SpatialMaxPooling(5, 5, 2, 2):ceil())
+
+where :code:`net` is a container like a :code:`Sequential` layer. DeepForge enforces this convention and, if it finds a setter function (which also returns :code:`self`) in the layer definition will expose the internal variable (in this case :code:`ceil_mode`) to the user in the visual editor.
+

docs/fundamentals/custom_operations.rst

@@ -0,0 +1,69 @@
+Custom Operations
+=================
+
+In this document we will outline the basics of custom operations including the operation editor and operation feedback utilities.
+
+The Basics
+----------
+Operations are used in pipelines and have named, typed inputs and outputs. When creating a pipeline, if you don't currently find an operation for the given task, you can easily create your own by selecting the `New Operation...` operation from the add operation dialog. This will create a new operation definition and open it in the operation editor. The operation editor has two main parts, the interface editor and the implementation editor.
+
+.. figure:: operation_editor.png
+    :align: center
+    :scale: 45 %
+
+    Editing the "train" operation provided in the "First Steps" section
+
+The interface editor is provided on the left and presents the interface as a diagram showing the input data and output data as objects flowing into or out of the given operation. Selecting the operation node in the operation interface editor will expand the node and allow the user to add or edit attributes for the given operation. These attributes are exposed when using this operation in a pipeline and can be set at design time - that is, these are set when creating the given pipeline. The interface diagram may also contain light blue nodes flowing into the operation. These nodes represent "references" that the operation accepts as input before running. When using the operation, references will appear alongside the attributes but will allow the user to select from a list of all possible targets when clicked.
+
+.. figure:: operation_interface.png
+    :align: center
+    :scale: 85 %
+
+    The train operation accepts training data, an architecture and criterion and returns a trained model
+
+On the right of the operation editor is the implementation editor. The implementation editor is a code editor specially tailored for programming the implementations of operations in DeepForge. This includes some autocomplete support for common globals in this context like the :code:`deepforge` and :code:`torch` globals. It also is synchronized with the interface editor and will provide input to the interface editor about unused variables, etc. These errors will present themselves as error or warning highlights on the data in the interface editor. A section of the implementation is shown below:
+
+.. code:: lua
+
+	trainer = nn.StochasticGradient(net, criterion)
+	trainer.learningRate = attributes.learningRate
+	trainer.maxIteration = attributes.maxIterations
+
+	print('training for ' .. tostring(attributes.maxIterations) .. ' iterations (max)')
+	print('learning rate is ' .. tostring(attributes.learningRate))
+	print(trainer)
+
+	-- Adding the error graph
+	graph = deepforge.Graph('Training Error') -- creating graph feedback
+	errLine = graph:line('error')
+	trainer.hookIteration = function(t, iter, currentErr)
+	   errLine:add(iter, currentErr) -- reporting the current error (will update in real time in DeepForge)
+	end
+
+	trainer:train(trainset)
+
+	return {
+	   net = net
+	}
+
+The "train" operation uses the :code:`StochasticGradient` functionality from the :code:`nn` package to perform stochastic gradient descent. This operation sets all the parameters using values provided to the operation as either attributes or references. In the implementation, attributes are provided by the :code:`attributes` variable and provides access to the user defined attributes from within the implementation. References are treated similarly to operation inputs and are defined in variables of the same name. This can be seen with the :code:`net` and :code:`criterion` variables in the first line. Finally, operations return a table of their named outputs; in this example, it returns a single output named :code:`net`, that is, the trained neural network.
+
+After defining the interface and implementation, we can now use the "train" operation in our pipelines! An example is shown below.
+
+.. figure:: train_operation.png
+    :align: center
+    :scale: 85 %
+
+    Using the custom "train" operation in a pipeline
+
+Operation feedback
+------------------
+Operations in DeepForge can generate metadata about its execution. This metadata is generated during the execution and provided back to the user in real-time. An example of this includes providing real-time plotting feedback of the loss function of a model while training. When implementing an operation in DeepForge, this metadata can be created using the :code:`deepforge` global.
+
+.. figure:: graph_example.png
+    :align: center
+    :scale: 75 %
+
+    An example graph of the loss function while training a neural network
+
+Detailed information about the available operation metadata types can be found in the `reference <reference/feedback_mechanisms.rst>`_.

docs/getting_started/getting_started.rst

@@ -0,0 +1,32 @@
+Getting Started
+===============
+
+.. _Torch: http://torch.ch
+
+What is DeepForge?
+------------------
+Deep learning is a very promising, yet complex, area of machine learning. This complexity can both create a barrier to entry for those wanting to get involved in deep learning as well as slow the development of those already comfortable in deep learning.
+
+DeepForge is a development environment for deep learning focused on alleviating these problems. Leveraging the flexibility of Torch_, DeepForge is able to reduce the complexity of using deep learning while still providing advanced features such as defining custom layers.
+
+Design Goals
+------------
+As mentioned above, DeepForge focuses on two main goals:
+
+1. **Improving the efficiency** of experienced data scientists/researchers in deep learning
+2. **Lowering the barrier to entry** for newcomers to deep learning
+
+It is important to highlight that although one of the goals is focused on lowering the barrier to entry, DeepForge is intended to be more than simply an educational tool; that is, it is important not to compromise on flexibility and effectiveness as a research/industry tool in order to provide an easier experience for beginners (that's what forks are for!).
+
+Overview and Features
+---------------------
+DeepForge provides a collaborative, distributed development environment for deep learning. The development environment is a hybrid visual and textual programming environment. Higher levels of abstraction, such as creating architectures, use visual environments to capture the overall structure of the task while lower levels of abstraction, such as defining custom layers, utilize text environments to maintain the flexibility provided by torch.
+
+Concepts and Terminology
+~~~~~~~~~~~~~~~~~~~~~~~~
+- *Architecture* - neural network architecture composed of torch defined layers
+- *Operation* - essentially a function written in torch (such as `SGD`)
+- *Pipeline* - directed acyclic graph composed of operations
+  - eg, a training pipeline may retrieve and normalize data, train an architecture and return the trained model
+- *Execution* - when a pipeline is run, an "execution" is created and reports the status of each operation as it is run (distributed over a number of worker machines)
+- *Artifact* - an artifact represents some data (either user uploaded or created during an execution)

docs/getting_started/hello_cifar.rst

@@ -0,0 +1,63 @@
+First Steps
+===========
+DeepForge provides an example project for creating a classifier using the `CIFAR10 <https://www.kaggle.com/c/cifar-10>`_ dataset.
+
+When first opening DeepForge in your browser (at `http://localhost:8888` if following the instructions from the `quick start <getting_started/installation.rst>`_), you will be prompted with a list of projects to open and provided the option to create a new project. For this example, let's click "Create new..." and name our project "hello_cifar".
+
+.. figure:: create_project.png
+    :align: center
+    :scale: 65 %
+
+    Creating our "hello_cifar" example project
+
+Clicking "Create" will bring us to a prompt for the "seed" for our project. Select "cifar10" from the dropdown and click "Create". This will now create our new project based on the cifar10 example provided with DeepForge.
+
+.. figure:: set_seed.png
+    :align: center
+    :scale: 75 %
+
+    Selecting the "cifar10" example seed
+
+In this example, we have three main pipelines: :code:`download-normalize`, :code:`train` and :code:`test`. :code:`download-normalize` downloads and prepares our data. The :code:`train` pipeline trains a neural network model on the cifar10 dataset and the :code:`test` pipeline tests our trained model on our test set from the cifar10 dataset.
+
+.. figure:: pipelines.png
+    :align: center
+    :scale: 65 %
+
+    Three main pipelines in the cifar10 example project
+
+First, we will have to retrieve and prepare the data by running the :code:`download-normalize` pipeline. This can be done by opening the given pipeline then selecting the `Execute Pipeline` option from the action button in the lower right. As soon as that pipeline finishes, we can now use this data to train a neural network.
+
+Next, we can open the :code:`train` pipeline. Before we execute the pipeline we have to set the input trainning data that we will be using. This is done by selecting the :code:`Input` operation then clicking the value for the :code:`artifact` field. This will provide all the possible options for the input data; for this example, we will want to select the "trainingdata" artifact. After setting the input, we can click on the :code:`train` operation to inspect the hyperparameters we are using and the architecture we are training. Selecting the :code:`Output` operation will allow you to change the name of the resulting artifact of this operation (in this case, a trained model). Finally, we can execute this pipeline like before to train the model.
+
+.. figure:: select_train_data.png
+    :align: center
+    :scale: 65 %
+
+    Selecting the training data for the input to the training pipeline
+
+As this operation trains, we can view the status by viewing the running execution. The easiest way to view the running execution is by clicking the given execution from the execution tray in the bottom left when viewing the originating pipeline.
+
+.. figure:: training_execution.png
+    :align: center
+    :scale: 65 %
+
+    Viewing the execution of the training pipeline
+
+Once the model has been trained, we can test the given model using the :code:`test` pipeline. In this pipeline, we have a few more inputs to set: "testing data", "model to test" and the "human-readable class labels". If you aren't clear which operation provides which input, you can simply hover over it's connected port on the :code:`test` operation. This will provide a tooltip with the full name of the input.
+
+.. figure:: test_pipeline.png
+    :align: center
+    :scale: 65 %
+
+    Viewing the execution of the testing pipeline
+
+After setting the inputs for the :code:`test` pipeline (using the trained model and data from the first two pipelines), we can simply execute this pipeline to test our model. After executing the :code:`test` pipeline, we can view the execution and open the :code:`test` job to view the stdout for the given job. In the :code:`test` operation, this will allow us to view the printed accuracies of the model over each class.
+
+.. figure:: test_results.png
+    :align: center
+    :scale: 65 %
+
+    Viewing the results of the testing operation
+
+And that's it! We have just trained and tested our first neural network model using DeepForge. Although there are still a lot more advanced features that can be used, this should at least familiarize us with some of the core concepts in DeepForge.

docs/getting_started/quick_start.rst

@@ -0,0 +1,29 @@
+Quick Start
+===========
+Before we can start with the examples, we will first install DeepForge locally.
+
+Dependencies
+------------
+First, install `NodeJS <https://nodejs.org/en/>`_ (v6) and `MongoDB <https://www.mongodb.org/>`_. You may also need to install git if you haven't already.
+
+Next, you can install DeepForge using npm:
+
+.. code-block:: bash
+
+    npm install -g deepforge
+
+Now, you can check that it installed correctly:
+
+.. code-block:: bash
+
+    deepforge --version
+
+DeepForge can now be started with:
+
+.. code-block:: bash
+
+    deepforge start
+
+However, the first time DeepForge is started, it will make sure that the deep learning framework is installed (if it isn't found on the host system). This may require you to start DeepForge a couple times; the first time it starts it will install Torch7 and require a terminal restart to update a couple environment variables (like `PATH`). The second time it starts it will install additional torch packages but will not require a terminal restart. Finally, DeepForge will start with all the required dependencies.
+
+For detailed instructions about deployment installations, check out our `deployment installation instructions <getting_started/configuration.rst>`_

docs/index.rst

@@ -0,0 +1,40 @@
+.. DeepForge documentation master file, created by
+   sphinx-quickstart on Mon Mar 13 18:56:27 2017.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to DeepForge's documentation!
+=====================================
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Getting Started
+
+   getting_started/getting_started.rst
+   getting_started/quick_start.rst
+   getting_started/hello_cifar.rst
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Fundamentals
+
+   fundamentals/custom_operations.rst
+   fundamentals/custom_layers.rst
+   fundamentals/custom_data_types.rst
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Deployment
+
+   deployment/overview.rst
+   deployment/native.rst
+   deployment/dockerized.rst
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Reference
+
+   reference/cli.rst
+   reference/configuration.rst
+   reference/operation_feedback.rst
+   reference/extensions.rst

docs/reference/cli.rst

@@ -0,0 +1,91 @@
+Command Line Interface
+======================
+
+This document outlines the functionality of the deepforge command line interface (provided after installing deepforge with :code:`npm install -g deepforge`).
+
+- Installation Configuration
+- Starting DeepForge or Components
+- Installing and Upgrading Torch
+- Update or Uninstall DeepForge
+- Managing Extensions
+
+Installation Configuration
+--------------------------
+Installation configuration including the installation location of Torch7 and data storage locations. These can be edited using the :code:`deepforge config` command as shown in the following examples:
+
+Printing all the configuration settings:
+
+.. code-block:: bash
+
+    deepforge config
+
+
+Printing the value of a configuration setting:
+
+.. code-block:: bash
+
+    deepforge config torch.dir
+
+
+Setting a configuration option, such as :code:`torch.dir` can be done with:
+
+.. code-block:: bash
+
+    deepforge config torch.dir /some/new/directory
+
+
+For more information about the configuration settings, check out the `configuration <configuration.rst>`_ page.
+
+
+Starting DeepForge Components
+-----------------------------
+DeepForge components, such as the server or the workers, can be started with the :code:`deepforge start` command. By default, this command will start all the necessary components to run including the server, a mongo database (if applicable) and a worker.
+
+The server can be started by itself using
+
+.. code-block:: bash
+
+    deepforge start --server
+
+
+The worker can be started by itself using
+
+.. code-block:: bash
+
+    deepforge start --worker http://154.95.87.1:7543
+
+
+where `http://154.95.87.1:7543` is the url of the deepforge server.
+
+Installing and Upgrading Torch7
+-------------------------------
+Torch7 is lazily installed when starting a worker (if torch isn't already installed) with the rnn package. This installation can be manually updated as described in the update and installation section.
+
+Update/Uninstall DeepForge
+--------------------------
+DeepForge can be updated or uninstalled using
+
+.. code-block:: bash
+
+    deepforge update
+
+
+The torch installation can be updated using
+
+.. code-block:: bash
+
+    deepforge update --torch
+
+
+DeepForge can be uninstalled using :code:`deepforge uninstall`
+
+Managing Extensions
+-------------------
+DeepForge extensions can be installed and removed using the :code:`deepforge extensions` subcommand. Extensions can be added, removed and listed as shown below
+
+.. code-block:: bash
+
+    deepforge extensions add https://github.com/example/some-extension
+    deepforge extensions remove some-extension
+    deepforge extensions list
+

docs/reference/configuration.rst

@@ -0,0 +1,77 @@
+Configuration
+=============
+
+Configuration of deepforge is done through the `deepforge config` command from the command line interface. To see all config options, simply run `deepforge config` with no additional arguments. This will print a JSON representation of the configuration settings similar to:
+
+.. code-block:: bash
+
+    Current config:
+    {
+      "torch": {
+        "dir": "/home/irishninja/.deepforge/torch"
+      },
+      "blob": {
+        "dir": "/home/irishninja/.deepforge/blob"
+      },
+      "worker": {
+        "cache": {
+          "useBlob": true,
+          "dir": "~/.deepforge/worker/cache"
+        },
+        "dir": "~/.deepforge/worker"
+      },
+      "mongo": {
+        "dir": "~/.deepforge/data"
+      }
+    }
+
+Setting an attribute, say `worker.cache.dir`, is done as follows
+
+.. code-block:: bash
+
+    deepforge config worker.cache.dir /tmp
+
+Environment Variables
+---------------------
+Most settings have a corresponding environment variable which can be used to override the value set in the cli's configuration. This allows the values to be temporarily set for a single run. For example, starting a worker with a different cache than set in `worker.cache.dir` can be done with:
+
+.. code-block:: bash
+
+    DEEPFORGE_WORKER_CACHE=/tmp deepforge start -w
+
+The complete list of the environment variable overrides for the configuration options can be found `here <https://github.com/deepforge-dev/deepforge/blob/master/bin/envConfig.json>`_.
+
+Settings
+--------
+
+torch.dir
+~~~~~~~~~
+The path to the local installation of torch to be used by the deepforge worker. This is used when installing, upgrading and removing the local torch installation
+
+blob.dir
+~~~~~~~~
+The path to the blob (large file storage containing models, datasets, etc) to be used by the deepforge server.
+
+This can be overridden with the `DEEPFORGE_BLOB_DIR` environment variable.
+
+worker.dir
+~~~~~~~~~~
+The path to the directory used for worker executions. The workers will run the executions from this directory.
+
+This can be overridden with the `DEEPFORGE_WORKER_DIR` environment variable.
+
+mongo.dir
+~~~~~~~~~
+The path to use for the `--dbpath` option of mongo if starting mongo using the command line interface. That is, if the MONGO_URI is set to a local uri and the cli is starting the deepforge server, the cli will check to verify that an instance of mongo is running locally. If not, it will start it on the given port and use this setting for the `--dbpath` setting of mongod.
+
+worker.cache.dir
+~~~~~~~~~~~~~~~~
+The path to the worker cache directory.
+
+This can be overridden with the `DEEPFORGE_WORKER_CACHE` environment variable.
+
+worker.cache.useBlob
+~~~~~~~~~~~~~~~~~~~~
+When running the worker on the same machine as the server, this allows the worker to use the blob as a cache and simply create symbolic links to the data (eg, training data, models) to prevent having to even perform a copy of the data on the given machine.
+
+This can be overridden with the `DEEPFORGE_WORKER_USE_BLOB` environment variable.

docs/reference/operation_feedback.rst

@@ -0,0 +1,53 @@
+Operation Feedback
+==================
+
+DeepForge provides the `deepforge` global object in operation implementations for providing feedback during the execution. The various types of metadata are provided and discussed below.
+
+Graphs
+------
+Real-time graphs can be created using the graph constructor:
+
+.. code-block:: lua
+
+    local graph = deepforge.Graph('My Graph')  -- created a new graph called "My Graph"
+
+After creating a graph, lines can be added similarly.
+
+.. code-block:: lua
+
+    local line1 = graph:line('first line')  -- created a new line called "first line"
+    local line2 = graph:line('second line')  -- created a second line called "second line"
+
+Finally, points can be added to the lines by calling the `:add` method on the line and passing the x and y values for the given point.
+
+.. code-block:: lua
+
+    line1:add(1, 3)  -- adding point (1, 3) to line1
+    line2:add(1, 4)  -- adding point (1, 4) to line2
+
+    line1:add(2, 5)  -- adding point (2, 5) to line1
+    line2:add(2, 6)  -- adding point (2, 6) to line2
+
+Graphs can then label their axis as follows:
+
+.. code-block:: lua
+
+    graph:xlabel('x axis')  -- label the x axis "x axis"
+    graph:ylabel('y axis')  -- label the y axis "y axis"
+
+
+Images
+------
+Images can be created using:
+
+.. code-block:: lua
+
+    local image = deepforge.Image('My Example Image', imageTensor)
+
+The first argument is the title of the image and the second argument is the tensor for the image (optional). Both the title and the tensor can be updated during execution as follows.
+
+.. code-block:: lua
+
+    image:title('My New Title')  -- updating the image title
+    image:update(newTensor)  -- updating the displayed image
+

npm-shrinkwrap.json

@@ -1,6 +1,6 @@
 {
   "name": "deepforge",
-  "version": "1.1.0",
+  "version": "1.3.0",
   "dependencies": {
     "abbrev": {
       "version": "1.1.0",

package.json

@@ -17,7 +17,7 @@
     "watch-test": "nodemon --exec 'mocha --recursive test'",
     "build-nn": "node ./utils/nn-parser.js"
   },
-  "version": "1.2.0",
+  "version": "1.3.0",
   "dependencies": {
     "commander": "^2.9.0",
     "dotenv": "^2.0.0",
@@ -43,6 +43,7 @@
     "webgme-simple-nodes": "^2.1.0"
   },
   "devDependencies": {
+    "brython": "^3.2.7",
     "chai": "^3.0.0",
     "jszip": "^2.5.0",
     "mocha": "^2.2.5",

src/common/LayerParser.js

@@ -1,378 +1,127 @@
 /* globals define*/
 (function(root, factory){
     if(typeof define === 'function' && define.amd) {
-        define(['./lua'], function(luajs){
-            return (root.LayerParser = factory(luajs));
+        // TODO: Load the brython script
+        define(['./lua'], function(brython){
+            return (root.LayerParser = factory(brython, console.assert));
         });
     } else if(typeof module === 'object' && module.exports) {
-        var luajs = require('./lua');
-        module.exports = (root.LayerParser = factory(luajs));
+        var brython = require('./node-brython'),
+            assert = require('assert');
+
+        module.exports = (root.LayerParser = factory(brython, assert));
     }
-}(this, function(luajs) {
+}(this, function(brython, assert) {
     var LayerParser = {};
 
-    //////////////////////// Setters //////////////////////// 
-    var returnsSelf = function(fnNode){
-        var stats = fnNode.block.stats,
-            last = stats[stats.length-1];
+    function build_ast(src) {
+        brython.$py_module_path['__main__']='./'
+        return brython.py2js(src,'__main__', '__main__', '__builtins__')
+    }
 
-        if (last.type === 'stat.return') {
-            return last.nret[0].type === 'variable' && last.nret[0].val === 'self';
-        }
-        return false;
-    };
-
-    var isAttrSetter = function(node){
-        if (node.type === 'stat.assignment' && node.lefts.length === 1) {
-            var left = node.lefts[0];
-            return left.type === 'expr.index' && left.self.val === 'self';
-        }
-        return false;
-    };
-
-    var getSettingAttrName = function(node){
-        if (isAttrSetter(node)) {
-            var left = node.lefts[0];
-            return left.key.val;
-        }
-        return null;
-    };
-
-    var getSettingAttrValue = function(node){
-        if (isAttrSetter(node)) {
-            return node.right;
-        }
-        return null;
-    };
-
-    var isSetterMethod = function(curr, parent, className){
-        if (parent && parent.type === 'stat.method') {
-            // is it a fn w/ two statements (stats)
-            if (parent.self.val === className && curr.type === 'function' &&
-                curr.block.stats.length === 2) {
-                // Is the first statement setting a value?
-                return returnsSelf(curr) && getSettingAttrName(curr.block.stats[0]);  // does it return itself?
+    // The provided tree gives us contexts which can have associated 'C'
+    function traverse(node, fn) {
+        var i;
+        if (node.children) {
+            for (i = node.children.length; i--;) {
+                traverse(node.children[i], fn);
+                fn(node.children[i]);
             }
         }
-        return false;
-    };
-
-    var isFnArg = function(method, name) {
-        return method.args.indexOf(name) !== -1;
-    };
-
-    var getSetterSchema = function(node, method) {
-        var setterType,
-            setterFn,
-            value = getSettingAttrValue(node);
-
-        if (value[0].type === 'variable' && isFnArg(method.func, value[0].val)) {
-            setterType = 'arg';
-            setterFn = method.key.val;
-        } else {
-            setterType = 'const';
-            setterFn = {};
-            setterFn[value[0].val] = method.key.val;
-        }
-
-        return {
-            setterType,
-            setterFn
-        };
-    };
-
-    //////////////////////// Setters END //////////////////////// 
-
-    var isInitFn = function(node, className) {
-        if (node.type === 'stat.method' && node.self.val === className) {
-            return node.key.val === '__init';
-        }
-        return false;
-    };
-
-    var getClassAttrDefs = function(method) {
-        var fn = method.func,
-            dict = {},
-            attr,
-            right,
-            value;
-
-        luajs.codegen.traverse(curr => {
-            if (isAttrSetter(curr)) {
-                // Store the value if it is set to a constant
-                attr = curr.lefts[0].key.val;
-                right = curr.right[0];
-                if (right.type.indexOf('const.') !== -1) {
-                    value = right.val;
-
-                    if (right.type === 'const.nil') {
-                        value = null;
-                    }
-
-                    dict[attr] = value;
-                }
-            }
-        })(fn);
-
-        return dict;
-    };
-
-    var getAttrsAndVals = function(method) {
-        // Given a method, get the 'self' attributes and the default values
-        var fn = method.func,
-            dict = {},
-            varName,
-            value,
-            varUsageCnt = {};
-
-        // Get the variables that are used only once (or updating themselves)
-        luajs.codegen.traverse(curr => {
-            if (curr.type === 'variable') {
-                varUsageCnt[curr.val] = varUsageCnt[curr.val] ?
-                    varUsageCnt[curr.val] + 1 : 1;
-            }
-        })(method);
-
-        luajs.codegen.traverse(curr => {
-            // If the variable is only used once and is 'or'-ed w/ a constant
-            // during this use, we can infer that this is the default value
-            if (curr.type === 'expr.op' && curr.op === 'op.or' &&
-                curr.left.type === 'variable' && curr.right.type.indexOf('const') !== -1) {
-                varName = curr.left.val;
-                if (varUsageCnt[varName] === 1) {
-                    value = curr.right.type === 'const.nil' ? null : curr.right;
-                    dict[varName] = value;
-                }
-            }
-        })(fn);
-
-        return dict;
-    };
-
-    var copyNodeValues = function(attrs, from, to) {
-        var value;
-        for (var i = attrs.length; i--;) {
-            value = from[attrs[i]] || null;
-            if (value) {
-                value = (value && value.hasOwnProperty('val')) ? value.val : value;
-                to[attrs[i]] = value;
+        if (node.C && node.C.tree) {
+            for (i = node.C.tree.length; i--;) {
+                traverse(node.C.tree[i], fn);
+                fn(node.C.tree[i]);
             }
         }
-        return to;
-    };
+    }
 
-    var getTypeCheckInfo = function(cond) {
-        var caller,
-            method,
-            target,
-            expType;
+    var types = {},
+        layers = [],
+        pCtx,
+        classNode,
+        params;
 
-        // Check for torch.isTypeOf:
-        if (cond.type === 'expr.call' && cond.func.type === 'expr.index') {
-            caller = cond.func.self.val;
-            method = cond.func.key.val;
+    function isClass(node) {
+        return node.type === 'class';
+    }
 
-            if (cond.type === 'expr.call' && caller === 'torch') {
-                target = cond.args[0].val;
-                if (method === 'isTypeOf' && target) {
-                    expType = cond.args[1].val;
-                    return {
-                        target,
-                        type: expType
-                    };
-                }
-            }
-        } else if (cond.type === 'expr.op') {  // torch.type() === ''
-            // Check right side, too!
-            var sides = [cond.left, cond.right],
-                side,
-                otherSide;
+    function isInitFn(node) {
+        return node.type === 'def' && node.name === '__init__';
+    }
 
-            for (var i = sides.length; i--;) {
-                side = sides[i];
-                otherSide = sides[(i+1)%2];
-                if (side.type === 'expr.call' && side.func.type === 'expr.index') {
-                    // Is it torch?
-                    caller = side.func.self.val;
-                    method = side.func.key.val;
-                    if (caller === 'torch' && method === 'type') {
-                        if (side.args[0].type === 'variable') {
-                            target = side.args[0].val;
-                            if (otherSide.type === 'const.string') {
-                                expType = otherSide.val;
+    function getBaseClass(node) {
+        assert(node.type === 'class');
+        return node.args.tree[0].tree[0].tree[0].value;
+    }
 
-                                return {
-                                    target: target,
-                                    type: expType
-                                };
+    function findTorchLayers(root) {
+        var defaults = {},
+            layers = [],
+            defTypes,
+            args,
+            def;
+
+        traverse(root, node => {
+            // Get the class for the given function
+            if (isInitFn(node)) {
+            // TODO: What if there is no constructor? Is this a potential problem?
+                pCtx = node.parent.node.parent;
+                classNode = pCtx.C.tree[0];
+
+                if (isClass(classNode)) {
+                    // remove the 'self' variable
+                    // TODO: May need to update this for kwargs
+                    // (use positional_list)
+                    args = node.tree[1].tree;
+                    defaults = {};
+                    params = node.args.slice(1);
+                    defTypes = {};
+                    for (var i = args.length; i--;) {
+                        if (args[i].tree[0]) {
+                            def = args[i].tree[0].tree[0];
+                            if (def.type === 'int') {
+                                defaults[params[i-1]] = parseInt.apply(null, def.value.reverse());
+                            } else {
+                                defaults[params[i-1]] = def.value;
+                            }
+                            if (/^(True|False)$/.test(defaults[params[i-1]])) {
+                                defTypes[params[i-1]] = 'boolean';
+                            } else {
+                                defTypes[params[i-1]] = def.type;
                             }
                         }
-                        
                     }
+                    layers.push({
+                        name: classNode.name,
+                        baseType: getBaseClass(classNode),
+                        //doc: classNode.doc_string || '',
+                        defaults: defaults,
+                        types: defTypes,
+                        setters: {},
+                        params: params
+                    });
                 }
             }
-            return null;
-        }
-    };
-
-    var isError = function(stat) {
-        var fn;
-        if (stat.type === 'stat.expr' && stat.expr.type === 'expr.call') {
-            fn = stat.expr.func.val;
-            return fn === 'error';
-        }
-        return false;
-    };
-
-    var inferParamTypes = function(node, paramDefs) {
-        var types = {},
-            check,
-            cond;
-
-        // Infer from assertions
-        luajs.codegen.traverse(curr => {
-            // check for 'assert's that check type
-            if (curr.type === 'expr.call' && curr.func.val === 'assert') {
-                cond = curr.args[0];
-                check = getTypeCheckInfo(cond);
-                if (check) {
-                    types[check.target] = check.type;
-                }
-            } else if (curr.type === 'stat.if' && curr.cond.op === 'uop.not') {
-                // if statements throwing errors on type mismatch
-                cond = curr.cond.operand;  // non-negated version
-                // Check that it throws an error on true
-                if (curr.tblock.stats.some(isError)) {
-                    check = getTypeCheckInfo(cond);
-                    if (check) {
-                        types[check.target] = check.type;
-                    }
-                }
-            }
-        })(node);
-
-        // Infer from defaults
-        Object.keys(paramDefs).forEach(param => {
-            var val = paramDefs[param];
-            if (val) {  // initialized to 'null' doesn't help us...
-                types[param] = val.type.replace('const.', '');
-            }
         });
 
-        return types;
-    };
+        return layers;
+    }
 
-    var findTorchClass = function(ast){ 
-        var torchClassArgs,  // args for `torch.class(...)`
-            name = '',
-            alias,
-            baseType,
-            params,
-            setters = {},
-            defaults = {},
-            paramDefs,
-            attrDefs;
-
-        if(ast.type == 'function'){
-            ast.block.stats.forEach(function(func){
-                if(func.type == 'stat.local' && func.right && func.right[0] &&
-                  func.right[0].func && func.right[0].func.self &&
-                  func.right[0].func.self.val == 'torch' &&
-                  func.right[0].func.key.val == 'class'){
-
-                    torchClassArgs = func.right[0].args.map(arg => arg.val);
-                    name = torchClassArgs[0];
-                    if(name !== ''){
-                        name = name.replace('nn.', '');
-                        alias = func.names[0] || name;
-                        if (torchClassArgs.length > 1) {
-                            baseType = torchClassArgs[1].replace('nn.', '');
-                        }
-                    }
-                }
-            });
-        }
-
-        // Get the setters, defaults and type info (inferred)
-        var setterNames,
-            schema,
-            types,
-            values;
-
-        luajs.codegen.traverse((curr, parent) => {
-            var firstLine,
-                attrName;
-
-            // Record the setter functions
-            if (isSetterMethod(curr, parent, alias)) {
-                firstLine = curr.block.stats[0];
-                // just use the attribute attrName for now...
-                attrName = getSettingAttrName(firstLine);
-
-                // merge schemas
-                schema = getSetterSchema(firstLine, parent);
-                if (setters[attrName] && setters[attrName].setterType === 'const') {  // merge
-                    for (var val in schema.setterFn) {
-                        setters[attrName].setterFn[val] = schema.setterFn[val];
-                    }
-                } else {
-                    setters[attrName] = schema;
-                }
-            } else if (isInitFn(curr, alias)) {  // Record the defaults
-                paramDefs = getAttrsAndVals(curr);
-                attrDefs = getClassAttrDefs(curr);
-                types = inferParamTypes(curr, paramDefs);
-
-                // get ctor args
-                params = curr.func.args;
-                if(params.length === 0 && curr.func.varargs){
-                    params.push('params');
-                }
-            }
-
-        })(ast);
-
-        // Get the defaults for the params from defs
-        if (paramDefs && params) {
-            copyNodeValues(params, paramDefs, defaults);
-        }
-
-        // Get the defaults for the setters from attrDefs
-        if (attrDefs) {
-            setterNames = Object.keys(setters);
-            copyNodeValues(setterNames, attrDefs, defaults);
-        }
-
-        // Remove any const setters w/ only one value and no default
-        setterNames = Object.keys(setters);
-        for (var i = setterNames.length; i--;) {
-            schema = setters[setterNames[i]];
-            if (schema.setterType === 'const') {
-                values = Object.keys(schema.setterFn);
-                if (values.length === 1 &&
-                    // boolean setters can have the default value inferred
-                    values[0] !== 'true' && values[0] !== 'false' &&
-                    !defaults[setterNames[i]]) {
-
-                    delete setters[setterNames[i]];
-                }
-            }
-        }
-
-        return {
-            name,
-            baseType,
-            params,
-            setters,
-            types,
-            defaults
-        };
-    };
-
-    LayerParser.parse = function(text) {
+    // Try to find the class definitions...
+    //
+    //  Need to create:
+    //
+    //    setters: (I don't think these are used in pytorch!
+    //    types: 
+    //    type: 
+    //////////////////////// Setters //////////////////////// 
+    LayerParser.parse = function(src) {
         try {
-            var ast = luajs.parser.parse(text);
-            return findTorchClass(ast);
+            brython.$py_module_path['__main__']='./';
+            var ast = brython.py2js(src,'__main__', '__main__', '__builtins__');
+            var layers = findTorchLayers(ast);
+            return layers;
         } catch (e) {
             return null;
         }

src/common/OperationCode.js

@@ -0,0 +1,331 @@
+/*globals Sk, define*/
+var isNodeJs = typeof module === 'object' && module.exports;
+(function(root, factory){
+    if(typeof define === 'function' && define.amd) {
+        define(['./skulpt.min'], function(){
+            return (root.OperationParser = factory(Sk));
+        });
+    } else if(isNodeJs) {
+        require('./skulpt.min');
+
+        module.exports = (root.OperationParser = factory(Sk));
+    }
+}(this, function(Sk) {
+    var MAIN_FN = 'execute';
+    var CTOR_FN = '__init__';
+    var OperationCode = function(code, filename) {
+        this._lines = code.split('\n');
+        this.filename = filename;
+    };
+
+    OperationCode.prototype.getName = function() {
+        if (!this._schema) this.updateSchema();
+
+        return this._schema.name;
+    };
+
+    OperationCode.prototype.getBase = function() {
+        if (!this._schema) this.updateSchema();
+
+        return this._schema.base;
+    };
+
+    OperationCode.prototype.getArguments = function(method) {
+        if (!this._schema) this.updateSchema();
+        if (!this._schema.methods[method]) return null;
+
+        return this._schema.methods[method].inputs.slice();
+    };
+
+    OperationCode.prototype.getReturnValues = function(method) {
+        if (!this._schema) this.updateSchema();
+        if (!this._schema.methods[method]) return null;
+
+        return this._schema.methods[method].outputs.slice();
+    };
+
+    OperationCode.prototype.getOutputs = function() {
+        return this.getReturnValues(MAIN_FN);
+    };
+
+    OperationCode.prototype.getInputs = function() {
+        return this.getArguments(MAIN_FN);
+    };
+
+    OperationCode.prototype.removeInput = function(name) {
+        return this._removeIOCode(this.getInputs(), name);
+    };
+
+    OperationCode.prototype.removeOutput = function(name) {
+        return this._removeIOCode(this.getOutputs(), name);
+    };
+
+    OperationCode.prototype._removeIOCode = function(ios, name) {
+        var match,
+            prev,
+            line,
+            startIndex,
+            endIndex;
+
+        for (var i = 0; i < ios.length; i++) {
+            match = ios[i];
+            prev = ios[i-1];
+
+            if (match.name === name) {
+                line = this._lines[match.pos.line-1];
+
+                startIndex = prev ? prev.pos.col + prev.value.toString().length : match.pos.col;
+                // only remove the following ',' if first input/output
+                endIndex = i === 0 && i < ios.length-1 ? ios[i+1].pos.col :
+                    match.pos.col + match.value.toString().length;
+                this._lines[match.pos.line-1] = line.substring(0, startIndex) +
+                    line.substring(endIndex);
+
+                this.clearSchema();
+                return match;
+            }
+        }
+        return null;
+    };
+
+    OperationCode.prototype.addInput = function(name) {
+        return this.addArgument(MAIN_FN, name);
+    };
+
+    OperationCode.prototype.addOutput = function(name) {
+        return this.addReturnValue(MAIN_FN, name);
+    };
+
+    OperationCode.prototype.addArgument = function(method, name) {
+        return this._addIOCode(method, name, true);
+    };
+
+    OperationCode.prototype.addReturnValue = function(method, name) {
+        return this._addIOCode(method, name, false);
+    };
+
+    OperationCode.prototype.addMethod = function(method) {
+        // TODO: get the position at the top of the class def
+        var line = this._schema.body.pos.line - 1,
+            indentSize = this._schema.body.pos.col,
+            indent = new Array(indentSize+1).join(' '),
+            snippet = indent + `def ${method}():`,
+            body = new Array(indentSize+5).join(' ') + 'return';
+
+        this._lines.splice(line-1, 0, '');
+        this._lines.splice(line-1, 0, snippet);
+        this._lines.splice(line, 0, body);
+
+        this.clearSchema();
+    };
+
+    OperationCode.prototype.hasMethod = function(method) {
+        if (!this._schema) this.updateSchema();
+        return this._schema.methods[method];
+    };
+
+    OperationCode.prototype._addIOCode = function(method, name, isInput) {
+        if (!this.hasMethod(method)) this.addMethod(method);
+
+        this.updateSchema();
+
+        var ios = this._schema.methods[method][isInput ? 'inputs' : 'outputs'].slice(),
+            node = this._schema.methods[method].node,
+            body = node.body,
+            content = name,
+            line,
+            startIndex,
+            endIndex,
+            lineIndex;
+
+        if (ios.length) {
+            var pos = ios[ios.length-1].pos;
+            var argLen = ios[ios.length-1].name.length;
+
+            line = this._lines[pos.line-1];
+            startIndex = pos.col + argLen;
+            endIndex = pos.col + argLen;
+            content = ', ' + name;
+            lineIndex = pos.line - 1;
+        } else if (isInput) {
+            var first = body[0];
+            lineIndex = first.lineno - 2;
+            line = this._lines[lineIndex];
+            this._lines[lineIndex] = line.replace(/\).*?:/, name + '):');
+
+            return this.clearSchema();
+        } else {
+            var ret = body.find(node => this._isNodeType(node, 'Return_'));
+            if (ret) {
+                lineIndex = ret.lineno-1;
+                startIndex = endIndex = ret.col_offset + 6;
+                content = ' ' + content;
+            } else {  // add to the end of the body (no return statement)
+                var lastNode = body[body.length-1];
+                var indent = new Array(lastNode.col_offset+1).join(' ');
+
+                lineIndex = lastNode.lineno;
+                this._lines.splice(lineIndex, 0, '');
+                startIndex = endIndex = 0;
+                content = indent + 'return ' + content;
+            }
+        }
+
+        line = this._lines[lineIndex];
+        this._lines[lineIndex] = line.substring(0, startIndex) + content +
+            line.substring(endIndex);
+
+        this.clearSchema();
+    };
+
+    OperationCode.prototype.rename = function(oldName, name) {
+        if (!this.hasMethod(MAIN_FN)) return;
+
+        var fnSchema = this._schema.methods[MAIN_FN];
+        var startLine = fnSchema.bounds.start.line - 1;
+        var endLine = fnSchema.bounds.end ? fnSchema.bounds.end.line - 1 : this._lines.length;
+        var pattern = new RegExp('\\b' + oldName + '\\b');
+
+        for (var i = startLine; i < endLine; i++) {
+            this._lines[i] = this._lines[i].replace(pattern, name);
+        }
+        this.clearSchema();
+    };
+
+    OperationCode.prototype.getCode = function() {
+        return this._lines.join('\n');
+    };
+
+    OperationCode.prototype.getAst = function () {
+        if (this._ast) return this._ast;
+
+        var filename = this.filename || 'operation.py';
+        var cst = Sk.parse(filename, this.getCode()).cst;
+        var ast = Sk.astFromParse(cst, filename);
+        return this._ast = ast;
+    };
+
+    OperationCode.prototype._isNodeType = function (node, name) {
+        return node.constructor.name === name;
+    };
+
+    OperationCode.prototype._parseFn = function (node, schema, next) {
+        var name = node.name.v;
+
+        schema.methods[name] = {};
+        // add inputs
+        schema.methods[name].inputs = node.args.args.map(arg => {
+            return {
+                name: arg.id.v,
+                value: arg.id.v,
+                pos: {
+                    line: arg.lineno,
+                    col: arg.col_offset
+                }
+            };
+        });
+
+        // add outputs
+        var ret = node.body.find(node => this._isNodeType(node, 'Return_'));
+        var retVals = [];
+        if (ret) {
+            retVals = ret.value && this._isNodeType(ret.value, 'Tuple') ?
+                ret.value.elts : [ret.value];
+        }
+
+        schema.methods[name].outputs = retVals
+            .filter(node => !!node)
+            .map((arg, index) => {
+                var isNameNode = this._isNodeType(arg, 'Name');
+                var name = isNameNode ? arg.id.v : 'result';
+                if (!isNameNode && index > 0) {
+                    name += '_' + index;
+                }
+
+                var value = this._isNodeType(arg, 'Num') ? arg.n.v : name;
+
+                return {
+                    name: name,
+                    value: value,
+                    pos: {
+                        line: arg.lineno,
+                        col: arg.col_offset
+                    }
+                };
+            });
+
+        // Get the function location
+        schema.methods[name].bounds = {};
+        schema.methods[name].bounds.start = {
+            line: node.lineno,
+            col: node.col_offset
+        };
+
+        if (next) {
+            schema.methods[name].bounds.end = {
+                line: next.lineno,
+                col: next.col_offset
+            };
+        }
+
+        schema.methods[name].node = node;
+    };
+
+    OperationCode.prototype.updateSchema = function () {
+        if (!this._schema) this._schema = this.getSchema();
+    };
+
+    OperationCode.prototype.clearSchema = function () {
+        this._ast = null;
+        this._schema = null;
+    };
+
+    OperationCode.prototype.getSchema = function () {
+        var schema = {
+            name: null,
+            base: null,
+            methods: {}
+        };
+        var ast = this.getAst();
+
+        // Find the class definition
+        var classDef = ast.body.find(node => this._isNodeType(node, 'ClassDef'));
+        if (classDef) {
+            schema.name = classDef.name.v;
+
+            // TODO: what if fn is inherited?
+            var nodes = classDef.body;
+            for (var i = 0; i < nodes.length; i++) {
+                if (this._isNodeType(nodes[i], 'FunctionDef')) {
+                    this._parseFn(nodes[i], schema, nodes[i+1]);
+                }
+            }
+            schema.body = {
+                pos: {
+                    line: nodes[0].lineno,
+                    col: nodes[0].col_offset,
+                }
+            };
+
+        }
+
+        schema.ast = ast;
+
+        return schema;
+    };
+
+    /////////////////////// Attributes /////////////////////// 
+    OperationCode.prototype.addAttribute = function(name, value) {
+        return this._addIOCode(CTOR_FN, name, true);
+    };
+
+    OperationCode.prototype.removeAttribute = function(name) {
+        // TODO
+    };
+
+    OperationCode.prototype.getAttributes = function() {
+        return this.getArguments(CTOR_FN);
+    };
+
+    return OperationCode;
+}));

src/common/OperationParser.js

@@ -0,0 +1,126 @@
+/* globals define, Sk*/
+var isNodeJs = typeof module === 'object' && module.exports;
+(function(root, factory){
+    if(typeof define === 'function' && define.amd) {
+        define(['./skulpt.min'], function(){
+            return (root.OperationParser = factory(Sk));
+        });
+    } else if(isNodeJs) {
+        require('./skulpt.min');
+
+        module.exports = (root.OperationParser = factory(Sk));
+    }
+}(this, function(Sk) {
+    Sk.python3 = true;
+    var OperationParser = {};
+
+    // The provided tree gives us contexts which can have associated 'C'
+    function traverse(node, fn) {
+        var i;
+        if (node.children) {
+            for (i = node.children.length; i--;) {
+                traverse(node.children[i], fn);
+                fn(node.children[i]);
+            }
+        }
+        if (node.C && node.C.tree) {
+            for (i = node.C.tree.length; i--;) {
+                traverse(node.C.tree[i], fn);
+                fn(node.C.tree[i]);
+            }
+        }
+    }
+
+    function isNodeType(node, name) {
+        return node.constructor.name === name;
+    }
+
+    function parseFn(node, schema) {
+        var name = node.name.v;
+
+        schema.methods[name] = {};
+        // add inputs
+        schema.methods[name].inputs = node.args.args.map(arg => {
+            return {
+                name: arg.id.v,
+                value: arg.id.v,
+                pos: {
+                    line: arg.lineno,
+                    col: arg.col_offset
+                }
+            };
+        });
+
+        // add outputs
+        var ret = node.body.find(node => isNodeType(node, 'Return_'));
+        var retVals = [];
+        if (ret) {
+            retVals = ret.value && isNodeType(ret.value, 'Tuple') ?
+                ret.value.elts : [ret.value];
+        }
+
+        schema.methods[name].outputs = retVals.map((arg, index) => {
+            var isNameNode = isNodeType(arg, 'Name');
+            var name = isNameNode ? arg.id.v : 'result';
+            if (!isNameNode && index > 0) {
+                name + '_' + index;
+            }
+
+            var value = isNodeType(arg, 'Num') ? arg.n.v : name;
+
+            return {
+                name: name,
+                value: value,
+                pos: {
+                    line: arg.lineno,
+                    col: arg.col_offset
+                }
+            };
+        });
+    }
+
+    function parseOperationAst(ast) {
+        var schema = {
+            name: null,
+            base: null,
+            methods: {}
+        };
+
+        // Find the class definition
+        var classDef = ast.body.find(node => isNodeType(node, 'ClassDef'));
+        if (classDef) {
+            schema.name = classDef.name.v;
+
+            // TODO: what if fn is inherited?
+            classDef.body
+                .filter(node => isNodeType(node, 'FunctionDef'))
+                .forEach(node => parseFn(node, schema));
+
+        }
+
+        schema.inputs = schema.methods.execute.inputs;
+        schema.outputs = schema.methods.execute.outputs;
+        schema.ast = ast;
+        return schema;
+    }
+
+    OperationParser._traverse = traverse;
+    OperationParser._getAst = function(src, filename) {
+        filename = filename || 'operation.py';
+        var cst = Sk.parse(filename, src).cst;
+        var ast = Sk.astFromParse(cst, filename);
+        return ast;
+    };
+
+    OperationParser.parse = function(src, filename) {
+        //try {
+            var ast = this._getAst(src, filename);
+            return  parseOperationAst(ast);
+        //} catch (e) {
+            //console.error('operation parsing failed:', e);
+            //return null;
+        //}
+    };
+
+    return OperationParser;
+}));

src/common/globals.js

@@ -59,8 +59,14 @@ define([
     var createNamedNode = function(baseId, parentId, isMeta) {
         var newId = client.createNode({parentId, baseId}),
             baseNode = client.getNode(baseId),
-            basename = 'New' + baseNode.getAttribute('name'),
-            newName = getUniqueName(parentId, basename);
+            basename,
+            newName;
+
+        basename = 'New';
+        if (baseNode.getAttribute('name') !== 'Operation') {
+            basename += baseNode.getAttribute('name');
+        }
+        newName = getUniqueName(parentId, basename);
 
         // If instance, make the first char lowercase
         if (!isMeta) {

src/common/lua.js

@@ -3234,7 +3234,7 @@ function runScripts(){
 
 if (window.addEventListener){
     window.addEventListener("DOMContentLoaded", runScripts, false);
-} else {
+} else if (window.attachEvent) {
     window.attachEvent("onload", runScripts);
 }
 

src/common/node-brython.js

@@ -0,0 +1,177 @@
+/*
+Author: Billy Earney
+Date: 04/19/2013
+License: MIT
+
+Description: This file can work as a "bridge" between nodejs and brython
+ so that client side brython code can be executed on the server side.
+Will brython replace Cython one day?  Only time will tell.
+:)
+
+*/
+
+var fs = require('fs'),
+    path = require('path'),
+    //brythonSrcPath = path.join(__dirname, '..', '..', 'node_modules', 'brython', 'www', 'src', 'brython.js');
+    brythonSrcPath = path.join(__dirname, 'brython.js');
+
+document={};
+document.getElementsByTagName = () => [{src: ''}];
+window={};
+window.location = {href: ''};
+window.navigator={}
+window.confirm = () => true;
+window.console = console;
+document.$py_src = {}
+document.$debug = 0
+
+self={};
+__BRYTHON__={}
+__BRYTHON__.$py_module_path = {}
+__BRYTHON__.$py_module_alias = {}
+__BRYTHON__.$py_next_hash = -Math.pow(2,53)
+__BRYTHON__.exception_stack = []
+__BRYTHON__.scope = {}
+__BRYTHON__.modules = {}
+
+// Read and eval library
+jscode = fs.readFileSync(brythonSrcPath, 'utf8');
+eval(jscode);
+
+//function node_import(module,alias,names) {
+function $import_single(module) {
+  var search_path=['../src/libs', '../src/Lib'];
+  var ext=['.js', '.py'];
+  var mods=[module, module+'/__init__'];
+
+  for(var i=0, _len_i = search_path.length; i < _len_i; i++) {
+     for (var j=0, _len_j = ext.length; j < _len_j; j++) {
+         for (var k=0, _len_k = mods.length; k < _len_k; k++) {
+             var path=search_path[i]+'/'+mods[k]+ext[j]
+
+             //console.log("searching for " + path);
+             var module_contents;
+             try {
+               module_contents=fs.readFileSync(path, 'utf8')
+             } catch(err) {}
+             if (module_contents !== undefined) {
+                console.log("imported " + module)
+                //console.log(module_contents);
+                if (ext[j] == '.js') {
+                   return $import_js_module(module,alias,names,path,module_contents)
+                }
+                return $import_py_module(module,alias,names,path,module_contents)
+             }
+         }
+     }
+  }
+  console.log("error time!");
+  res = Error()
+  res.name = 'NotFoundError'
+  res.message = "No module named '"+module+"'"
+  throw res
+}
+
+$compile_python=function(module_contents,module) {
+    var root = __BRYTHON__.py2js(module_contents,module)
+    var body = root.children
+    root.children = []
+    // use the module pattern : module name returns the results of an anonymous function
+    var mod_node = new $Node('expression')
+    //if(names!==undefined){alias='$module'}
+    new $NodeJSCtx(mod_node,'$module=(function()')
+    root.insert(0,mod_node)
+    mod_node.children = body
+    // search for module-level names : functions, classes and variables
+    var mod_names = []
+    for(var i=0, _len_i = mod_node.children.length; i < _len_i;i++){
+        var node = mod_node.children[i]
+        // use function get_ctx() 
+        // because attribute 'context' is renamed by make_dist...
+        var ctx = node.get_ctx().tree[0]
+        if(ctx.type==='def'||ctx.type==='class'){
+            if(mod_names.indexOf(ctx.name)===-1){mod_names.push(ctx.name)}
+        } else if(ctx.type==='from') {
+            for (var j=0, _len_j = ctx.names.length; j < _len_j; j++) {
+                var name=ctx.names[j];
+                if (name === '*') {
+                   // just pass, we don't want to include '*'
+                } else if (ctx.aliases[name] !== undefined) {
+                   if (mod_names.indexOf(ctx.aliases[name])===-1){
+                      mod_names.push(ctx.aliases[name])
+                   }
+                } else {
+                   if (mod_names.indexOf(ctx.names[j])===-1){
+                     mod_names.push(ctx.names[j])
+                   }
+                }
+            }
+        }else if(ctx.type==='assign'){
+            var left = ctx.tree[0]
+            if(left.type==='expr'&&left.tree[0].type==='id'&&left.tree[0].tree.length===0){
+                var id_name = left.tree[0].value
+                if(mod_names.indexOf(id_name)===-1){mod_names.push(id_name)}
+            }
+        }
+    }
+    // create the object that will be returned when the anonymous function is run
+    var ret_code = 'return {'
+    for(var i=0, _len_i = mod_names.length; i < _len_i;i++){
+        ret_code += mod_names[i]+':'+mod_names[i]+','
+    }
+    ret_code += '__getattr__:function(attr){return this[attr]},'
+    ret_code += '__setattr__:function(attr,value){this[attr]=value}'
+    ret_code += '}'
+    var ret_node = new $Node('expression')
+    new $NodeJSCtx(ret_node,ret_code)
+    mod_node.add(ret_node)
+    // add parenthesis for anonymous function execution
+    
+    var ex_node = new $Node('expression')
+    new $NodeJSCtx(ex_node,')()')
+    root.add(ex_node)
+    
+    try{
+        var js = root.to_js()
+        return js;
+    }catch(err){
+        eval('throw '+err.name+'(err.message)')
+    }
+    return undefined;
+}
+
+function build_ast(src) {
+  __BRYTHON__.$py_module_path['__main__']='./'
+  return __BRYTHON__.py2js(src,'__main__', '__main__', '__builtins__')
+}
+
+function execute_python_script(filename) {
+  _py_src=fs.readFileSync(filename, 'utf8')
+  var root = build_ast(_py_src)
+  var js = root.to_js()
+  //eval(js);
+}
+
+//console.log("try to execute compile script");
+
+__BRYTHON__.$py_module_path = __BRYTHON__.$py_module_path || {}
+__BRYTHON__.$py_module_alias = __BRYTHON__.$py_module_alias || {}
+__BRYTHON__.exception_stack = __BRYTHON__.exception_stack || []
+__BRYTHON__.scope = __BRYTHON__.scope || {}
+__BRYTHON__.imported = __BRYTHON__.imported || {}
+__BRYTHON__.modules = __BRYTHON__.modules || {}
+__BRYTHON__.compile_python=$compile_python
+
+__BRYTHON__.debug = 0
+__BRYTHON__.$options = {}
+__BRYTHON__.$options.debug = 0
+
+// other import algs don't work in node
+//import_funcs=[node_import]
+
+if (!module.parent) {
+    var filename=process.argv[2];
+    execute_python_script(filename)
+}
+
+module.exports = __BRYTHON__;

src/common/plugin/Operation.js

@@ -0,0 +1,35 @@
+/*globals define */
+// This is a mixin containing helpers for working with operation nodes
+define([],function() {
+
+    var OperationOps = function() {
+    };
+
+    OperationOps.prototype.getOutputs = function (node) {
+        return this.getOperationData(node, this.META.Outputs);
+    };
+
+    OperationOps.prototype.getInputs = function (node) {
+        return this.getOperationData(node, this.META.Inputs);
+    };
+
+    OperationOps.prototype.getOperationData = function (node, metaType) {
+        // Load the children and the output's children
+        return this.core.loadChildren(node)
+            .then(containers => {
+                var outputs = containers.find(c => this.core.isTypeOf(c, metaType));
+                return outputs ? this.core.loadChildren(outputs) : [];
+            })
+            .then(outputs => {
+                var bases = outputs.map(node => this.core.getMetaType(node));
+                // return [[arg1, Type1, node1], [arg2, Type2, node2]]
+                return outputs.map((node, i) => [
+                    this.getAttribute(node, 'name'),
+                    this.getAttribute(bases[i], 'name'),
+                    node
+                ]);
+            });
+    };
+
+    return OperationOps;
+});

src/common/plugin/PtrCodeGen.js

@@ -6,27 +6,64 @@ define([
     PluginUtils,
     Q
 ) {
+
+    var CodeGen = {
+        Operation: {
+            pluginId: 'GenerateJob',
+            namespace: 'pipeline'
+        }
+    };
+
     var PtrCodeGen = function() {
     };
 
+    PtrCodeGen.prototype.getCodeGenPluginIdFor = function(node) {
+        var base = this.core.getBase(node),
+            name = this.core.getAttribute(node, 'name'),
+            namespace = this.core.getNamespace(node),
+            pluginId;
+
+        //this.logger.debug(`loaded pointer target of ${ptrId}: ${ptrNode}`);
+        pluginId = (this.core.getOwnRegistry(node, 'validPlugins') || '').split(' ').shift();
+        //this.logger.info(`generating code for ${this.core.getAttribute(ptrNode, 'name')} using ${pluginId}`);
+
+        if (this.core.isMetaNode(node) && CodeGen[name]) {
+            pluginId = CodeGen[name].pluginId || CodeGen[name];
+            namespace = CodeGen[name].namespace;
+        }
+
+        if (pluginId) {
+            return {
+                namespace: namespace,
+                pluginId: pluginId
+            };
+        } else if (base) {
+            return this.getCodeGenPluginIdFor(base);
+        } else {
+            return null;
+        }
+    };
+
     PtrCodeGen.prototype.getPtrCodeHash = function(ptrId) {
         return this.core.loadByPath(this.rootNode, ptrId)
             .then(ptrNode => {
                 // Look up the plugin to use
-                var metanode = this.core.getMetaType(ptrNode),
-                    pluginId;
+                var genInfo = this.getCodeGenPluginIdFor(ptrNode);
 
-                this.logger.debug(`loaded pointer target of ${ptrId}: ${ptrNode}`);
-                pluginId = this.core.getRegistry(ptrNode, 'validPlugins').split(' ').shift();
-                this.logger.info(`generating code for ${this.core.getAttribute(ptrNode, 'name')} using ${pluginId}`);
+                if (genInfo.pluginId) {
+                    var context = {
+                        namespace: genInfo.namespace,
+                        activeNode: this.core.getPath(ptrNode)
+                    };
 
-                var context = {
-                    namespace: this.core.getNamespace(metanode),
-                    activeNode: this.core.getPath(ptrNode)
-                };
-
-                // Load and run the plugin
-                return this.executePlugin(pluginId, context);
+                    // Load and run the plugin
+                    return this.executePlugin(genInfo.pluginId, context);
+                } else {
+                    var metanode = this.core.getMetaType(ptrNode),
+                        type = this.core.getAttribute(metanode, 'name');
+                    this.logger.warn(`Could not find plugin for ${type}. Will try to proceed anyway`);
+                    return null;
+                }
             })
             .then(hashes => hashes[0]);  // Grab the first asset for now
     };
@@ -56,12 +93,13 @@ define([
         return PluginUtils.loadNodesAtCommitHash(
             this.project,
             this.core,
-            this.commitHash,
+            this.currentHash,
             this.logger,
             opts
         ).then(config => {
             plugin.initialize(logger, this.blobClient, this.gmeConfig);
             config.core = this.core;
+            config.project = this.project;
             plugin.configure(config);
             return plugin;
         });

src/common/viz/OperationControl.js

@@ -38,5 +38,103 @@ define([
         return typeId ? this._client.getNode(typeId).getChildrenIds() : [];
     };
 
+    OperationControl.prototype.createIONode = function(opId, typeId, isInput, baseName, silent) {
+        var cntrId = this.getDataContainerId(opId, isInput),
+            name = this._client.getNode(opId).getAttribute('name'),
+            dataName,
+            msg;
+
+        baseName = baseName || this._client.getNode(typeId).getAttribute('name').toLowerCase();
+        dataName = this._getDataName(cntrId, baseName);
+
+        msg = `Adding ${isInput ? 'input' : 'output'} "${dataName}" to ${name} interface`;
+        if (!silent) {
+            this._client.startTransaction(msg);
+        }
+
+        var id = this._client.createNode({
+            parentId: cntrId,
+            baseId: typeId
+        });
+
+        // Set the name of the new input
+        this._client.setAttribute(id, 'name', dataName);
+
+        if (!silent) {
+            this._client.completeTransaction();
+        }
+        return id;
+    };
+
+    OperationControl.prototype._getDataName = function(cntrId, baseName) {
+        var otherNames = this._getDataNames(cntrId),
+            name = baseName,
+            i = 1;
+
+        while (otherNames.indexOf(name) !== -1) {
+            i++;
+            name = baseName + '_' + i;
+        }
+        return name;
+    };
+
+    OperationControl.prototype._getDataNames = function(cntrId) {
+        var otherIds = this._client.getNode(cntrId).getChildrenIds();
+
+        return otherIds.map(id => this._client.getNode(id).getAttribute('name'));
+    };
+
+    OperationControl.prototype.getDataNames = function(opId, isInput) {
+        return this._getDataNames(this.getDataContainerId(opId, isInput));
+    };
+
+    OperationControl.prototype.getDataContainerId = function(opId, isInput) {
+        var node = this._client.getNode(opId),
+            cntrs = node.getChildrenIds(),
+            cntrType = isInput ? 'Inputs' : 'Outputs';
+
+        return cntrs.find(id => this.hasMetaName(id, cntrType));
+    };
+
+    OperationControl.prototype.getDataTypeId = function() {
+        var dataNode = this._client.getAllMetaNodes()
+            .find(node => node.getAttribute('name') === 'Data');
+
+        return dataNode.getId();
+    };
+
+    OperationControl.prototype.addInputData = function(opId, name) {
+        return this.createIONode(opId, this.getDataTypeId(), true, name, true);
+    };
+
+    OperationControl.prototype.removeInputData = function(opId, name) {
+        var cntrId = this.getDataContainerId(opId, true),
+            otherIds = this._client.getNode(cntrId).getChildrenIds(),
+            dataId = otherIds.find(id => this._client.getNode(id).getAttribute('name') === name);
+
+        if (dataId) {  // ow, data not found
+            this._client.deleteNode(dataId);
+        }
+    };
+
+    OperationControl.prototype.addOutputData = function(opId, name) {
+        return this.createIONode(opId, this.getDataTypeId(), false, name, true);
+    };
+
+    OperationControl.prototype.removeOutputData = function(opId, name) {
+        var cntrId = this.getDataContainerId(opId),
+            otherIds = this._client.getNode(cntrId).getChildrenIds(),
+            dataId = otherIds.find(id => this._client.getNode(id).getAttribute('name') === name);
+
+        if (dataId) {  // ow, data not found
+            this._client.deleteNode(dataId);
+        }
+    };
+
+    OperationControl.prototype.isInputData = function(nodeId) {
+        var node = this._client.getNode(nodeId);
+        return this.hasMetaName(node.getParentId(), 'Inputs');
+    };
+
     return OperationControl;
 });

src/decorators/OpIntDecorator/EasyDAG/OpIntDecorator.EasyDAGWidget.js

@@ -34,7 +34,13 @@ define([
             this.enableTooltip(this._node.baseName, 'dark');
         }
         DecoratorBase.prototype.initialize.call(this);
-        this.$name.on('dblclick', this.editName.bind(this));
+        this.$name.on('click', () => {
+            // Operations must already be selected. Otherwise, they will animate
+            // after the edit name box is created and it will be placed incorrectly
+            if (this.expanded || !this.isOperation()) {
+                this.editName();
+            }
+        });
     };
 
     OpIntDecorator.prototype.AttributeField = AttributeField;
@@ -97,6 +103,7 @@ define([
             () => this.deleteAttribute(name));
     };
 
+    // TODO: implement this in the widget controller (so we can update the op code)
     OpIntDecorator.prototype.deleteAttribute = function(name) {
         var opName = this._node.attributes.name.value,
             msg = `Deleting "${name}" attribute from "${opName}" operation`;
@@ -107,6 +114,7 @@ define([
         this.client.completeTransaction();
     };
 
+    // TODO: implement this in the widget controller (so we can update the op code)
     OpIntDecorator.prototype.setAttributeMeta = function(name, desc) {
         var schema,
             opName = this._node.attributes.name.value,

src/plugins/CreateExecution/CreateExecution.js

@@ -173,7 +173,10 @@ define([
             name,
             i = 2;
 
-        basename = basename.replace(/[^\da-zA-Z_]/g, '_');
+        basename = basename
+            .replace(/^\s*/, '')
+            .replace(/\s*$/, '')
+            .replace(/[^\da-zA-Z_]/g, '_');
         name = basename;
 
         // Get a unique name wrt the tags and the other executions

src/plugins/CreateTorchMeta/CreateTorchMeta.js

@@ -205,9 +205,10 @@ define([
     };
 
     // Some helper methods w/ attribute handling
-    var LUA_TO_GME = {
+    var PYTHON_TO_GME = {
         boolean: 'boolean',
-        number: 'float',
+        float: 'float',
+        int: 'integer',
         string: 'string'
     };
 
@@ -301,7 +302,7 @@ define([
             attrs.forEach(name => {
                 desc = {};
                 defVal = defaults.hasOwnProperty(name) ? defaults[name] : '';
-                type = LUA_TO_GME[types[name]];
+                type = PYTHON_TO_GME[types[name]];
                 if (type) {
                     desc.type = type;
                 }
@@ -376,6 +377,11 @@ define([
             // Set the min, max
             schema.max = +schema.max;
         }
+        // Add the enum for booleans so we use python style True/False
+        if (schema.type === 'boolean') {
+            schema.enum = ['True', 'False'];
+            schema.type = 'string';
+        }
 
         // Create the attribute and set the schema
         this.core.setAttributeMeta(node, name, schema);

src/plugins/CreateTorchMeta/metadata.json

@@ -17,7 +17,6 @@
       "value": "all",
       "valueItems": [
         "nn",
-        "rnn",
         "all"
       ],
       "valueType": "string",

src/plugins/CreateTorchMeta/schemas/index.js

@@ -1,13 +1,10 @@
 /*globals define*/
 define([
-    'text!./nn.json',
-    'text!./rnn.json'
+    'text!./nn.json'
 ], function(
-    nn,
-    rnn
+    nn
 ) {
     return {
-        nn: nn,
-        rnn: rnn
+        nn: nn
     };
 });

src/plugins/CreateTorchMeta/schemas/rnn.json

@@ -1,178 +0,0 @@
-[
-  {
-    "name": "CopyGrad",
-    "baseType": "Identity",
-    "setters": {},
-    "defaults": {},
-    "type": "RNN"
-  },
-  {
-    "name": "FastLSTM",
-    "baseType": "LSTM",
-    "params": [
-      "inputSize",
-      "outputSize",
-      "rho",
-      "eps",
-      "momentum",
-      "affine"
-    ],
-    "setters": {},
-    "types": {
-      "eps": "number",
-      "momentum": "number"
-    },
-    "defaults": {
-      "momentum": 0.1,
-      "eps": 0.1
-    },
-    "type": "RNN"
-  },
-  {
-    "name": "LSTM",
-    "baseType": "AbstractRecurrent",
-    "params": [
-      "inputSize",
-      "outputSize",
-      "rho",
-      "cell2gate"
-    ],
-    "setters": {},
-    "types": {
-      "rho": "number"
-    },
-    "defaults": {
-      "rho": 9999
-    },
-    "type": "RNN"
-  },
-  {
-    "name": "LinearNoBias",
-    "baseType": "Linear",
-    "params": [
-      "inputSize",
-      "outputSize"
-    ],
-    "setters": {},
-    "types": {},
-    "defaults": {},
-    "type": "Simple"
-  },
-  {
-    "name": "LookupTableMaskZero",
-    "baseType": "LookupTable",
-    "params": [
-      "nIndex",
-      "nOutput"
-    ],
-    "setters": {},
-    "types": {},
-    "defaults": {},
-    "type": "RNN"
-  },
-  {
-    "name": "NormStabilizer",
-    "baseType": "AbstractRecurrent",
-    "params": [
-      "beta"
-    ],
-    "setters": {},
-    "defaults": {},
-    "type": "RNN"
-  },
-  {
-    "name": "Recurrent",
-    "baseType": "AbstractRecurrent",
-    "params": [
-      "start",
-      "input",
-      "feedback",
-      "transfer",
-      "rho",
-      "merge"
-    ],
-    "setters": {},
-    "types": {
-      "start": "nn.Module",
-      "transfer": "nn.Module",
-      "feedback": "nn.Module",
-      "input": "nn.Module"
-    },
-    "defaults": {},
-    "type": "RNN"
-  },
-  {
-    "name": "SAdd",
-    "baseType": "Module",
-    "params": [
-      "addend",
-      "negate"
-    ],
-    "setters": {},
-    "types": {},
-    "defaults": {},
-    "type": "RNN"
-  },
-  {
-    "name": "SeqBRNN",
-    "baseType": "Container",
-    "params": [
-      "inputDim",
-      "hiddenDim",
-      "batchFirst"
-    ],
-    "setters": {},
-    "types": {},
-    "defaults": {},
-    "type": "RNN"
-  },
-  {
-    "name": "SeqGRU",
-    "baseType": "Module",
-    "params": [
-      "inputSize",
-      "outputSize"
-    ],
-    "setters": {},
-    "types": {},
-    "defaults": {},
-    "type": "RNN"
-  },
-  {
-    "name": "SeqLSTM",
-    "baseType": "Module",
-    "params": [
-      "inputsize",
-      "hiddensize",
-      "outputsize"
-    ],
-    "setters": {},
-    "types": {},
-    "defaults": {},
-    "type": "RNN"
-  },
-  {
-    "name": "SeqLSTMP",
-    "baseType": "SeqLSTM",
-    "params": [
-      "inputsize",
-      "hiddensize",
-      "outputsize"
-    ],
-    "setters": {},
-    "types": {},
-    "defaults": {},
-    "type": "RNN"
-  },
-  {
-    "name": "SeqReverseSequence",
-    "baseType": "Module",
-    "params": [
-      "dim"
-    ],
-    "setters": {},
-    "types": {},
-    "defaults": {},
-    "type": "RNN"
-  }
-]

src/plugins/ExecuteJob/ExecuteJob.Files.js

@@ -1,346 +0,0 @@
-/*globals define*/
-define([
-    './templates/index',
-    'q',
-    'underscore',
-    'deepforge/Constants'
-], function(
-    Templates,
-    Q,
-    _,
-    CONSTANTS
-) {
-
-    var SKIP_ATTRIBUTES = [
-        'code',
-        'stdout',
-        'execFiles',
-        'jobId',
-        'secret',
-        CONSTANTS.LINE_OFFSET,
-        CONSTANTS.DISPLAY_COLOR
-    ];
-    var ExecuteJob = function() {
-    };
-
-    ExecuteJob.prototype.createOperationFiles = function (node) {
-        var files = {};
-        // For each operation, generate the output files:
-        //   inputs/<arg-name>/init.lua  (respective data deserializer)
-        //   pointers/<name>/init.lua  (result of running the main plugin on pointer target - may need a rename)
-        //   outputs/<name>/  (make dirs for each of the outputs)
-        //   outputs/init.lua  (serializers for data outputs)
-        //
-        //   attributes.lua (returns lua table of operation attributes)
-        //   init.lua (main file -> calls main and serializes outputs)
-        //   <name>.lua (entry point -> calls main operation code)
-
-        // add the given files
-        this.logger.info('About to create dist execution files');
-        files['start.js'] = _.template(Templates.START)(CONSTANTS);
-        return this.createEntryFile(node, files)
-            .then(() => this.createClasses(node, files))
-            .then(() => this.createCustomLayers(node, files))
-            .then(() => this.createInputs(node, files))
-            .then(() => this.createOutputs(node, files))
-            .then(() => this.createMainFile(node, files))
-            .then(() => {
-                this.createAttributeFile(node, files);
-                return Q.ninvoke(this, 'createPointers', node, files);
-            });
-    };
-
-    ExecuteJob.prototype.createEntryFile = function (node, files) {
-        this.logger.info('Creating entry files...');
-        return this.getOutputs(node)
-            .then(outputs => {
-                var name = this.getAttribute(node, 'name'),
-                    content = {};
-
-                // inputs and outputs
-                content.name = name;
-                content.outputs = outputs;
-
-                files['init.lua'] = _.template(Templates.ENTRY)(content);
-
-                // Create the deepforge file
-                files['deepforge.lua'] = _.template(Templates.DEEPFORGE)(CONSTANTS);
-            });
-    };
-
-    ExecuteJob.prototype.createClasses = function (node, files) {
-        var metaDict = this.core.getAllMetaNodes(this.rootNode),
-            isClass,
-            metanodes,
-            classNodes,
-            inheritanceLvl = {},
-            code;
-
-        this.logger.info('Creating custom layer file...');
-        metanodes = Object.keys(metaDict).map(id => metaDict[id]);
-        isClass = this.getTypeDictFor('Complex', metanodes);
-
-        classNodes = metanodes.filter(node => {
-            var base = this.core.getBase(node),
-                baseId,
-                count = 1;
-
-            // Count the sets back to a class node
-            while (base) {
-                baseId = this.core.getPath(base);
-                if (isClass[baseId]) {
-                    inheritanceLvl[this.core.getPath(node)] = count;
-                    return true;
-                }
-                base = this.core.getBase(base);
-                count++;
-            }
-
-            return false;
-        });
-
-        // Get the code definitions for each
-        // Sort by levels of inheritance...
-        code = classNodes.sort((a, b) => {
-            var aId = this.core.getPath(a),
-                bId = this.core.getPath(b);
-
-            return inheritanceLvl[aId] > inheritanceLvl[bId];
-        }).map(node =>
-            `require './${this.getAttribute(node, 'name')}.lua'`
-        ).join('\n');
-
-        // Create the class files
-        classNodes.forEach(node => {
-            var name = this.getAttribute(node, 'name');
-            files[`classes/${name}.lua`] = this.getAttribute(node, 'code');
-        });
-
-        // Create the custom layers file
-        files['classes/init.lua'] = code;
-    };
-
-    ExecuteJob.prototype.getTypeDictFor = function (name, metanodes) {
-        var isType = {};
-        // Get all the custom layers
-        for (var i = metanodes.length; i--;) {
-            if (this.getAttribute(metanodes[i], 'name') === name) {
-                isType[this.core.getPath(metanodes[i])] = true;
-            }
-        }
-        return isType;
-    };
-
-    ExecuteJob.prototype.createCustomLayers = function (node, files) {
-        var metaDict = this.core.getAllMetaNodes(this.rootNode),
-            isCustomLayer,
-            metanodes,
-            customLayers,
-            code;
-
-        this.logger.info('Creating custom layer file...');
-        metanodes = Object.keys(metaDict).map(id => metaDict[id]);
-        isCustomLayer = this.getTypeDictFor('CustomLayer', metanodes);
-
-        customLayers = metanodes.filter(node =>
-            this.core.getMixinPaths(node).some(id => isCustomLayer[id]));
-
-        // Get the code definitions for each
-        code = 'require \'nn\'\n\n' + customLayers
-            .map(node => this.getAttribute(node, 'code')).join('\n');
-
-        // Create the custom layers file
-        files['custom-layers.lua'] = code;
-    };
-
-    ExecuteJob.prototype.createInputs = function (node, files) {
-        var tplContents,
-            inputs;
-
-        this.logger.info('Retrieving inputs and deserialize fns...');
-        return this.getInputs(node)
-            .then(allInputs => {
-                // For each input, match the connection with the input name
-                //   [ name, type ] => [ name, type, node ]
-                //
-                // For each input,
-                //  - create the deserializer
-                //  - put it in inputs/<name>/init.lua
-                //  - copy the data asset to /inputs/<name>/init.lua
-                inputs = allInputs
-                    .filter(pair => !!this.getAttribute(pair[2], 'data'));  // remove empty inputs
-
-                files.inputAssets = {};  // data assets
-                return Q.all(inputs.map(pair => {
-                    var name = pair[0],
-                        node = pair[2],
-                        nodeId = this.core.getPath(node),
-                        fromNodeId;
-
-                    // Get the deserialize function. First, try to get it from
-                    // the source method (this guarantees that the correct
-                    // deserialize method is used despite any auto-upcasting
-                    fromNodeId = this.inputPortsFor[nodeId][0] || nodeId;
-
-                    return this.core.loadByPath(this.rootNode, fromNodeId)
-                        .then(fromNode => {
-                            var deserFn,
-                                base,
-                                className;
-
-                            deserFn = this.getAttribute(fromNode, 'deserialize');
-
-                            if (this.isMetaTypeOf(node, this.META.Complex)) {
-                                // Complex objects are expected to define their own
-                                // (static) deserialize factory method
-                                base = this.core.getMetaType(node);
-                                className = this.getAttribute(base, 'name');
-                                deserFn = `return ${className}.deserialize(path)`;
-                            }
-
-                            return {
-                                name: name,
-                                code: deserFn
-                            };
-                        });
-                }));
-            })
-            .then(_tplContents => {
-                tplContents = _tplContents;
-                var hashes = inputs.map(pair => {
-                    var hash = this.getAttribute(pair[2], 'data');
-                    files.inputAssets[pair[0]] = hash;
-                    return {
-                        hash: hash,
-                        name: pair[0]
-                    };
-                });
-
-                return Q.all(hashes.map(pair => 
-                    this.blobClient.getMetadata(pair.hash)
-                        .fail(err => this.onBlobRetrievalFail(node, pair.name, err))));
-            })
-            .then(metadatas => {
-                // Create the deserializer
-                tplContents.forEach((ctnt, i) => {
-                    // Get the name of the given asset
-                    ctnt.filename = metadatas[i].name;
-                    files['inputs/' + ctnt.name + '/init.lua'] = _.template(Templates.DESERIALIZE)(ctnt);
-                });
-                return files;
-            });
-    };
-
-    ExecuteJob.prototype.createOutputs = function (node, files) {
-        // For each of the output types, grab their serialization functions and
-        // create the `outputs/init.lua` file
-        this.logger.info('Creating outputs/init.lua...');
-        return this.getOutputs(node)
-            .then(outputs => {
-                var outputTypes = outputs
-                // Get the serialize functions for each
-                    .map(tuple => {
-                        var node = tuple[2],
-                            serFn = this.getAttribute(node, 'serialize');
-
-                        if (this.isMetaTypeOf(node, this.META.Complex)) {
-                            // Complex objects are expected to define their own
-                            // serialize methods
-                            serFn = 'if data ~= nil then data:serialize(path) end';
-                        }
-
-                        return [tuple[1], serFn];
-                    });
-
-                files['outputs/init.lua'] = _.template(Templates.SERIALIZE)({types: outputTypes});
-            });
-    };
-
-    ExecuteJob.prototype.createMainFile = function (node, files) {
-        this.logger.info('Creating main file...');
-        return this.getInputs(node)
-            .then(inputs => {
-                var name = this.getAttribute(node, 'name'),
-                    code = this.getAttribute(node, 'code'),
-                    pointers = this.core.getPointerNames(node).filter(ptr => ptr !== 'base'),
-                    content = {
-                        name: name
-                    };
-
-                // Get input data arguments
-                content.inputs = inputs
-                    .map(pair => [pair[0], !this.getAttribute(pair[2], 'data')]);  // remove empty inputs
-
-                // Defined variables for each pointers
-                content.pointers = pointers
-                    .map(id => [id, this.core.getPointerPath(node, id) === null]);
-
-                // Add remaining code
-                content.code = code;
-
-                files['main.lua'] = _.template(Templates.MAIN)(content);
-
-                // Set the line offset
-                var lineOffset = this.getLineOffset(files['main.lua'], code);
-                this.setAttribute(node, CONSTANTS.LINE_OFFSET, lineOffset);
-            });
-    };
-
-    ExecuteJob.prototype.getLineOffset = function (main, snippet) {
-        var i = main.indexOf(snippet),
-            lines = main.substring(0, i).match(/\n/g);
-
-        return lines ? lines.length : 0;
-    };
-
-    ExecuteJob.prototype.createAttributeFile = function (node, files) {
-        var numOrBool = /^(-?\d+\.?\d*((e|e-)\d+)?|(true|false))$/,
-            table;
-
-        this.logger.info('Creating attributes file...');
-        table = '{\n\t' + this.core.getAttributeNames(node)
-            .filter(attr => SKIP_ATTRIBUTES.indexOf(attr) === -1)
-            .map(name => {
-                var value = this.getAttribute(node, name);
-                if (!numOrBool.test(value)) {
-                    value = `"${value}"`;
-                }
-                return [name, value];
-            })
-            .map(pair => pair.join(' = '))
-            .join(',\n\t') + '\n}';
-
-        files['attributes.lua'] = `-- attributes of ${this.getAttribute(node, 'name')}\nreturn ${table}`;
-    };
-
-    ExecuteJob.prototype.createPointers = function (node, files, cb) {
-        var pointers,
-            nIds;
-
-        this.logger.info('Creating pointers file...');
-        pointers = this.core.getPointerNames(node)
-            .filter(name => name !== 'base')
-            .filter(id => this.core.getPointerPath(node, id) !== null);
-
-        nIds = pointers.map(p => this.core.getPointerPath(node, p));
-        files.ptrAssets = {};
-        Q.all(
-            nIds.map(nId => this.getPtrCodeHash(nId))
-        )
-        .then(resultHashes => {
-            var name = this.getAttribute(node, 'name');
-            this.logger.info(`Pointer generation for ${name} FINISHED!`);
-            resultHashes.forEach((hash, index) => {
-                files.ptrAssets[`pointers/${pointers[index]}/init.lua`] = hash;
-            });
-            return cb(null, files);
-        })
-        .fail(e => {
-            this.logger.error(`Could not generate pointer files for ${this.getAttribute(node, 'name')}: ${e.toString()}`);
-            return cb(e);
-        });
-    };
-
-    return ExecuteJob;
-});
-

src/plugins/ExecuteJob/ExecuteJob.js

@@ -8,10 +8,10 @@ define([
     'plugin/PluginBase',
     'deepforge/plugin/LocalExecutor',
     'deepforge/plugin/PtrCodeGen',
+    'deepforge/plugin/Operation',
     'deepforge/api/JobLogsClient',
     'deepforge/api/JobOriginClient',
     'deepforge/api/ExecPulseClient',
-    './ExecuteJob.Files',
     './ExecuteJob.Metadata',
     './ExecuteJob.SafeSave',
     'deepforge/Constants',
@@ -26,11 +26,11 @@ define([
     PluginBase,
     LocalExecutor,  // DeepForge operation primitives
     PtrCodeGen,
+    OperationPlugin,
     JobLogsClient,
     JobOriginClient,
     ExecPulseClient,
 
-    ExecuteJobFiles,
     ExecuteJobMetadata,
     ExecuteJobSafeSave,
 
@@ -44,8 +44,7 @@ define([
 
     pluginMetadata = JSON.parse(pluginMetadata);
 
-    var OUTPUT_INTERVAL = 1500,
-        STDOUT_FILE = 'job_stdout.txt';
+    var STDOUT_FILE = 'job_stdout.txt';
 
     /**
      * Initializes a new instance of ExecuteJob.
@@ -453,9 +452,10 @@ define([
             children.find(child => this.isMetaTypeOf(child, this.META.Operation)));
     };
 
-    ExecuteJob.prototype.onBlobRetrievalFail = function (node, input, err) {
+    // Handle the blob retrieval failed error
+    ExecuteJob.prototype.onBlobRetrievalFail = function (node, input) {
         var job = this.core.getParent(node),
-            e = `Failed to retrieve "${input}" (${err})`,
+            e = `Failed to retrieve "${input}" (BLOB_FETCH_FAILED)`,
             consoleErr = `Failed to execute operation: ${e}`;
 
         consoleErr += [
@@ -473,14 +473,8 @@ define([
 
     ExecuteJob.prototype.executeJob = function (job) {
         return this.getOperation(job).then(node => {
-            var jobId = this.core.getPath(job),
-                name = this.getAttribute(node, 'name'),
-                localTypeId = this.getLocalOperationType(node),
-                artifact,
-                artifactName,
-                files,
-                data = {},
-                inputs;
+            var name = this.getAttribute(node, 'name'),
+                localTypeId = this.getLocalOperationType(node);
 
             // Execute any special operation types here - not on an executor
             this.logger.debug(`Executing operation "${name}"`);
@@ -488,126 +482,22 @@ define([
                 return this.executeLocalOperation(localTypeId, node);
             } else {
                 // Generate all execution files
-                return this.createOperationFiles(node).then(results => {
-                    this.logger.info('Created operation files!');
-                    files = results;
-                    artifactName = `${name}_${jobId.replace(/\//g, '_')}-execution-files`;
-                    artifact = this.blobClient.createArtifact(artifactName);
-
-                    // Add the input assets
-                    //   - get the metadata (name)
-                    //   - add the given inputs
-                    inputs = Object.keys(files.inputAssets);
-
-                    return Q.all(
-                        inputs.map(input => {  // Get the metadata for each input
-                            var hash = files.inputAssets[input];
-
-                            // data asset for "input"
-                            return this.blobClient.getMetadata(hash)
-                                .fail(err => this.onBlobRetrievalFail(job, input, err));
-                        })
-                    );
-                })
-                .then(mds => {
-                    // Record the large files
-                    var inputData = {},
-                        runsh = '# Bash script to download data files and run job\n' +
-                        'if [ -z "$DEEPFORGE_URL" ]; then\n  echo "Please set DEEPFORGE_URL and' +
-                        ' re-run:"\n  echo ""  \n  echo "  DEEPFORGE_URL=http://my.' +
-                        'deepforge.server.com:8080 bash run.sh"\n  echo ""\n exit 1\nfi\n';
-
-                    mds.forEach((metadata, i) => {
-                        // add the hashes for each input
-                        var input = inputs[i], 
-                            hash = files.inputAssets[input],
-                            dataPath = 'inputs/' + input + '/data',
-                            url = this.blobClient.getRelativeDownloadURL(hash);
-
-                        inputData[dataPath] = {
-                            req: hash,
-                            cache: metadata.content
-                        };
-
-                        // Add to the run.sh file
-                        runsh += `wget $DEEPFORGE_URL${url} -O ${dataPath}\n`;
+                return this.getPtrCodeHash(this.core.getPath(node))
+                    .fail(err => {
+                        this.logger.error(`Could not generate files: ${err}`);
+                        if (err.message.indexOf('BLOB_FETCH_FAILED') > -1) {
+                            this.onBlobRetrievalFail(node, err.message.split(':')[1]);
+                        }
+                        throw err;
+                    })
+                    .then(hash => {
+                        this.logger.info(`Saved execution files`);
+                        this.result.addArtifact(hash);  // Probably only need this for debugging...
+                        this.executeDistOperation(job, node, hash);
+                    })
+                    .fail(e => {
+                        this.onOperationFail(node, `Distributed operation "${name}" failed ${e}`);
                     });
-
-                    delete files.inputAssets;
-                    files['input-data.json'] = JSON.stringify(inputData, null, 2);
-                    runsh += 'th init.lua';
-                    files['run.sh'] = runsh;
-
-                    // Add pointer assets
-                    Object.keys(files.ptrAssets)
-                        .forEach(path => data[path] = files.ptrAssets[path]);
-
-                    // Add the executor config
-                    return this.getOutputs(node);
-                })
-                .then(outputArgs => {
-                    var config,
-                        outputs,
-                        fileList,
-                        ptrFiles = Object.keys(files.ptrAssets),
-                        file;
-
-                    delete files.ptrAssets;
-                    fileList = Object.keys(files).concat(ptrFiles);
-
-                    outputs = outputArgs.map(pair => pair[0])
-                        .map(name => {
-                            return {
-                                name: name,
-                                resultPatterns: [`outputs/${name}`]
-                            };
-                        });
-
-                    outputs.push(
-                        {
-                            name: 'stdout',
-                            resultPatterns: [STDOUT_FILE]
-                        },
-                        {
-                            name: name + '-all-files',
-                            resultPatterns: fileList
-                        }
-                    );
-
-                    config = {
-                        cmd: 'node',
-                        args: ['start.js'],
-                        outputInterval: OUTPUT_INTERVAL,
-                        resultArtifacts: outputs
-                    };
-                    files['executor_config.json'] = JSON.stringify(config, null, 4);
-
-                    // Save the artifact
-                    // Remove empty hashes
-                    for (file in data) {
-                        if (!data[file]) {
-                            this.logger.warn(`Empty data hash has been found for file "${file}". Removing it...`);
-                            delete data[file];
-                        }
-                    }
-                    return artifact.addObjectHashes(data);
-                })
-                .then(() => {
-                    this.logger.info(`Added ptr/input data hashes for "${artifactName}"`);
-                    return artifact.addFiles(files);
-                })
-                .then(() => {
-                    this.logger.info(`Added execution files for "${artifactName}"`);
-                    return artifact.save();
-                })
-                .then(hash => {
-                    this.logger.info(`Saved execution files "${artifactName}"`);
-                    this.result.addArtifact(hash);  // Probably only need this for debugging...
-                    this.executeDistOperation(job, node, hash);
-                })
-                .fail(e => {
-                    this.onOperationFail(node, `Distributed operation "${name}" failed ${e}`);
-                });
             }
         });
     };
@@ -881,32 +771,6 @@ define([
             .fail(e => this.onOperationFail(node, `Operation ${nodeId} failed: ${e}`));
     };
 
-    ExecuteJob.prototype.getOutputs = function (node) {
-        return this.getOperationData(node, this.META.Outputs);
-    };
-
-    ExecuteJob.prototype.getInputs = function (node) {
-        return this.getOperationData(node, this.META.Inputs);
-    };
-
-    ExecuteJob.prototype.getOperationData = function (node, metaType) {
-        // Load the children and the output's children
-        return this.core.loadChildren(node)
-            .then(containers => {
-                var outputs = containers.find(c => this.core.isTypeOf(c, metaType));
-                return outputs ? this.core.loadChildren(outputs) : [];
-            })
-            .then(outputs => {
-                var bases = outputs.map(node => this.core.getMetaType(node));
-                // return [[arg1, Type1, node1], [arg2, Type2, node2]]
-                return outputs.map((node, i) => [
-                    this.getAttribute(node, 'name'),
-                    this.getAttribute(bases[i], 'name'),
-                    node
-                ]);
-            });
-    };
-
     //////////////////////////// Special Operations ////////////////////////////
     ExecuteJob.prototype.executeLocalOperation = function (type, node) {
         // Retrieve the given LOCAL_OP type
@@ -920,7 +784,7 @@ define([
 
     _.extend(
         ExecuteJob.prototype,
-        ExecuteJobFiles.prototype,
+        OperationPlugin.prototype,
         ExecuteJobMetadata.prototype,
         ExecuteJobSafeSave.prototype,
         PtrCodeGen.prototype,

src/plugins/ExecuteJob/templates/deepforge.ejs

@@ -1,105 +0,0 @@
--- Instantiate the deepforge object
-deepforge = {}
-
-function deepforge.initialize()
-    require 'nn'
-    require 'rnn'
-    require './classes/init'
-    require './custom-layers'
-end
-
-function deepforge.id()
-    if __deepforge_id == nil then
-        __deepforge_id = 0
-    end
-    __deepforge_id = __deepforge_id + 1
-    return __deepforge_id
-end
-
-function deepforge._cmd(...)
-    local cmd = '<%= START_CMD %>'
-    local arg = {...}
-    local n = #arg
-    for i=1,n do
-        cmd = cmd .. ' ' .. tostring(arg[i])
-    end
-    print(cmd .. ' ')  -- guarantee ends w/ space
-end
-
--- Graph support
-Graph = torch.class('deepforge.Graph')
-
-function Graph:__init(name)
-    self.id = deepforge.id()
-    self.name = name
-    deepforge._cmd('<%= GRAPH_CREATE %>', self.id, name)
-end
-
-_Line = torch.class('deepforge._Line')
-
-function _Line:__init(graphId, name, opts)
-    self.id = deepforge.id()
-    self.name = name
-    deepforge._cmd('<%= GRAPH_CREATE_LINE %>', graphId, self.id, name)
-end
-
-function _Line:add(x, y)
-    assert(type(x) == "number" and type(y) == "number", "adding point (" .. tostring(x) .. ", " .. tostring(y) .. ") to " .. self.name .. " failed: expected (number, number)")
-    deepforge._cmd('<%= GRAPH_PLOT %>', self.id, x, y)
-end
-
-function Graph:line(name, opts)
-    return deepforge._Line(self.id, name, opts)
-end
-
-function Graph:xlabel(name)
-    deepforge._cmd('<%= GRAPH_LABEL_AXIS.X %>', self.id, name)
-end
-
-function Graph:ylabel(name)
-    deepforge._cmd('<%= GRAPH_LABEL_AXIS.Y %>', self.id, name)
-end
-
--- Image support
-local function saveImage(name, tensor)
-    require 'image'
-    require 'paths'
-
-    -- save it in the tmp directory
-    local filename = name .. '.png'
-    local path = paths.concat('metadata', filename)
-
-    if paths.dir('metadata') == nil then
-       paths.mkdir('metadata')
-    end
-
-    image.save(path, tensor)
-end
-
-function deepforge.image(name, tensor)
-    saveImage(name, tensor)
-    deepforge._cmd("<%= IMAGE.BASIC %>", deepforge.id(), name)
-end
-
-Image = torch.class('deepforge.Image')
-function Image:__init(name, tensor)
-    self.id = deepforge.id()
-    self.name = name
-
-    if tensor ~= nil then
-        saveImage(name, tensor)
-        deepforge._cmd('<%= IMAGE.CREATE %>', self.id, self.name)
-    end
-end
-
-function Image:update(tensor)
-    saveImage(self.name, tensor)
-    deepforge._cmd('<%= IMAGE.UPDATE %>', self.id, self.name)
-end
-
-function Image:title(name)
-    self.name = name
-    deepforge._cmd('<%= IMAGE.NAME %>', self.id, self.name)
-end
-
-return deepforge

src/plugins/ExecuteJob/templates/entry.ejs

@@ -1,16 +0,0 @@
--- Instantiate the deepforge object
-require './deepforge'
-
--- run the <%= name %> and serialize the results
-print('\n############### Running "<%= name.replace(/'/g, '\\\'') %>" Operation ############### ')
-results = require './main'
-print('############### "<%= name.replace(/'/g, '\\\'') %>" Operation Complete! ###############')
-
--- serialize by type
-outputs = require './outputs'
-<% outputs.forEach(pair => { 
-    var name = pair[0],
-        type = pair[1];
-%>
-outputs.<%= type %>('<%= name %>', results.<%= name %>)
-<% }); %>

src/plugins/ExecuteJob/templates/main.ejs

@@ -1,12 +0,0 @@
--- load custom layers and classes
-deepforge.initialize()
-
--- input data<% inputs.forEach(function(pair) { var input = pair[0], isNil = pair[1];%>
-local <%= input %> = <% if (isNil) { %>nil<% } else { %>require './inputs/<%= input %>'<%}}); %>
-
--- load references<% pointers.forEach(function(pair) { var pointer = pair[0], isNil = pair[1];%>
-local <%= pointer %> = <% if (isNil) { %>nil<% } else { %>require './pointers/<%= pointer %>'<%}}); %>
-local attributes = require './attributes'
-
--- main operation code for <%= name %>
-<%= code %>

src/plugins/ExecuteJob/templates/serialize.ejs

@@ -1,16 +0,0 @@
--- Serialization functions for: <%= types.map(function(type) {return type[0];}).join('\n--   ') %>
-require 'paths'
-
-local serializer = {}
-<% types.forEach(function(pair) {
-var type = pair[0],
-    fn = pair[1];
-%>
-function serializer.<%= type %> (name, data)
-    local path = 'outputs/' .. name
-    local abs_path = paths.concat('outputs', name)
-    <%= fn.replace('\n', '\n    ') %>
-end
-<% }); %>
-
-return serializer

src/plugins/ExecutePipeline/ExecutePipeline.js

@@ -175,6 +175,7 @@ define([
     ExecutePipeline.prototype.resumePipeline = function () {
         var nodes = Object.keys(this.nodes).map(id => this.nodes[id]),
             allJobs = nodes.filter(node => this.core.isTypeOf(node, this.META.Job)),
+            name = this.getAttribute(this.activeNode, 'name'),
             status,
             jobs = {
                 success: [],
@@ -208,6 +209,7 @@ define([
         return Q.all(allJobs.map(job => this.recordOldMetadata(job, true)))
             .then(() => Q.all(jobs.success.map(job => this.getOperation(job))))
             .then(ops => ops.forEach(op => this.updateJobCompletionRecords(op)))
+            .then(() => this.save(`Resuming pipeline execution: ${name}`))
             .then(() => {
                 
                 if (jobs.running.length) {  // Resume all running jobs
@@ -531,10 +533,12 @@ define([
         this.logger.info(`Setting ${jobId} status to "success"`);
         this.logger.info(`There are now ${this.runningJobs} running jobs`);
         this.logger.debug(`Making a commit from ${this.currentHash}`);
+
+        counts = this.updateJobCompletionRecords(opNode);
+
         this.save(`Operation "${name}" in ${this.pipelineName} completed successfully`)
             .then(() => {
 
-                counts = this.updateJobCompletionRecords(opNode);
                 hasReadyOps = counts.indexOf(0) > -1;
 
                 this.logger.debug(`Operation "${name}" completed. ` + 

src/plugins/GenerateArchitecture/GenerateArchitecture.js

@@ -49,10 +49,6 @@ define([
         this.LayerDict = createLayerDict(this.core, this.META);
         this.uniqueId = 2;
         this.varnames = {net: true};
-        this.definitions = [
-            'require \'nn\'',
-            'require \'rnn\''
-        ];
 
         return PluginBase.prototype.main.apply(this, arguments);
     };
@@ -76,6 +72,11 @@ define([
             result = {},
             code = '';
 
+        this.definitions = [
+            'import torch',
+            'import torch.nn as nn'
+        ];
+
         // Add an index to each layer
         layers.forEach((l, index) => l[INDEX] = index);
 
@@ -85,6 +86,7 @@ define([
             code += this.genLayerDefinitions(layers);
         }
 
+        // TODO: Define the network w/ 'class ARCHITECTURE_NAME'
         this.logger.debug('Generating architecture code...');
         code += this.genArchCode(layers);
         this.logger.debug('Prepending hoisted code...');

src/plugins/GenerateJob/GenerateJob.js

@@ -0,0 +1,547 @@
+/*globals define*/
+/*jshint node:true, browser:true*/
+
+define([
+    './templates/index',
+    'q',
+    'underscore',
+    'deepforge/Constants',
+    'deepforge/plugin/Operation',
+    'deepforge/plugin/PtrCodeGen',
+    'text!./metadata.json',
+    'plugin/PluginBase'
+], function (
+    Templates,
+    Q,
+    _,
+    CONSTANTS,
+    OperationHelpers,
+    PtrCodeGen,
+    pluginMetadata,
+    PluginBase
+) {
+    'use strict';
+
+    pluginMetadata = JSON.parse(pluginMetadata);
+    var OUTPUT_INTERVAL = 1500,
+        STDOUT_FILE = 'job_stdout.txt',
+        SKIP_ATTRIBUTES = [
+            'code',
+            'stdout',
+            'execFiles',
+            'jobId',
+            'secret',
+            CONSTANTS.LINE_OFFSET,
+            CONSTANTS.DISPLAY_COLOR
+        ];
+
+    /**
+     * Initializes a new instance of GenerateJob.
+     * @class
+     * @augments {PluginBase}
+     * @classdesc This class represents the plugin GenerateJob.
+     * @constructor
+     */
+    var GenerateJob = function () {
+        // Call base class' constructor.
+        PluginBase.call(this);
+        this.pluginMetadata = pluginMetadata;
+    };
+
+    /**
+     * Metadata associated with the plugin. Contains id, name, version, description, icon, configStructue etc.
+     * This is also available at the instance at this.pluginMetadata.
+     * @type {object}
+     */
+    GenerateJob.metadata = pluginMetadata;
+
+    // Prototypical inheritance from PluginBase.
+    GenerateJob.prototype = Object.create(PluginBase.prototype);
+    GenerateJob.prototype.constructor = GenerateJob;
+
+    /**
+     * Main function for the plugin to execute. This will perform the execution.
+     * Notes:
+     * - Always log with the provided logger.[error,warning,info,debug].
+     * - Do NOT put any user interaction logic UI, etc. inside this method.
+     * - callback always has to be called even if error happened.
+     *
+     * @param {function(string, plugin.PluginResult)} callback - the result callback
+     */
+    GenerateJob.prototype.main = function (callback) {
+        var files,
+            artifactName,
+            artifact,
+            data = {},
+            inputs,
+            name,
+            opId;
+
+        name = this.getAttribute(this.activeNode, 'name');
+        opId = this.core.getPath(this.activeNode);
+
+        return this.createOperationFiles(this.activeNode)
+            .then(results => {
+                this.logger.info('Created operation files!');
+                files = results;
+                artifactName = `${name}_${opId.replace(/\//g, '_')}-execution-files`;
+                artifact = this.blobClient.createArtifact(artifactName);
+
+                // Add the input assets
+                //   - get the metadata (name)
+                //   - add the given inputs
+                inputs = Object.keys(files.inputAssets);
+
+                return Q.all(
+                    inputs.map(input => {  // Get the metadata for each input
+                        var hash = files.inputAssets[input];
+
+                        // data asset for "input"
+                        return this.blobClient.getMetadata(hash)
+                            .fail(() => {
+                                throw Error(`BLOB_FETCH_FAILED:${input}`);
+                            });
+                    })
+                );
+            })
+            .then(mds => {
+                // Record the large files
+                var inputData = {},
+                    runsh = [
+                        '# Bash script to download data files and run job',
+                        'if [ -z "$DEEPFORGE_URL" ]; then',
+                        '  echo "Please set DEEPFORGE_URL and re-run:"',
+                        '  echo ""',
+                        '  echo "  DEEPFORGE_URL=http://my.deepforge.server.com:8080 bash run.sh"',
+                        '  echo ""',
+                        '  exit 1',
+                        'fi',
+                        'mkdir outputs\n'
+                    ].join('\n');
+
+                mds.forEach((metadata, i) => {
+                    // add the hashes for each input
+                    var input = inputs[i], 
+                        hash = files.inputAssets[input],
+                        dataDir = 'inputs/' + input + '/',
+                        dataPath = dataDir + 'data',
+                        url = this.blobClient.getRelativeDownloadURL(hash);
+
+                    inputData[dataPath] = {
+                        req: hash,
+                        cache: metadata.content
+                    };
+
+                    // Add to the run.sh file
+                    runsh += `mkdir -p ${dataDir}\n`;
+                    runsh += `wget $DEEPFORGE_URL${url} -O ${dataPath}\n`;
+                });
+
+                delete files.inputAssets;
+                files['input-data.json'] = JSON.stringify(inputData, null, 2);
+                runsh += 'python main.py';
+                files['run.sh'] = runsh;
+
+                // Add pointer assets
+                Object.keys(files.ptrAssets)
+                    .forEach(path => data[path] = files.ptrAssets[path]);
+
+                // Add the executor config
+                return this.getOutputs(this.activeNode);
+            })
+            .then(outputArgs => {
+                var config,
+                    outputs,
+                    fileList,
+                    ptrFiles = Object.keys(files.ptrAssets),
+                    file;
+
+                delete files.ptrAssets;
+                fileList = Object.keys(files).concat(ptrFiles);
+
+                outputs = outputArgs.map(pair => pair[0])
+                    .map(name => {
+                        return {
+                            name: name,
+                            resultPatterns: [`outputs/${name}`]
+                        };
+                    });
+
+                outputs.push(
+                    {
+                        name: 'stdout',
+                        resultPatterns: [STDOUT_FILE]
+                    },
+                    {
+                        name: name + '-all-files',
+                        resultPatterns: fileList
+                    }
+                );
+
+                config = {
+                    cmd: 'node',
+                    args: ['start.js'],
+                    outputInterval: OUTPUT_INTERVAL,
+                    resultArtifacts: outputs
+                };
+                files['executor_config.json'] = JSON.stringify(config, null, 4);
+
+                // Save the artifact
+                // Remove empty hashes
+                for (file in data) {
+                    if (!data[file]) {
+                        this.logger.warn(`Empty data hash has been found for file "${file}". Removing it...`);
+                        delete data[file];
+                    }
+                }
+                return artifact.addObjectHashes(data);
+            })
+            .then(() => {
+                this.logger.info(`Added ptr/input data hashes for "${artifactName}"`);
+                return artifact.addFiles(files);
+            })
+            .then(() => {
+                this.logger.info(`Added execution files for "${artifactName}"`);
+                return artifact.save();
+            })
+            .then(hash => {
+                this.result.setSuccess(true);
+                this.result.addArtifact(hash);
+                callback(null, this.result);
+            })
+            .fail(err => {
+                this.result.setSuccess(false);
+                callback(err, this.result);
+            });
+    }; 
+
+    GenerateJob.prototype.createOperationFiles = function (node) {
+        var files = {};
+        // For each operation, generate the output files:
+        //   inputs/<arg-name>/init.py  (respective data deserializer)
+        //   pointers/<name>/init.py  (result of running the main plugin on pointer target - may need a rename)
+        //   outputs/<name>/  (make dirs for each of the outputs)
+        //   outputs/init.py  (serializers for data outputs)
+        //
+        //   attributes.py (returns py table of operation attributes)
+        //   init.py (main file -> calls main and serializes outputs)
+        //   <name>.py (entry point -> calls main operation code)
+
+        // add the given files
+        this.logger.info('About to generate operation execution files');
+        return this.createEntryFile(node, files)
+            .then(() => this.createClasses(node, files))
+            .then(() => this.createCustomLayers(node, files))
+            .then(() => this.createInputs(node, files))
+            .then(() => this.createMainFile(node, files))
+            .then(() => {
+                this.createAttributeFile(node, files);
+                return Q.ninvoke(this, 'createPointers', node, files);
+            })
+            .fail(err => {
+                this.logger.error(err);
+                throw err;
+            });
+    };
+
+    GenerateJob.prototype.createEntryFile = function (node, files) {
+        this.logger.info('Creating deepforge.py file...');
+        files['deepforge.py'] = _.template(Templates.DEEPFORGE)(CONSTANTS);
+        return this.getOutputs(node)
+            .then(outputs => {
+                var name = this.getAttribute(node, 'name'),
+                    content = {};
+
+                // inputs and outputs
+                content.name = name;
+                content.outputs = outputs.map(output => output[0]);
+
+                // Create the deepforge file
+            });
+    };
+
+    GenerateJob.prototype.createClasses = function (node, files) {
+        var metaDict = this.core.getAllMetaNodes(this.rootNode),
+            isClass,
+            metanodes,
+            classNodes,
+            inheritanceLvl = {},
+            code;
+
+        this.logger.info('Creating custom layer file...');
+        metanodes = Object.keys(metaDict).map(id => metaDict[id]);
+        isClass = this.getTypeDictFor('Complex', metanodes);
+
+        classNodes = metanodes.filter(node => {
+            var base = this.core.getBase(node),
+                baseId,
+                count = 1;
+
+            // Count the sets back to a class node
+            while (base) {
+                baseId = this.core.getPath(base);
+                if (isClass[baseId]) {
+                    inheritanceLvl[this.core.getPath(node)] = count;
+                    return true;
+                }
+                base = this.core.getBase(base);
+                count++;
+            }
+
+            return false;
+        });
+
+        // Get the code definitions for each
+        // Sort by levels of inheritance...
+        code = classNodes.sort((a, b) => {
+            var aId = this.core.getPath(a),
+                bId = this.core.getPath(b);
+
+            return inheritanceLvl[aId] > inheritanceLvl[bId];
+        }).map(node =>
+            // FIXME: update this
+            `require './${this.getAttribute(node, 'name')}.py'`
+        ).join('\n');
+
+        // Create the class files
+        classNodes.forEach(node => {
+            var name = this.getAttribute(node, 'name');
+            files[`classes/${name}.py`] = this.getAttribute(node, 'code');
+        });
+
+        // Create the custom layers file
+        files['classes/init.py'] = code;
+    };
+
+    GenerateJob.prototype.getTypeDictFor = function (name, metanodes) {
+        var isType = {};
+        // Get all the custom layers
+        for (var i = metanodes.length; i--;) {
+            if (this.getAttribute(metanodes[i], 'name') === name) {
+                isType[this.core.getPath(metanodes[i])] = true;
+            }
+        }
+        return isType;
+    };
+
+    // TODO: update this to nn modules
+    GenerateJob.prototype.createCustomLayers = function (node, files) {
+        var metaDict = this.core.getAllMetaNodes(this.rootNode),
+            isCustomLayer,
+            metanodes,
+            customLayers,
+            code;
+
+        this.logger.info('Creating custom layer file...');
+        metanodes = Object.keys(metaDict).map(id => metaDict[id]);
+        isCustomLayer = this.getTypeDictFor('CustomLayer', metanodes);
+
+        customLayers = metanodes.filter(node =>
+            this.core.getMixinPaths(node).some(id => isCustomLayer[id]));
+
+        // Get the code definitions for each
+        code = 'require \'nn\'\n\n' + customLayers
+            .map(node => this.getAttribute(node, 'code')).join('\n');
+
+        // Create the custom layers file
+        files['custom-layers.py'] = code;
+    };
+
+    GenerateJob.prototype.getConnectionContainer = function () {
+        var container = this.core.getParent(this.activeNode);
+
+        if (this.isMetaTypeOf(container, this.META.Job)) {
+            container = this.core.getParent(container);
+        }
+
+        return container;
+    };
+
+    GenerateJob.prototype.getInputPortsFor = function (nodeId) {
+        var container = this.getConnectionContainer();
+
+        // Get the connections to this node
+        return this.core.loadChildren(container)
+            .then(children => {
+                return children.filter(child =>
+                    this.core.getPointerPath(child, 'dst') === nodeId)
+                    .map(conn => this.core.getPointerPath(conn, 'src'))[0];
+            });
+    };
+
+    GenerateJob.prototype.createInputs = function (node, files) {
+        var tplContents,
+            inputs;
+
+        this.logger.info('Retrieving inputs and deserialize fns...');
+        return this.getInputs(node)
+            .then(allInputs => {
+                // For each input, match the connection with the input name
+                //   [ name, type ] => [ name, type, node ]
+                //
+                // For each input,
+                //  - create the deserializer
+                //  - put it in inputs/<name>/init.py
+                //  - copy the data asset to /inputs/<name>/init.py
+                inputs = allInputs
+                    .filter(pair => !!this.getAttribute(pair[2], 'data'));  // remove empty inputs
+
+                files['start.js'] = _.template(Templates.START)({
+                    CONSTANTS,
+                    inputs: inputs.map(pair => pair[0])
+                });
+                files.inputAssets = {};  // data assets
+                return Q.all(inputs.map(pair => {
+                    var name = pair[0],
+                        node = pair[2],
+                        nodeId = this.core.getPath(node);
+
+                    // Get the deserialize function. First, try to get it from
+                    // the source method (this guarantees that the correct
+                    // deserialize method is used despite any auto-upcasting
+                    return this.getInputPortsFor(nodeId)
+                        .then(fromNodeId => this.core.loadByPath(this.rootNode, fromNodeId || nodeId))
+                        .then(fromNode => {
+                            var deserFn,
+                                base,
+                                className;
+
+                            deserFn = this.getAttribute(fromNode, 'deserialize');
+
+                            if (this.isMetaTypeOf(node, this.META.Complex)) {
+                                // Complex objects are expected to define their own
+                                // (static) deserialize factory method
+                                base = this.core.getMetaType(node);
+                                className = this.getAttribute(base, 'name');
+                                deserFn = `return ${className}.deserialize(path)`;
+                            }
+
+                            return {
+                                name: name,
+                                code: deserFn
+                            };
+                        });
+                }));
+            })
+            .then(_tplContents => {
+                tplContents = _tplContents;
+                inputs.forEach(pair => {
+                    var hash = this.getAttribute(pair[2], 'data');
+                    files.inputAssets[pair[0]] = hash;
+                });
+
+                return files;
+            });
+    };
+
+    GenerateJob.prototype.createMainFile = function (node, files) {
+        this.logger.info('Creating main file...');
+        var content = {};
+        return this.getInputs(node)
+            .then(inputs => {
+                var name = this.getAttribute(node, 'name'),
+                    code = this.getAttribute(node, 'code'),
+                    pointers = this.core.getPointerNames(node).filter(ptr => ptr !== 'base');
+
+                content.name = name;
+
+                // Get input data arguments
+                content.inputs = inputs
+                    .map(pair => [pair[0], !this.getAttribute(pair[2], 'data')]);  // remove empty inputs
+
+                // Defined variables for each pointers
+                content.pointers = pointers
+                    .map(id => [id, this.core.getPointerPath(node, id) === null]);
+
+                // Add remaining code
+                content.code = code;
+                return this.getOutputs(node);
+            })
+            .then(outputs => {
+                content.outputs = outputs.map(output => output[0]);
+
+                files['main.py'] = _.template(Templates.MAIN)(content);
+                files['operations.py'] = content.code;
+
+                // Set the line offset
+                var lineOffset = 0;
+                this.setAttribute(node, CONSTANTS.LINE_OFFSET, lineOffset);
+            });
+    };
+
+    GenerateJob.prototype.getLineOffset = function (main, snippet) {
+        var i = main.indexOf(snippet),
+            lines = main.substring(0, i).match(/\n/g);
+
+        return lines ? lines.length : 0;
+    };
+
+    GenerateJob.prototype.createAttributeFile = function (node, files) {
+        var numOrBool = /^(-?\d+\.?\d*((e|e-)\d+)?|(true|false))$/,
+            isBool = /^(true|false)$/,
+            table;
+
+        this.logger.info('Creating attributes file...');
+        table = '{\n\t' + this.core.getAttributeNames(node)
+            .filter(attr => SKIP_ATTRIBUTES.indexOf(attr) === -1)
+            .map(name => {
+                var value = this.getAttribute(node, name);
+                if (!numOrBool.test(value)) {
+                    value = `"${value}"`;
+                }
+                if (isBool.test(value)) {  // Convert to python bool
+                    value = value.toString();
+                    value = value[0].toUpperCase() + value.slice(1);
+                }
+                return [`'${name}'`, value];
+            })
+            .map(pair => pair.join(': '))
+            .join(',\n    ') + '\n}';
+
+        files['attributes.py'] = `# attributes of ${this.getAttribute(node, 'name')}\nattributes = ${table}`;
+    };
+
+    GenerateJob.prototype.createPointers = function (node, files, cb) {
+        var pointers,
+            nIds;
+
+        this.logger.info('Creating pointers file...');
+        pointers = this.core.getPointerNames(node)
+            .filter(name => name !== 'base')
+            .filter(id => this.core.getPointerPath(node, id) !== null);
+
+        nIds = pointers.map(p => this.core.getPointerPath(node, p));
+        files.ptrAssets = {};
+        Q.all(
+            nIds.map(nId => this.getPtrCodeHash(nId))
+        )
+        .then(resultHashes => {
+            var name = this.getAttribute(node, 'name');
+            this.logger.info(`Pointer generation for ${name} FINISHED!`);
+            resultHashes.forEach((hash, index) => {
+                files.ptrAssets[`pointers/${pointers[index]}/init.py`] = hash;
+            });
+            return cb(null, files);
+        })
+        .fail(e => {
+            this.logger.error(`Could not generate pointer files for ${this.getAttribute(node, 'name')}: ${e.toString()}`);
+            return cb(e);
+        });
+    };
+
+    GenerateJob.prototype.getAttribute = function (node, attr) {
+        return this.core.getAttribute(node, attr);
+    };
+
+    GenerateJob.prototype.setAttribute = function (node, attr, value) {
+        return this.core.setAttribute(node, attr, value);
+    };
+
+    _.extend(
+        GenerateJob.prototype,
+        OperationHelpers.prototype,
+        PtrCodeGen.prototype
+    );
+
+    return GenerateJob;
+});

src/plugins/GenerateJob/metadata.json

@@ -0,0 +1,14 @@
+{
+  "id": "GenerateJob",
+  "name": "GenerateJob",
+  "version": "0.1.0",
+  "description": "",
+  "icon": {
+    "class": "glyphicon glyphicon-cog",
+    "src": ""
+  },
+  "disableServerSideExecution": false,
+  "disableBrowserSideExecution": false,
+  "writeAccessRequired": false,
+  "configStructure": []
+}

src/plugins/GenerateJob/templates/deepforge.ejs

@@ -0,0 +1,105 @@
+# Instantiate the deepforge object
+
+#class deepforge()
+# function deepforge.initialize()
+    # require 'nn'
+    # require 'rnn'
+    # require './classes/init'
+    # require './custom-layers'
+# end
+
+# function deepforge.id()
+    # if __deepforge_id == nil then
+        # __deepforge_id = 0
+    # end
+    # __deepforge_id = __deepforge_id + 1
+    # return __deepforge_id
+# end
+# 
+# function deepforge._cmd(...)
+    # local cmd = '<%= START_CMD %>'
+    # local arg = {...}
+    # local n = #arg
+    # for i=1,n do
+        # cmd = cmd .. ' ' .. tostring(arg[i])
+    # end
+    # print(cmd .. ' ')  # guarantee ends w/ space
+# end
+# 
+# # Graph support
+# Graph = torch.class('deepforge.Graph')
+# 
+# function Graph:__init(name)
+    # self.id = deepforge.id()
+    # self.name = name
+    # deepforge._cmd('<%= GRAPH_CREATE %>', self.id, name)
+# end
+# 
+# _Line = torch.class('deepforge._Line')
+# 
+# function _Line:__init(graphId, name, opts)
+    # self.id = deepforge.id()
+    # self.name = name
+    # deepforge._cmd('<%= GRAPH_CREATE_LINE %>', graphId, self.id, name)
+# end
+# 
+# function _Line:add(x, y)
+    # assert(type(x) == "number" and type(y) == "number", "adding point (" .. tostring(x) .. ", " .. tostring(y) .. ") to " .. self.name .. " failed: expected (number, number)")
+    # deepforge._cmd('<%= GRAPH_PLOT %>', self.id, x, y)
+# end
+# 
+# function Graph:line(name, opts)
+    # return deepforge._Line(self.id, name, opts)
+# end
+# 
+# function Graph:xlabel(name)
+    # deepforge._cmd('<%= GRAPH_LABEL_AXIS.X %>', self.id, name)
+# end
+# 
+# function Graph:ylabel(name)
+    # deepforge._cmd('<%= GRAPH_LABEL_AXIS.Y %>', self.id, name)
+# end
+# 
+# # Image support
+# local function saveImage(name, tensor)
+    # require 'image'
+    # require 'paths'
+# 
+    # # save it in the tmp directory
+    # local filename = name .. '.png'
+    # local path = paths.concat('metadata', filename)
+# 
+    # if paths.dir('metadata') == nil then
+       # paths.mkdir('metadata')
+    # end
+# 
+    # image.save(path, tensor)
+# end
+# 
+# function deepforge.image(name, tensor)
+    # saveImage(name, tensor)
+    # deepforge._cmd("<%= IMAGE.BASIC %>", deepforge.id(), name)
+# end
+# 
+# Image = torch.class('deepforge.Image')
+# function Image:__init(name, tensor)
+    # self.id = deepforge.id()
+    # self.name = name
+# 
+    # if tensor ~= nil then
+        # saveImage(name, tensor)
+        # deepforge._cmd('<%= IMAGE.CREATE %>', self.id, self.name)
+    # end
+# end
+# 
+# function Image:update(tensor)
+    # saveImage(self.name, tensor)
+    # deepforge._cmd('<%= IMAGE.UPDATE %>', self.id, self.name)
+# end
+# 
+# function Image:title(name)
+    # self.name = name
+    # deepforge._cmd('<%= IMAGE.NAME %>', self.id, self.name)
+# end
+# 
+# return deepforge

src/plugins/GenerateJob/templates/entry.ejs

@@ -0,0 +1,7 @@
+import main
+import pickle
+
+# run the <%= name %> and serialize the results
+
+# serialize by type
+import outputs

src/plugins/GenerateJob/templates/index.js

@@ -1,14 +1,12 @@
 /*globals define*/
 define([
     'text!./start.ejs',
-    'text!./entry.ejs',
     'text!./main.ejs',
     'text!./deepforge.ejs',
     'text!./serialize.ejs',
     'text!./deserialize.ejs'
 ], function(
     START,
-    ENTRY,
     MAIN,
     DEEPFORGE,
     SERIALIZE,
@@ -17,7 +15,6 @@ define([
 
     return {
         START,
-        ENTRY,
         MAIN,
         SERIALIZE,
         DEEPFORGE,

src/plugins/GenerateJob/templates/main.ejs

@@ -0,0 +1,29 @@
+import pickle
+# load custom layers and classes
+# from deepforge import deepforge
+
+# deepforge.initialize()
+
+# input data<% inputs.forEach(function(pair) { var input = pair[0], isNil = pair[1];%>
+<%= input %> = <% if (isNil) { %>None<% } else { %>pickle.load(open('./inputs/<%= input %>/data', 'rb')) <%}}); %>
+
+# load references<% pointers.forEach(function(pair) { var pointer = pair[0], isNil = pair[1];%>
+from pointers import <%= pointer %>
+<%}); %>
+
+from operations import <%= name %>Operation
+from attributes import attributes
+
+# main operation code for <%= name %>
+operation = <%= name %>Operation()
+print('\n############### Running "<%= name.replace(/'/g, '\\\'') %>" Operation ############### ')
+<%= outputs.length ? outputs.join(', ') : 'result' %> = operation.execute(<%= inputs.map(function(pair) {
+    var name = pair[0],
+        isNone = pair[1];
+    return isNone ? 'None' : name;
+})%>)
+print('############### "<%= name.replace(/'/g, '\\\'') %>" Operation Complete! ###############')
+
+<% outputs.forEach(name => { %>
+pickle.dump(<%= name %>, open('outputs/<%= name %>', 'wb'))
+<% }); %>

src/plugins/GenerateJob/templates/serialize.ejs

@@ -0,0 +1,17 @@
+# Serialization functions for: <%= types.map(function(type) {return type[0];}).join('\n--   ') %>
+
+serializer = {}
+<% types.forEach(function(pair) {
+var type = pair[0],
+    fn = pair[1],
+    safeType = type.replace(/[^a-zA-Z\d_]/g, '_');
+
+%>
+
+def <%= safeType %> (name, data):
+    path = 'outputs/' .. name
+    <%= fn.replace('\n', '\n    ') %>
+
+serializer['<%= type %>'] = <%= safeType %>
+
+<% }); %>

src/plugins/GenerateJob/templates/start.ejs

@@ -11,8 +11,8 @@ var spawn = require('child_process').spawn,
 ['error', 'warn', 'info', 'log', 'debug'].forEach(method => logger[method] = log);
 
 // Get the BlobClient...
-var COMMAND_PREFIX = '<%= START_CMD %>',
-    IMAGE = '<%= IMAGE.PREFIX %>',
+var COMMAND_PREFIX = '<%= CONSTANTS.START_CMD %>',
+    IMAGE = '<%= CONSTANTS.IMAGE.PREFIX %>',
     requirejs = require('webgme').requirejs,
     remainingImageCount = 0,
     exitCode = null;
@@ -35,14 +35,23 @@ requirejs([
 
     // Create CACHE_DIR if it doesn't exist
     var prepareCache = function() {
-    var dirs = CACHE_DIR.replace(/\/$/, '').split('/'),
-        cacheParent;
+        var dirs = CACHE_DIR.replace(/\/$/, '').split('/'),
+            cacheParent;
 
         dirs.pop();
         cacheParent = dirs.join('/');
         return makeIfNeeded(cacheParent).then(() => makeIfNeeded(CACHE_DIR));
     };
 
+    var prepareInputsOutputs = function() {
+        var dirs = ['inputs', <% inputs.forEach(function(input) { %>
+            'inputs/<%= input %>',
+            <% }) %>
+            'outputs'];
+        return Q.all(dirs.map(dir => makeIfNeeded(dir)));
+    };
+
+
     var makeIfNeeded = function(dir) {
         var deferred = Q.defer(),
             job;
@@ -109,7 +118,7 @@ requirejs([
     var onStderr = function(data) {
         var text = data.toString();
         // Filter out directory label from stack traces
-        process.stdout.write(text.replace(/\.\.\.\/.*\/(main|deepforge|init).lua/g, '$1'));
+        process.stdout.write(text.replace(/\.\.\.\/.*\/(main|deepforge|init).py/g, '$1'));
     };
 
     var onStdout = function(data) {
@@ -213,10 +222,10 @@ requirejs([
             job = null;
             log(`killing process group: ${pid}`);
             process.kill(-pid, 'SIGTERM');
-            if (exitCode !== null) {
-                log(`exiting w/ code ${exitCode}`);
-                process.exit(exitCode);
-            }
+        }
+        if (exitCode !== null) {
+            log(`exiting w/ code ${exitCode}`);
+            process.exit(exitCode);
         }
     };
 
@@ -229,17 +238,19 @@ requirejs([
         cleanup();
         process.exit(130);
     });
-    process.on('uncaughtException', () => {
+    process.on('uncaughtException', err => {
         log('received "uncaughtException" event')
+        log(err);
         cleanup();
     });
 
     // Request the data from the blob
     prepareCache()
+        .then(prepareInputsOutputs)
         .then(() => Q.all(inputPaths.map(ipath => getData(ipath, inputData[ipath]))))
         .then(() => {
-            // Run 'th init.lua' and merge the stdout, stderr
-            job = spawn('th', ['init.lua'], {detached: true});
+            // Run 'python main.py' and merge the stdout, stderr
+            job = spawn('python', ['main.py'], {detached: true});
             job.stdout.on('data', onStdout);
             job.stderr.on('data', onStderr);
             job.on('close', code => {

src/seeds/nn/version.txt

@@ -1 +1 @@
-0.5.0
+1.0.3

src/seeds/pipeline/version.txt

@@ -1 +1 @@
-0.6.0
+0.7.1

src/visualizers/panels/LayerEditor/LayerEditorControl.js

@@ -15,7 +15,7 @@ define([
 
     'use strict';
 
-    var NO_CODE_MESSAGE = '-- <%= name %> is not an editable layer!',
+    var NO_CODE_MESSAGE = '<%= name %> is not an editable layer!',
         LayerEditorControl;
 
     LayerEditorControl = function (options) {
@@ -45,10 +45,10 @@ define([
                 // Retrieve the template from the mixin
                 template = node.getMixinPaths()
                     .map(id => this._client.getNode(id).getAttribute('code'))
-                    .find(code => !!code) || NO_CODE_MESSAGE;
+                    .find(code => !!code) || this.comment(NO_CODE_MESSAGE);
             }
         } else {
-            template = NO_CODE_MESSAGE;
+            template = this.comment(NO_CODE_MESSAGE);
         }
 
         if (template) {

src/visualizers/panels/OperationCodeEditor/OperationCodeEditorControl.js

@@ -3,13 +3,17 @@
 
 define([
     'panels/TextEditor/TextEditorControl',
+    'text!./boilerplate.ejs',
     'deepforge/viz/OperationControl',
+    'deepforge/OperationCode',
     'deepforge/viz/Execute',
     'deepforge/Constants',
     'underscore'
 ], function (
     TextEditorControl,
+    CodeTemplate,
     OperationControl,
+    OperationCode,
     Execute,
     CONSTANTS,
     _
@@ -18,6 +22,7 @@ define([
     'use strict';
 
     var OperationCodeEditorControl;
+    var GenerateBoilerplate = _.template(CodeTemplate);
 
     OperationCodeEditorControl = function (options) {
         options.attributeName = 'code';
@@ -48,6 +53,11 @@ define([
         desc.inputs = this.getOperationInputs(node).map(id => this.formatIO(id));
         desc.outputs = this.getOperationOutputs(node).map(id => this.formatIO(id));
         desc.references = node.getPointerNames().filter(name => name !== 'base');
+
+        // Create the boilerplate operation code, if applicable
+        if (!desc.ownText) {
+            desc.text = GenerateBoilerplate(desc);
+        }
         return desc;
     };
 
@@ -68,6 +78,51 @@ define([
         }
     };
 
+    OperationCodeEditorControl.prototype.saveTextFor = function (id, code) {
+        try {
+            // Parse the operation implementation and detect change in inputs/outputs
+            // TODO: Update this to use the code object
+            var operation = new OperationCode(code),
+                oldInputs = this.getDataNames(this._currentNodeId, true),
+                currentInputs = operation.getInputs().map(input => input.name),
+                name = this._client.getNode(this._currentNodeId).getAttribute('name'),
+                newInputs,
+                rmInputs,
+                oldOutputs = this.getDataNames(this._currentNodeId),
+                currentOutputs = operation.getOutputs().map(input => input.name),
+                newOutputs,
+                rmOutputs;
+
+            // Check for input nodes to remove
+            if (currentInputs[0] === 'self') currentInputs.shift();
+            newInputs = _.difference(currentInputs, oldInputs);
+            rmInputs = _.difference(oldInputs, currentInputs);
+            newOutputs = _.difference(currentOutputs, oldOutputs);
+            rmOutputs = _.difference(oldOutputs, currentOutputs);
+
+            if (rmInputs.length || newInputs.length || rmOutputs.length || newOutputs.length) {
+                var msg = `Updating operation implementation for ${name}`;
+
+                this._client.startTransaction(msg);
+                TextEditorControl.prototype.saveTextFor.call(this, id, code, true);
+
+                // update the inputs
+                rmInputs.forEach(input => this.removeInputData(this._currentNodeId, input));
+                newInputs.map(input => this.addInputData(this._currentNodeId, input));
+
+                // update the outputs
+                rmOutputs.forEach(output => this.removeOutputData(this._currentNodeId, output));
+                newOutputs.map(output => this.addOutputData(this._currentNodeId, output));
+                this._client.completeTransaction();
+            } else {
+                return TextEditorControl.prototype.saveTextFor.call(this, id, code);
+            }
+        } catch (e) {
+            this._logger.debug(`failed parsing operation: ${e}`);
+            return TextEditorControl.prototype.saveTextFor.call(this, id, code);
+        }
+    };
+
     OperationCodeEditorControl.prototype.getOperationAttributes = function () {
         var node = this._client.getNode(this._currentNodeId),
             attrs = node.getValidAttributeNames(),

src/visualizers/panels/OperationCodeEditor/boilerplate.ejs

@@ -0,0 +1,4 @@
+class <%= name %>Operation():
+
+    def execute(self):
+        # Execute your operation here!

src/visualizers/panels/OperationEditor/OperationEditorPanel.js

@@ -71,6 +71,33 @@ define([
         this.setTitle(name || '');
     };
 
+    OperationEditorPanel.prototype.editTitle = function () {
+        this.$panelHeaderTitle.editInPlace({
+            css: {
+                'z-index': 1000
+            },
+            onChange: (oldValue, newValue) => {
+                var nodeId = this.currentNodeId(),
+                    type = this.currentBaseName(),
+                    words = newValue.split(' '),
+                    msg;
+
+                if (words.length > 1) {
+                    newValue = words.map(word => word[0].toUpperCase() + word.substring(1)).join('');
+                }
+
+                newValue = newValue.replace(/Operation$/, '');
+                msg = `Renamed ${type}: ${oldValue} -> ${newValue}`;
+
+                if (!/^\s*$/.test(newValue)) {
+                    this._client.startTransaction(msg);
+                    this._client.setAttribute(nodeId, 'name', newValue);
+                    this._client.completeTransaction();
+                }
+            }
+        });
+    };
+
     OperationEditorPanel.prototype.getPanels = function () {
         return [InterfaceEditor, CodeEditor];
     };

src/visualizers/panels/OperationInterfaceEditor/OperationInterfaceEditorControl.EventHandlers.js

@@ -1,7 +1,11 @@
 /*globals define*/
 define([
+    'panels/EasyDAG/EasyDAGControl.WidgetEventHandlers',
+    'deepforge/OperationCode',
     './Colors'
 ], function(
+    EasyDAGControlEventHandlers,
+    OperationCode,
     COLORS
 ) {
     'use strict';
@@ -66,36 +70,14 @@ define([
             .filter(node => !node.isAbstract());
     };
 
-    OperationInterfaceEditorEvents.prototype.getValidSuccessors = function(nodeId, isInput) {
-        var dataTypeIds;
-
+    OperationInterfaceEditorEvents.prototype.getValidSuccessors = function(nodeId) {
         if (nodeId !== this._currentNodeId) {
             return [];
         }
 
-        // Return all data types in the meta
-        // If input, include abstract types
-        dataTypeIds = this.allDataTypeIds(isInput);
-
-        return dataTypeIds.map(id => {
-            return {
-                node: this._getObjectDescriptor(id)
-            };
-        });
-    };
-
-    OperationInterfaceEditorEvents.prototype._getDataName = function(cntrId, typeId) {
-        var otherIds = this._client.getNode(cntrId).getChildrenIds(),
-            otherNames = otherIds.map(id => this._client.getNode(id).getAttribute('name')),
-            baseName = this._client.getNode(typeId).getAttribute('name').toLowerCase(),
-            name = baseName,
-            i = 1;
-
-        while (otherNames.indexOf(name) !== -1) {
-            i++;
-            name = baseName + '_' + i;
-        }
-        return name;
+        return [{
+            node: this._getObjectDescriptor(this.getDataTypeId())
+        }];
     };
 
     OperationInterfaceEditorEvents.prototype.getRefName = function(node, basename) {
@@ -193,26 +175,92 @@ define([
         this._client.completeTransaction();
     };
 
-    OperationInterfaceEditorEvents.prototype._createConnectedNode = function(typeId, isInput) {
+    OperationInterfaceEditorEvents.prototype._createConnectedNode = function(typeId, isInput, baseName) {
         var node = this._client.getNode(this._currentNodeId),
             name = node.getAttribute('name'),
-            cntrs = node.getChildrenIds(),
-            cntrType = isInput ? 'Inputs' : 'Outputs',
-            cntrId = cntrs.find(id => this.hasMetaName(id, cntrType)),
-            dataName = this._getDataName(cntrId, typeId),
+            msg = `Updating the interface of ${name}`,
+            code = node.getAttribute('code'),
+            id,
+            operation,
+            dataName;
+
+        // Update the source code if the inputs/outputs changed
+        // we know that we are adding a node, so we don't need to do 
+        // the comparing and diffing current vs new
+
+        this._client.startTransaction(msg);
+        id = this.createIONode(this._currentNodeId, typeId, isInput, baseName, true);
+        dataName = this._client.getNode(id).getAttribute('name');
+
+        try {
+            operation = new OperationCode(code);
+            if (isInput) {
+                operation.addInput(dataName);
+            } else {
+                operation.addOutput(dataName);
+            }
+
+            this._client.setAttribute(this._currentNodeId, 'code', operation.getCode());
+        } catch(e) {
+            this.logger.debug(`could not update the code - invalid python!: ${e}`);
+        }
+        this._client.completeTransaction();
+
+        return id;
+    };
+
+    OperationInterfaceEditorEvents.prototype._deleteNode = function(nodeId) {
+        var dataName = this._client.getNode(nodeId).getAttribute('name'),
+            node = this._client.getNode(this._currentNodeId),
+            name = node.getAttribute('name'),
+            isInput = this.isInputData(nodeId),
+            msg = `Updating the interface of ${name}`,
+            code = node.getAttribute('code'),
+            operation = new OperationCode(code);
+
+        // If the input name is used in the code, maybe just comment it out in the args
+        this._client.startTransaction(msg);
+        try {
+            if (isInput) {
+                operation.removeInput(dataName);
+            } else {
+                operation.removeOutput(dataName);
+            }
+            this._client.setAttribute(this._currentNodeId, 'code', operation.getCode());
+        } catch(e) {
+            this.logger.debug(`could not update the code - invalid python!: ${e}`);
+        }
+        this._client.deleteNode(nodeId);
+        //EasyDAGControlEventHandlers.prototype._deleteNode.apply(this, nodeId, true);
+        this._client.completeTransaction();
+    };
+
+    OperationInterfaceEditorEvents.prototype._saveAttributeForNode = function(nodeId, attr, value) {
+        // If nodeId is an input data node, rename the input
+        // If nodeId is an output data node, rename the output
+        var isDataNode = nodeId.indexOf(this._currentNodeId) === 0,
+            node = this._client.getNode(this._currentNodeId),
+            code = node.getAttribute('code'),
             msg;
 
-        msg = `Adding ${isInput ? 'input' : 'output'} "${dataName}" to ${name} interface`;
-        this._client.startTransaction(msg);
-        var id = this._client.createNode({
-            parentId: cntrId,
-            baseId: typeId
-        });
+        if (isDataNode && attr === 'name') {  // rename input/output
+            var operation = new OperationCode(code),
+                dataNode = this._client.getNode(nodeId),
+                oldName = dataNode.getAttribute(attr);
 
-        // Set the name of the new input
-        this._client.setAttribute(id, 'name', dataName);
+            operation.rename(oldName, value);
 
-        this._client.completeTransaction();
+            msg = `Renaming ${oldName}->${value} in ${name}`;
+            this._client.startTransaction(msg);
+            EasyDAGControlEventHandlers.prototype._saveAttributeForNode.apply(this, arguments);
+            this._client.setAttribute(this._currentNodeId, 'code', operation.getCode());
+            this._client.completeTransaction();
+        } else if (nodeId === this._currentNodeId) {  // edit operation attributes
+            // TODO: rename operation
+            // TODO: set operation attribute default
+            console.log('setting attr', arguments);
+            EasyDAGControlEventHandlers.prototype._saveAttributeForNode.apply(this, arguments);
+        }
     };
 
     return OperationInterfaceEditorEvents;

src/visualizers/panels/OperationInterfaceEditor/OperationInterfaceEditorControl.js

@@ -10,7 +10,6 @@ define([
     'panels/EasyDAG/EasyDAGControl',
     'js/Constants',
     'deepforge/Constants',
-    'deepforge/lua',
     'deepforge/viz/OperationControl',
     './OperationInterfaceEditorControl.EventHandlers',
     './Colors',
@@ -19,7 +18,6 @@ define([
     EasyDAGControl,
     GME_CONSTANTS,
     CONSTANTS,
-    luajs,
     OperationControl,
     OperationInterfaceEditorControlEvents,
     COLORS,
@@ -229,20 +227,13 @@ define([
 
             code = this._client.getNode(this._currentNodeId).getAttribute('code');
             try {
-                ast = luajs.parser.parse(code);
-                for (var i = variableIds.length; i--;) {
-                    wasUsed = this._usage[variableIds[i]];
-                    name = this._client.getNode(variableIds[i]).getAttribute('name');
-
-                    isUsed = this._inputs[variableIds[i]] ?
-                        this.isUsedInput(name, ast) :
-                        this.isUsedOutput(name, ast);
-                    if (isUsed !== wasUsed) {
-                        this._onUpdate(variableIds[i]);
-                    }
-                }
+                // Parse the operation implementation for visual cues
+                // TODO
+                // Parse the operation implementation and detect change in inputs/outputs
+                //var schema = OperationParser.parse(code);
+                //console.log(schema);
             } catch (e) {
-                this._logger.debug(`failed parsing lua: ${e}`);
+                this._logger.debug(`failed parsing operation: ${e}`);
             }
 
         } else if (this.containedInCurrent(gmeId) && this.hasMetaName(gmeId, 'Data')) {
@@ -375,11 +366,13 @@ define([
 
     ////////////////////// Unused input checking //////////////////////
     OperationInterfaceEditorControl.prototype.isUsedInput = function(name, ast) {
-        return this._isUsed(name, true, ast);
+        return true;
+        //return this._isUsed(name, true, ast);
     };
 
     OperationInterfaceEditorControl.prototype.isUsedOutput = function(name, ast) {
-        return this._isUsed(name, false, ast);
+        return true;
+        //return this._isUsed(name, false, ast);
     };
 
     OperationInterfaceEditorControl.prototype._isUsed = function(name, isInput, ast) {
@@ -390,8 +383,9 @@ define([
         // verify that it is not used only in the left side of an assignment
         if (hasText) {
             try {
-                ast = ast || luajs.parser.parse(code);
-                return isInput ? this.isUsedVariable(name, ast) : this.isReturnValue(name, ast);
+                return true;
+                //ast = ast || luajs.parser.parse(code);
+                //return isInput ? this.isUsedVariable(name, ast) : this.isReturnValue(name, ast);
             } catch(e) {
                 this._logger.debug(`failed parsing lua: ${e}`);
                 return null;
@@ -406,19 +400,20 @@ define([
         var isUsed = false,
             checker;
 
-        checker = luajs.codegen.traverse((curr, parent) => {
-            if (curr.type === 'variable' && curr.val === name) {
-                // Ignore if it is being assigned...
-                if (parent.type === 'stat.assignment') {
-                    isUsed = isUsed || parent.right.indexOf(curr) !== -1;
-                } else {
-                    isUsed = true;
-                }
-            }
-            return curr;
-        });
+        return true;
+        //checker = luajs.codegen.traverse((curr, parent) => {
+            //if (curr.type === 'variable' && curr.val === name) {
+                //// Ignore if it is being assigned...
+                //if (parent.type === 'stat.assignment') {
+                    //isUsed = isUsed || parent.right.indexOf(curr) !== -1;
+                //} else {
+                    //isUsed = true;
+                //}
+            //}
+            //return curr;
+        //});
 
-        checker(node);
+        //checker(node);
         return isUsed;
     };
 

src/visualizers/panels/OperationInterfaceEditor/OperationInterfaceEditorPanel.js

@@ -4,14 +4,17 @@
  * Generated by VisualizerGenerator 1.7.0 from webgme on Tue May 31 2016 09:16:24 GMT-0500 (CDT).
  */
 
-define(['js/PanelBase/PanelBaseWithHeader',
+define([
+    'js/PanelBase/PanelBaseWithHeader',
     'js/PanelManager/IActivePanel',
     'widgets/OperationInterfaceEditor/OperationInterfaceEditorWidget',
     './OperationInterfaceEditorControl'
-], function (PanelBaseWithHeader,
-             IActivePanel,
-             OperationInterfaceEditorWidget,
-             OperationInterfaceEditorControl) {
+], function (
+    PanelBaseWithHeader,
+    IActivePanel,
+    OperationInterfaceEditorWidget,
+    OperationInterfaceEditorControl
+) {
     'use strict';
 
     var OperationInterfaceEditorPanel;

src/visualizers/panels/PipelineEditor/PipelineEditorControl.js

@@ -660,9 +660,9 @@ define([
             items = this._client.getAllMetaNodes()
                 .filter(node => node.isTypeOf(criterionId));
 
-            return items.map(id => {
+            return items.map(node => {
                 return {
-                    node: this._getObjectDescriptor(id)
+                    node: this._getObjectDescriptor(node.getId())
                 };
             });
         } else {

src/visualizers/widgets/ArchEditor/ArchEditorWidget.js

@@ -61,9 +61,11 @@ define([
             var layer = this;
             this._widget.showHoverButtons(layer);
         };
+
         this.ItemClass.prototype.hideHoverButtons = function() {
             this._widget.hideHoverButtons();
         };
+
         this.ItemClass.prototype.isHoverAllowed = function() {
             return !this._widget.isConnecting();
         };
@@ -169,6 +171,7 @@ define([
         createNews = Object.keys(types).map(type =>
             this._creationNode(type, types[type], Decorator));
 
+        nodes.sort((a, b) => a.node.name < b.node.name ? -1 : 1);
         nodes = nodes.concat(createNews);
 
         // Sort by layer type

src/visualizers/widgets/ClassCodeEditor/ClassCodeEditorWidget.js

@@ -19,9 +19,7 @@ define([
     _.extend(ClassCodeEditorWidget.prototype, TextEditorWidget.prototype);
 
     ClassCodeEditorWidget.prototype.getHeader = function(desc) {
-        return [
-            `-- The class definition for ${desc.name}`
-        ].join('\n');
+        return this.comment(`The class definition for ${desc.name}`);
     };
 
     ClassCodeEditorWidget.prototype.updateNode = function() {

src/visualizers/widgets/DeserializeEditor/DeserializeEditorWidget.js

@@ -26,13 +26,13 @@ define([
 
     DeserializeEditorWidget.prototype.getHeader = function(desc) {
         this._name = desc.name;
-        return [
-            `-- The deserialization function for ${desc.name}`,
-            '-- Globals:',
-            '--   `path` - target filename to load',
-            '--',
-            `-- return the loaded ${desc.name}`
-        ].join('\n');
+        return this.comment([
+            `The deserialization function for ${desc.name}`,
+            'Globals:',
+            '  `path` - target filename to load',
+            '',
+            `return the loaded ${desc.name}`
+        ].join('\n'));
     };
 
     DeserializeEditorWidget.prototype.getNameRegex = function() {

src/visualizers/widgets/OperationCodeEditor/OperationCodeEditorWidget.js

@@ -32,24 +32,24 @@ define([
 
     OperationCodeEditorWidget.prototype.getHeader = function (desc) {
         // Add comment about the inputs, attributes and references
-        var inputs = desc.inputs.map(pair => `-- ${pair[0]} (${pair[1]})`).join('\n'),
-            refs = desc.references.map(name => `-- ${name}`).join('\n'),
+        var inputs = desc.inputs.map(pair => `${pair[0]} (${pair[1]})`).join('\n'),
+            refs = desc.references.map(name => `${name}`).join('\n'),
             header = [
-                `-- Editing "${desc.name}" Implementation`
+                `Editing "${desc.name}" Implementation`
             ];
 
         if (inputs.length) {
-            header.push('--');
-            header.push('-- Defined variables:');
+            header.push('');
+            header.push('Defined variables:');
             header.push(inputs);
         }
         if (refs) {
             header.push(refs);
         }
-        header.push('--');
-        header.push('-- The following will be executed when the operation is run:');
+        header.push('');
+        header.push('The \'execute\' method will be called when the operation is run');
 
-        return header.join('\n');
+        return this.comment(header.join('\n'));
     };
 
     OperationCodeEditorWidget.prototype.canAddReturnTmpl = function (desc) {

src/visualizers/widgets/OperationInterfaceEditor/Item.js

@@ -11,6 +11,9 @@ define([
         DAGItem.call(this, parentEl, desc);
         this.decorator.color = desc.displayColor || this.decorator.color;
 
+        this._hovering = false;
+        this.$el.on('mouseenter', () => this.onHover());
+        this.$el.on('mouseleave', () => this._hovering && this.onUnhover());
 
         // Show the warnings
         this.$warning = null;
@@ -19,6 +22,18 @@ define([
     };
 
     _.extend(Item.prototype, DAGItem.prototype);
+
+    Item.prototype.onUnhover = function() {
+        this._hovering = false;
+        this.hideHoverButtons();
+    };
+
+    Item.prototype.onHover = function() {
+        if (!this.isSelected()) {
+            this._hovering = true;
+            this.showHoverButtons();
+        }
+    };
     
     Item.prototype.update = function(desc) {
         this.decorator.color = desc.displayColor || this.decorator.color;
@@ -82,6 +97,10 @@ define([
         if (this.desc.isUnknown) {
             this.onSetRefClicked(this.desc.name);
         }
+
+        if (this._hovering) {
+            this.onUnhover();
+        }
     };
 
     Item.prototype.setupDecoratorCallbacks = function() {

src/visualizers/widgets/OperationInterfaceEditor/OperationInterfaceEditorWidget.js

@@ -6,6 +6,7 @@ define([
     'widgets/EasyDAG/EasyDAGWidget',
     'widgets/EasyDAG/AddNodeDialog',
     './SelectionManager',
+    './Buttons',
     './Item',
     'underscore',
     'css!./styles/OperationInterfaceEditorWidget.css'
@@ -14,6 +15,7 @@ define([
     EasyDAG,
     AddNodeDialog,
     SelectionManager,
+    Buttons,
     Item,
     _
 ) {
@@ -38,6 +40,20 @@ define([
         // Add ptr rename callback
         this.ItemClass.prototype.changePtrName = (from, to) => this.changePtrName(from, to);
         this.ItemClass.prototype.onSetRefClicked = OperationInterfaceEditorWidget.prototype.onSetRefClicked.bind(this);
+
+        this.ItemClass.prototype.showHoverButtons = function() {
+            var item = this;
+            this._widget.showHoverButtons(item);
+        };
+
+        this.ItemClass.prototype.hideHoverButtons = function() {
+            this._widget.hideHoverButtons();
+        };
+
+        this.ItemClass.prototype.isHoverAllowed = function() {
+            return true;
+        };
+
     };
 
     OperationInterfaceEditorWidget.prototype.onAddItemSelected = function(selected, isInput) {
@@ -45,31 +61,8 @@ define([
     };
 
     OperationInterfaceEditorWidget.prototype.onAddButtonClicked = function(item, isInput) {
-        var successorPairs = this.getValidSuccessors(item.id, isInput),
-            newClass = this.getCreationNode('Complex', NEW_CLASS_ID),
-            newPrim = this.getCreationNode('Primitive', NEW_PRIM_ID),
-            opts = {};
-
-        // Add the 'Create Class' node
-        successorPairs.push(newClass);
-        successorPairs.push(newPrim);
-
-        // Add tabs
-        opts.tabs = ['Primitive', 'Classes'];
-        opts.tabFilter = (tab, pair) => {
-            return pair.node.isPrimitive === (tab === 'Primitive');
-        };
-
-        AddNodeDialog.prompt(successorPairs, opts)
-            .then(selected => {
-                if (selected.node.id === NEW_CLASS_ID) {
-                    DeepForge.create.Complex();
-                } else if (selected.node.id === NEW_PRIM_ID) {
-                    DeepForge.create.Primitive();
-                } else {
-                    this.onAddItemSelected(selected, isInput);
-                }
-            });
+        var successorPairs = this.getValidSuccessors(item.id, isInput);
+        return this.onAddItemSelected(successorPairs[0], isInput);
     };
 
     OperationInterfaceEditorWidget.prototype.onDeactivate = function() {
@@ -126,5 +119,54 @@ define([
         conn.$el.on('click', null);
     };
 
+    // Hover buttons
+    OperationInterfaceEditorWidget.prototype.showHoverButtons = function(item) {
+        var refNodes = this.allValidReferences(),
+            height = item.height,
+            cx = item.width/2;
+
+        if (this.$hoverBtns) {
+            this.hideHoverButtons();
+        }
+
+        this.$hoverBtns = item.$el
+            .append('g')
+            .attr('class', 'hover-container');
+
+        if (item.desc.baseName === 'Operation') {
+            new Buttons.AddOutput({  // Add output data
+                context: this,
+                $pEl: this.$hoverBtns,
+                item: item,
+                x: cx,
+                y: height
+            });
+
+            new Buttons.AddInput({  // Add input data
+                context: this,
+                $pEl: this.$hoverBtns,
+                item: item,
+                x: item.width/3,
+                y: 0
+            });
+
+            new Buttons.AddRef({  // Add reference
+                context: this,
+                $pEl: this.$hoverBtns,
+                disabled: refNodes.length === 0,
+                item: item,
+                x: 2*item.width/3,
+                y: 0
+            });
+        }
+    };
+
+    OperationInterfaceEditorWidget.prototype.hideHoverButtons = function() {
+        if (this.$hoverBtns) {
+            this.$hoverBtns.remove();
+            this.$hoverBtns = null;
+        }
+    };
+
     return OperationInterfaceEditorWidget;
 });

src/visualizers/widgets/OperationInterfaceEditor/SelectionManager.js

@@ -1,4 +1,4 @@
-/*globals define*/
+/*globals define, $*/
 
 define([
     'widgets/EasyDAG/SelectionManager',
@@ -25,12 +25,10 @@ define([
 
     SelectionManager.prototype.createActionButtons = function(width, height) {
         var selectedType = this.selectedItem.desc.baseName,
-            dataNodes,
             refNodes,
             cx = width/2;
 
         if (selectedType === 'Operation') {
-            dataNodes = this._widget.allDataTypeIds();
             refNodes = this._widget.allValidReferences();
 
             new Buttons.AddOutput({  // Add output data
@@ -38,15 +36,13 @@ define([
                 $pEl: this.$selection,
                 item: this.selectedItem,
                 x: cx,
-                y: height,
-                disabled: dataNodes.length === 0
+                y: height
             });
 
             new Buttons.AddInput({  // Add input data
                 context: this._widget,
                 $pEl: this.$selection,
                 item: this.selectedItem,
-                disabled: dataNodes.length === 0,
                 x: width/3,
                 y: 0
             });

src/visualizers/widgets/SerializeEditor/SerializeEditorWidget.js

@@ -23,12 +23,12 @@ define([
 
     SerializeEditorWidget.prototype.getHeader = function(desc) {
         this._name = desc.name;
-        return [
-            `-- The serialization function for ${desc.name}`,
-            '-- Globals:',
-            '--   `path` - target filename',
-            `--   \`data\` - the ${desc.name} to store`
-        ].join('\n');
+        return this.comment([
+            `The serialization function for ${desc.name}`,
+            'Globals:',
+            '  `path` - target filename',
+            `  \`data\` - the ${desc.name} to store`
+        ].join('\n'));
     };
 
     SerializeEditorWidget.prototype.getNameRegex = function () {

src/visualizers/widgets/TextEditor/TextEditorWidget.js

@@ -5,20 +5,33 @@ define([
     'ace/ace',
     'underscore',
     './completer',
+    'js/Utils/ComponentSettings',
+    'jquery-contextMenu',
     'css!./styles/TextEditorWidget.css'
 ], function (
     ace,
     _,
-    Completer
+    Completer,
+    ComponentSettings
 ) {
     'use strict';
 
     var TextEditorWidget,
-        WIDGET_CLASS = 'text-editor';
+        WIDGET_CLASS = 'text-editor',
+        LINE_COMMENT = {
+            python: '#',
+            lua: '--'
+        },
+        DEFAULT_SETTINGS = {
+            keybindings: 'default',
+            theme: 'solarized_dark',
+            fontSize: 12
+        };
 
     TextEditorWidget = function (logger, container) {
         this._logger = logger.fork('Widget');
 
+        this.language = this.language || 'python';
         this._el = container;
         this._el.css({height: '100%'});
         this.$editor = $('<div/>');
@@ -27,9 +40,13 @@ define([
 
         this.readOnly = this.readOnly || false;
         this.editor = ace.edit(this.$editor[0]);
+        this._initialize();
 
         // Get the config from component settings for themes
         this.editor.getSession().setOptions(this.getSessionOptions());
+        var handler = this.editorSettings.keybindings;
+        this.editor.setKeyboardHandler(handler === 'default' ?
+            null : 'ace/keyboard/' + handler);
         this.addExtensions();
         this.editor.$blockScrolling = Infinity;
         this.DELAY = 750;
@@ -47,7 +64,6 @@ define([
         this.setReadOnly(this.readOnly);
         this.currentHeader = '';
         this.activeNode = null;
-        this._initialize();
 
         this._logger.debug('ctor finished');
     };
@@ -68,14 +84,15 @@ define([
         return {
             enableBasicAutocompletion: true,
             enableLiveAutocompletion: true,
-            fontSize: '12pt'
+            theme: 'ace/theme/' + this.editorSettings.theme,
+            fontSize: this.editorSettings.fontSize + 'pt'
         };
     };
 
     TextEditorWidget.prototype.getSessionOptions = function () {
         return {
-            mode: 'ace/mode/lua',
-            tabSize: 3,
+            mode: 'ace/mode/' + this.language,
+            tabSize: 4,
             useSoftTabs: true
         };
     };
@@ -83,6 +100,124 @@ define([
     TextEditorWidget.prototype._initialize = function () {
         // set widget class
         this._el.addClass(WIDGET_CLASS);
+
+        // Add context menu
+        $.contextMenu('destroy', '.' + WIDGET_CLASS);
+        $.contextMenu({
+            selector: '.' + WIDGET_CLASS,
+            build: $trigger => {
+                return {
+                    items: this.getMenuItemsFor($trigger)
+                };
+            }
+        });
+
+        // Create the editor settings
+        this.editorSettings = _.extend({}, DEFAULT_SETTINGS),
+        ComponentSettings.resolveWithWebGMEGlobal(
+            this.editorSettings,
+            this.getComponentId()
+        );
+    };
+
+    TextEditorWidget.prototype.getComponentId = function () {
+        return 'TextEditor';
+    };
+
+    TextEditorWidget.prototype.getMenuItemsFor = function () {
+        var fontSizes = [8, 10, 11, 12, 14],
+            themes = [
+                'Solarized Light',
+                'Solarized Dark',
+                'Twilight',
+                'Tomorrow Night',
+                'Eclipse',
+                'Monokai'
+            ],
+            keybindings = [
+                'default',
+                'vim',
+                'emacs'
+            ],
+            menuItems = {
+                setKeybindings: {
+                    name: 'Keybindings...',
+                    items: {}
+                },
+                setFontSize: {
+                    name: 'Font Size...',
+                    items: {}
+                },
+                setTheme: {
+                    name: 'Theme...',
+                    items: {}
+                }
+            };
+
+        fontSizes.forEach(fontSize => {
+            var name = fontSize + ' pt',
+                isSet = fontSize === this.editorSettings.fontSize;
+
+            if (isSet) {
+                name = '<span style="font-weight: bold">' + name + '</span>';
+            }
+
+            menuItems.setFontSize.items['font' + fontSize] = {
+                name: name,
+                isHtmlName: isSet,
+                callback: () => {
+                    this.editorSettings.fontSize = fontSize;
+                    this.editor.setOptions(this.getEditorOptions());
+                    this.onUpdateEditorSettings();
+                }
+            };
+        });
+
+        themes.forEach(name => {
+            var theme = name.toLowerCase().replace(/ /g, '_'),
+                isSet = theme === this.editorSettings.theme;
+
+            if (isSet) {
+                name = '<span style="font-weight: bold">' + name + '</span>';
+            }
+
+            menuItems.setTheme.items[theme] = {
+                name: name,
+                isHtmlName: isSet,
+                callback: () => {
+                    this.editorSettings.theme = theme;
+                    this.editor.setOptions(this.getEditorOptions());
+                    this.onUpdateEditorSettings();
+                }
+            };
+        });
+
+        keybindings.forEach(name => {
+            var handler = name.toLowerCase().replace(/ /g, '_'),
+                isSet = handler === this.editorSettings.keybindings;
+
+            if (isSet) {
+                name = '<span style="font-weight: bold">' + name + '</span>';
+            }
+
+            menuItems.setKeybindings.items[handler] = {
+                name: name,
+                isHtmlName: isSet,
+                callback: () => {
+                    this.editorSettings.keybindings = handler;
+                    this.editor.setKeyboardHandler(handler === 'default' ?
+                        null : 'ace/keyboard/' + handler);
+                    this.onUpdateEditorSettings();
+                }
+            };
+        });
+
+        return menuItems;
+    };
+
+    TextEditorWidget.prototype.onUpdateEditorSettings = function () {
+        ComponentSettings.overwriteComponentSettings(this.getComponentId(), this.editorSettings,
+            err => err && this._logger.error(`Could not save editor settings: ${err}`));
     };
 
     TextEditorWidget.prototype.onWidgetContainerResize = function () {
@@ -90,18 +225,33 @@ define([
     };
 
     // Adding/Removing/Updating items
+    TextEditorWidget.prototype.comment = function (text) {
+        var prefix = LINE_COMMENT[this.language] + ' ';
+        return text.replace(
+            new RegExp('^(' + LINE_COMMENT[this.language] + ')?','mg'),
+            prefix
+        );
+    };
+
     TextEditorWidget.prototype.getHeader = function (desc) {
-        return `-- Editing "${desc.name}"`;
+        return this.comment(`Editing "${desc.name}"`);
     };
 
     TextEditorWidget.prototype.addNode = function (desc) {
         // Set the current text based on the given
         // Create the header
-        var header = this.getHeader(desc);
+        var header = this.getHeader(desc),
+            content,
+            newContent = header + '\n' + desc.text,
+            //patches = diff.diffChars(content, newContent),
+            cursorPos;
 
+        // TODO: if we are updating the value, we should make sure the cursor position
+        // remains in the same spot (ie, diff the text and update the positions
+        // based on the size of the patches
         this.activeNode = desc.id;
         this.silent = true;
-        this.editor.setValue(header + '\n' + desc.text, 2);
+        this.editor.setValue(newContent, 2);
         this.silent = false;
         this.currentHeader = header;
     };
@@ -147,6 +297,7 @@ define([
     /* * * * * * * * Visualizer life cycle callbacks * * * * * * * */
     TextEditorWidget.prototype.destroy = function () {
         this.editor.destroy();
+        $.contextMenu('destroy', '.' + WIDGET_CLASS);
     };
 
     TextEditorWidget.prototype.onActivate = function () {

test/common/OperationCode.spec.js

@@ -0,0 +1,353 @@
+describe.only('OperationCode', function() {
+    var fs = require('fs');
+    var path = require('path');
+    var assert = require('assert');
+    var OperationCode = require('../../src/common/OperationCode');
+    var operation;
+
+    describe('example', function() {
+        var code;
+
+        before(function() {
+            // load the example
+            var filePath = path.join(__dirname, '..', 'test-cases', 'operations', 'example.py');
+            code = fs.readFileSync(filePath, 'utf8');
+        });
+
+        describe('removeInput', function() {
+            before(function() {
+                operation = new OperationCode(code);
+                operation.removeInput('world');
+            });
+
+            it('should have 2 remaining inputs', function() {
+                assert.equal(operation.getInputs().length, 2);
+            });
+
+        });
+
+        describe('attributes', function() {
+            describe('add', function() {
+                beforeEach(function() {
+                    operation = new OperationCode(code);
+                });
+
+                it('should add argument to __init__ method', function() {
+                    operation.addAttribute('number');
+                    var attrs = operation.getAttributes();
+                    // TODO
+                });
+
+                it('should set the default value', function() {
+                    // TODO
+                });
+            });
+
+            // TODO: add attribute
+            // TODO: remove attribute
+            // TODO: rename attribute?
+        });
+
+        describe('rename', function() {
+            before(function() {
+                operation = new OperationCode(code);
+                operation.rename('hello', 'goodbye');
+            });
+
+            it('should rename input arg', function() {
+                var inputs = operation.getInputs();
+                var oldInput = inputs.find(input => input.name === 'hello');
+                var newInput = inputs.find(input => input.name === 'goodbye');
+
+                assert(!oldInput);
+                assert(newInput);
+            });
+
+            it('should rename occurrences in the fn', function() {
+                assert(!operation.getCode().includes('hello'));
+            });
+        });
+
+        describe('parsing', function() {
+            before(function() {
+                operation = new OperationCode(code);
+            });
+
+            it('should parse the correct name', function() {
+                assert.equal(operation.getName(), 'ExampleOperation');
+            });
+
+            it.skip('should parse the correct base', function() {
+                assert.equal(operation.getBase(), 'Operation');
+            });
+
+            it('should parse the input names', function() {
+                const names = ['hello', 'world', 'count'];
+                assert.deepEqual(operation.getInputs().map(input => input.name), names);
+            });
+
+            it.skip('should parse the input types', function() {
+                const types = ['str', 'str', 'int'];
+                assert.deepEqual(operation.getInputs().map(input => input.type), types);
+            });
+
+            it('should parse the output names', function() {
+                const names = ['concat', 'count'];
+                assert.deepEqual(operation.getOutputs().map(output => output.name), names);
+            });
+
+            it.skip('should parse the output types', function() {
+                const types = ['str', 'int'];
+                assert.deepEqual(operation.getOutputs().map(output => output.type), types);
+            });
+        });
+    });
+
+    describe('multi-anon-results', function() {
+        before(function() {
+            var filePath = path.join(__dirname, '..', 'test-cases', 'operations', 'multi-anon-results.py');
+            var example = fs.readFileSync(filePath, 'utf8');
+            operation = new OperationCode(example);
+        });
+
+        it('should parse multiple return values', function() {
+            assert.equal(operation.getOutputs().length, 2);
+        });
+
+        it('should create unique names for each', function() {
+            var [first, second] = operation.getOutputs();
+            assert.notEqual(first.name, second.name);
+        });
+    });
+
+    describe('no-inputs/outputs', function() {
+        var code;
+
+        before(function() {
+            var filePath = path.join(__dirname, '..', 'test-cases', 'operations', 'no-inputs.py');
+            code = fs.readFileSync(filePath, 'utf8');
+        });
+
+        describe('parsing', function() {
+            beforeEach(function() {
+                operation = new OperationCode(code);
+            });
+
+            it('should not require base class', function() {
+                assert.equal(operation.getBase(), null);
+            });
+
+            it('should detect zero output', function() {
+                assert.equal(operation.getOutputs().length, 0);
+            });
+
+            it('should detect zero inputs', function() {
+                assert.equal(operation.getInputs().length, 0);
+            });
+        });
+
+        describe('addInput', function() {
+            var operation;
+
+            before(function() {
+                operation = new OperationCode(code);
+                operation.addInput('first');
+            });
+
+            it('should clear schema', function() {
+                assert(!operation._schema);
+            });
+
+            it('should add input to `execute` fn', function() {
+                var code = operation.getCode();
+                assert(code.includes('first'));
+            });
+
+            it('should have an additional input arg', function() {
+                var inputs = operation.getInputs();
+                assert.equal(inputs.length, 1);
+            });
+        });
+
+        describe('addOutput', function() {
+            var operation;
+
+            describe('lone return', function() {
+                before(function() {
+                    operation = new OperationCode(code);
+                    operation.addOutput('myNewOutput');
+                });
+
+                it('should clear schema', function() {
+                    assert(!operation._schema);
+                });
+
+                it('should add input to `execute` fn', function() {
+                    var code = operation.getCode();
+                    assert(code.includes('myNewOutput'));
+                });
+
+                it('should have an additional input arg', function() {
+                    var inputs = operation.getOutputs();
+                    assert.equal(inputs.length, 1);
+                });
+            });
+
+            describe('no return', function() {
+                before(function() {
+                    operation = new OperationCode(code);
+                    operation.addReturnValue('no_return', 'myNewOutput');
+                });
+
+                it('should clear schema', function() {
+                    assert(!operation._schema);
+                });
+
+                it('should add input to `execute` fn', function() {
+                    var code = operation.getCode();
+                    assert(code.includes('myNewOutput'));
+                });
+
+                it('should have an additional input arg', function() {
+                    var outputs = operation.getReturnValues('no_return');
+                    assert.equal(outputs.length, 1);
+                });
+            });
+        });
+
+    });
+
+    describe('simple', function() {
+        var code;
+
+        before(function() {
+            var filePath = path.join(__dirname, '..', 'test-cases', 'operations', 'simple.py');
+            code = fs.readFileSync(filePath, 'utf8');
+        });
+
+        describe('parsing', function() {
+            beforeEach(function() {
+                operation = new OperationCode(code);
+            });
+
+            it('should not require base class', function() {
+                assert.equal(operation.getBase(), null);
+            });
+
+            it('should detect one output', function() {
+                assert.equal(operation.getOutputs().length, 1);
+            });
+
+            it('should provide the value', function() {
+                assert.equal(operation.getOutputs()[0].value, '20');
+            });
+
+            it('should detect two inputs', function() {
+                assert.equal(operation.getInputs().length, 2);
+            });
+        });
+
+        describe('addInput', function() {
+            var operation;
+
+            before(function() {
+                operation = new OperationCode(code);
+                operation.addInput('myNewInput');
+            });
+
+            it('should clear schema', function() {
+                assert(!operation._schema);
+            });
+
+            it('should add input to `execute` fn', function() {
+                var code = operation.getCode();
+                assert(code.includes('myNewInput'));
+            });
+
+            it('should have an additional input arg', function() {
+                var inputs = operation.getInputs();
+                assert.equal(inputs.length, 3);
+            });
+        });
+
+        describe('addOutput', function() {
+            var operation;
+
+            before(function() {
+                operation = new OperationCode(code);
+                operation.addOutput('myNewOutput');
+            });
+
+            it('should clear schema', function() {
+                assert(!operation._schema);
+            });
+
+            it('should add input to `execute` fn', function() {
+                var code = operation.getCode();
+                assert(code.includes('myNewOutput'));
+            });
+
+            it('should have an additional input arg', function() {
+                var inputs = operation.getOutputs();
+                assert.equal(inputs.length, 2);
+            });
+        });
+
+        describe('removeInput', function() {
+            var operation,
+                result;
+
+            beforeEach(function() {
+                operation = new OperationCode(code);
+                result = operation.removeInput('number');
+            });
+
+            it('should return removed arg', function() {
+                assert.equal(result.name, 'number');
+            });
+
+            it('should only have one remaining argument', function() {
+                assert.equal(operation.getInputs().length, 1);
+            });
+
+            it('should only not have removed argument', function() {
+                assert(!operation.getCode().includes('number'));
+            });
+
+            it('should return null if arg doesn\'t exist', function() {
+                var result = operation.removeInput('numdasfber');
+                assert.equal(result, null);
+            });
+
+        });
+
+        describe('removeOutput', function() {
+            var operation,
+                result;
+
+            beforeEach(function() {
+                operation = new OperationCode(code);
+                result = operation.removeOutput('result');
+            });
+
+            it('should return removed arg', function() {
+                assert.equal(result.name, 'result');
+            });
+
+            it('should have no remaining results', function() {
+                assert.equal(operation.getOutputs().length, 0);
+            });
+
+            it('should only not have removed argument', function() {
+                assert(!operation.getCode().includes('20'));
+            });
+
+            it('should return null if arg doesn\'t exist', function() {
+                var result = operation.removeOutput('numdasfber');
+                assert.equal(result, null);
+            });
+
+        });
+    });
+
+});