Refactored code gen out of ExecuteJob. Fixes #1014 (#1018)

* WIP created GenerateJob plugin for generating exec code

* WIP moved templates to GenerateJob

* WIP #1014 GenerateJob tests passing

* WIP #1014 Added attribute file test

* WIP #1014 Added more GenerateJob tests

* WIP #1014 removed extra logs

* WIP #1014 Follow base for plugin, namespace

* WIP #1014 Added CodeGen component settings

* WIP #1014 Removed component settings

* WIP #1014 Fixed the loading of inputs

* WIP #1014 Fixed namespace detection

* WIP #1014 Fixed code gen of job refs

* WIP #1014 Fixed 2nd degree code gen

* WIP #1014 Removed old code generation file

* WIP #1014 Fixed code linting issues

* WIP #1014 Fixed blob fetch error handling

* WIP #1014 fixed error handling for bad blob fetch

* WIP #1014 Updated the commit hash for the ptr code gen

* WIP #1014 save after forwarding data from the successful job

* WIP #1014 updated error msg on blob retrieval fail

* WIP #1014 fixed code linting issues

src/common/plugin/Operation.js

@@ -0,0 +1,35 @@
+/*globals define */
+// This is a mixin containing helpers for working with operation nodes
+define([],function() {
+
+    var OperationOps = function() {
+    };
+
+    OperationOps.prototype.getOutputs = function (node) {
+        return this.getOperationData(node, this.META.Outputs);
+    };
+
+    OperationOps.prototype.getInputs = function (node) {
+        return this.getOperationData(node, this.META.Inputs);
+    };
+
+    OperationOps.prototype.getOperationData = function (node, metaType) {
+        // Load the children and the output's children
+        return this.core.loadChildren(node)
+            .then(containers => {
+                var outputs = containers.find(c => this.core.isTypeOf(c, metaType));
+                return outputs ? this.core.loadChildren(outputs) : [];
+            })
+            .then(outputs => {
+                var bases = outputs.map(node => this.core.getMetaType(node));
+                // return [[arg1, Type1, node1], [arg2, Type2, node2]]
+                return outputs.map((node, i) => [
+                    this.getAttribute(node, 'name'),
+                    this.getAttribute(bases[i], 'name'),
+                    node
+                ]);
+            });
+    };
+
+    return OperationOps;
+});

src/common/plugin/PtrCodeGen.js

@@ -6,27 +6,64 @@ define([
     PluginUtils,
     Q
 ) {
+
+    var CodeGen = {
+        Operation: {
+            pluginId: 'GenerateJob',
+            namespace: 'pipeline'
+        }
+    };
+
     var PtrCodeGen = function() {
     };
 
+    PtrCodeGen.prototype.getCodeGenPluginIdFor = function(node) {
+        var base = this.core.getBase(node),
+            name = this.core.getAttribute(node, 'name'),
+            namespace = this.core.getNamespace(node),
+            pluginId;
+
+        //this.logger.debug(`loaded pointer target of ${ptrId}: ${ptrNode}`);
+        pluginId = (this.core.getOwnRegistry(node, 'validPlugins') || '').split(' ').shift();
+        //this.logger.info(`generating code for ${this.core.getAttribute(ptrNode, 'name')} using ${pluginId}`);
+
+        if (this.core.isMetaNode(node) && CodeGen[name]) {
+            pluginId = CodeGen[name].pluginId || CodeGen[name];
+            namespace = CodeGen[name].namespace;
+        }
+
+        if (pluginId) {
+            return {
+                namespace: namespace,
+                pluginId: pluginId
+            };
+        } else if (base) {
+            return this.getCodeGenPluginIdFor(base);
+        } else {
+            return null;
+        }
+    };
+
     PtrCodeGen.prototype.getPtrCodeHash = function(ptrId) {
         return this.core.loadByPath(this.rootNode, ptrId)
             .then(ptrNode => {
                 // Look up the plugin to use
-                var metanode = this.core.getMetaType(ptrNode),
+                var genInfo = this.getCodeGenPluginIdFor(ptrNode);
-                    pluginId;
 
-                this.logger.debug(`loaded pointer target of ${ptrId}: ${ptrNode}`);
+                if (genInfo.pluginId) {
-                pluginId = this.core.getRegistry(ptrNode, 'validPlugins').split(' ').shift();
+                    var context = {
-                this.logger.info(`generating code for ${this.core.getAttribute(ptrNode, 'name')} using ${pluginId}`);
+                        namespace: genInfo.namespace,
+                        activeNode: this.core.getPath(ptrNode)
+                    };
 
-                var context = {
+                    // Load and run the plugin
-                    namespace: this.core.getNamespace(metanode),
+                    return this.executePlugin(genInfo.pluginId, context);
-                    activeNode: this.core.getPath(ptrNode)
+                } else {
-                };
+                    var metanode = this.core.getMetaType(ptrNode),
-
+                        type = this.core.getAttribute(metanode, 'name');
-                // Load and run the plugin
+                    this.logger.warn(`Could not find plugin for ${type}. Will try to proceed anyway`);
-                return this.executePlugin(pluginId, context);
+                    return null;
+                }
             })
             .then(hashes => hashes[0]);  // Grab the first asset for now
     };
@@ -56,12 +93,13 @@ define([
         return PluginUtils.loadNodesAtCommitHash(
             this.project,
             this.core,
-            this.commitHash,
+            this.currentHash,
             this.logger,
             opts
         ).then(config => {
             plugin.initialize(logger, this.blobClient, this.gmeConfig);
             config.core = this.core;
+            config.project = this.project;
             plugin.configure(config);
             return plugin;
         });

src/plugins/ExecuteJob/ExecuteJob.js

@@ -8,10 +8,10 @@ define([
     'plugin/PluginBase',
     'deepforge/plugin/LocalExecutor',
     'deepforge/plugin/PtrCodeGen',
+    'deepforge/plugin/Operation',
     'deepforge/api/JobLogsClient',
     'deepforge/api/JobOriginClient',
     'deepforge/api/ExecPulseClient',
-    './ExecuteJob.Files',
     './ExecuteJob.Metadata',
     './ExecuteJob.SafeSave',
     'deepforge/Constants',
@@ -26,11 +26,11 @@ define([
     PluginBase,
     LocalExecutor,  // DeepForge operation primitives
     PtrCodeGen,
+    OperationPlugin,
     JobLogsClient,
     JobOriginClient,
     ExecPulseClient,
 
-    ExecuteJobFiles,
     ExecuteJobMetadata,
     ExecuteJobSafeSave,
 
@@ -44,8 +44,7 @@ define([
 
     pluginMetadata = JSON.parse(pluginMetadata);
 
-    var OUTPUT_INTERVAL = 1500,
+    var STDOUT_FILE = 'job_stdout.txt';
-        STDOUT_FILE = 'job_stdout.txt';
 
     /**
      * Initializes a new instance of ExecuteJob.
@@ -453,9 +452,10 @@ define([
             children.find(child => this.isMetaTypeOf(child, this.META.Operation)));
     };
 
-    ExecuteJob.prototype.onBlobRetrievalFail = function (node, input, err) {
+    // Handle the blob retrieval failed error
+    ExecuteJob.prototype.onBlobRetrievalFail = function (node, input) {
         var job = this.core.getParent(node),
-            e = `Failed to retrieve "${input}" (${err})`,
+            e = `Failed to retrieve "${input}" (BLOB_FETCH_FAILED)`,
             consoleErr = `Failed to execute operation: ${e}`;
 
         consoleErr += [
@@ -473,14 +473,8 @@ define([
 
     ExecuteJob.prototype.executeJob = function (job) {
         return this.getOperation(job).then(node => {
-            var jobId = this.core.getPath(job),
+            var name = this.getAttribute(node, 'name'),
-                name = this.getAttribute(node, 'name'),
+                localTypeId = this.getLocalOperationType(node);
-                localTypeId = this.getLocalOperationType(node),
-                artifact,
-                artifactName,
-                files,
-                data = {},
-                inputs;
 
             // Execute any special operation types here - not on an executor
             this.logger.debug(`Executing operation "${name}"`);
@@ -488,126 +482,22 @@ define([
                 return this.executeLocalOperation(localTypeId, node);
             } else {
                 // Generate all execution files
-                return this.createOperationFiles(node).then(results => {
+                return this.getPtrCodeHash(this.core.getPath(node))
-                    this.logger.info('Created operation files!');
+                    .fail(err => {
-                    files = results;
+                        this.logger.error(`Could not generate files: ${err}`);
-                    artifactName = `${name}_${jobId.replace(/\//g, '_')}-execution-files`;
+                        if (err.message.indexOf('BLOB_FETCH_FAILED') > -1) {
-                    artifact = this.blobClient.createArtifact(artifactName);
+                            this.onBlobRetrievalFail(node, err.message.split(':')[1]);
-
+                        }
-                    // Add the input assets
+                        throw err;
-                    //   - get the metadata (name)
+                    })
-                    //   - add the given inputs
+                    .then(hash => {
-                    inputs = Object.keys(files.inputAssets);
+                        this.logger.info(`Saved execution files`);
-
+                        this.result.addArtifact(hash);  // Probably only need this for debugging...
-                    return Q.all(
+                        this.executeDistOperation(job, node, hash);
-                        inputs.map(input => {  // Get the metadata for each input
+                    })
-                            var hash = files.inputAssets[input];
+                    .fail(e => {
-
+                        this.onOperationFail(node, `Distributed operation "${name}" failed ${e}`);
-                            // data asset for "input"
-                            return this.blobClient.getMetadata(hash)
-                                .fail(err => this.onBlobRetrievalFail(job, input, err));
-                        })
-                    );
-                })
-                .then(mds => {
-                    // Record the large files
-                    var inputData = {},
-                        runsh = '# Bash script to download data files and run job\n' +
-                        'if [ -z "$DEEPFORGE_URL" ]; then\n  echo "Please set DEEPFORGE_URL and' +
-                        ' re-run:"\n  echo ""  \n  echo "  DEEPFORGE_URL=http://my.' +
-                        'deepforge.server.com:8080 bash run.sh"\n  echo ""\n exit 1\nfi\n';
-
-                    mds.forEach((metadata, i) => {
-                        // add the hashes for each input
-                        var input = inputs[i], 
-                            hash = files.inputAssets[input],
-                            dataPath = 'inputs/' + input + '/data',
-                            url = this.blobClient.getRelativeDownloadURL(hash);
-
-                        inputData[dataPath] = {
-                            req: hash,
-                            cache: metadata.content
-                        };
-
-                        // Add to the run.sh file
-                        runsh += `wget $DEEPFORGE_URL${url} -O ${dataPath}\n`;
                     });
-
-                    delete files.inputAssets;
-                    files['input-data.json'] = JSON.stringify(inputData, null, 2);
-                    runsh += 'th init.lua';
-                    files['run.sh'] = runsh;
-
-                    // Add pointer assets
-                    Object.keys(files.ptrAssets)
-                        .forEach(path => data[path] = files.ptrAssets[path]);
-
-                    // Add the executor config
-                    return this.getOutputs(node);
-                })
-                .then(outputArgs => {
-                    var config,
-                        outputs,
-                        fileList,
-                        ptrFiles = Object.keys(files.ptrAssets),
-                        file;
-
-                    delete files.ptrAssets;
-                    fileList = Object.keys(files).concat(ptrFiles);
-
-                    outputs = outputArgs.map(pair => pair[0])
-                        .map(name => {
-                            return {
-                                name: name,
-                                resultPatterns: [`outputs/${name}`]
-                            };
-                        });
-
-                    outputs.push(
-                        {
-                            name: 'stdout',
-                            resultPatterns: [STDOUT_FILE]
-                        },
-                        {
-                            name: name + '-all-files',
-                            resultPatterns: fileList
-                        }
-                    );
-
-                    config = {
-                        cmd: 'node',
-                        args: ['start.js'],
-                        outputInterval: OUTPUT_INTERVAL,
-                        resultArtifacts: outputs
-                    };
-                    files['executor_config.json'] = JSON.stringify(config, null, 4);
-
-                    // Save the artifact
-                    // Remove empty hashes
-                    for (file in data) {
-                        if (!data[file]) {
-                            this.logger.warn(`Empty data hash has been found for file "${file}". Removing it...`);
-                            delete data[file];
-                        }
-                    }
-                    return artifact.addObjectHashes(data);
-                })
-                .then(() => {
-                    this.logger.info(`Added ptr/input data hashes for "${artifactName}"`);
-                    return artifact.addFiles(files);
-                })
-                .then(() => {
-                    this.logger.info(`Added execution files for "${artifactName}"`);
-                    return artifact.save();
-                })
-                .then(hash => {
-                    this.logger.info(`Saved execution files "${artifactName}"`);
-                    this.result.addArtifact(hash);  // Probably only need this for debugging...
-                    this.executeDistOperation(job, node, hash);
-                })
-                .fail(e => {
-                    this.onOperationFail(node, `Distributed operation "${name}" failed ${e}`);
-                });
             }
         });
     };
@@ -881,32 +771,6 @@ define([
             .fail(e => this.onOperationFail(node, `Operation ${nodeId} failed: ${e}`));
     };
 
-    ExecuteJob.prototype.getOutputs = function (node) {
-        return this.getOperationData(node, this.META.Outputs);
-    };
-
-    ExecuteJob.prototype.getInputs = function (node) {
-        return this.getOperationData(node, this.META.Inputs);
-    };
-
-    ExecuteJob.prototype.getOperationData = function (node, metaType) {
-        // Load the children and the output's children
-        return this.core.loadChildren(node)
-            .then(containers => {
-                var outputs = containers.find(c => this.core.isTypeOf(c, metaType));
-                return outputs ? this.core.loadChildren(outputs) : [];
-            })
-            .then(outputs => {
-                var bases = outputs.map(node => this.core.getMetaType(node));
-                // return [[arg1, Type1, node1], [arg2, Type2, node2]]
-                return outputs.map((node, i) => [
-                    this.getAttribute(node, 'name'),
-                    this.getAttribute(bases[i], 'name'),
-                    node
-                ]);
-            });
-    };
-
     //////////////////////////// Special Operations ////////////////////////////
     ExecuteJob.prototype.executeLocalOperation = function (type, node) {
         // Retrieve the given LOCAL_OP type
@@ -920,7 +784,7 @@ define([
 
     _.extend(
         ExecuteJob.prototype,
-        ExecuteJobFiles.prototype,
+        OperationPlugin.prototype,
         ExecuteJobMetadata.prototype,
         ExecuteJobSafeSave.prototype,
         PtrCodeGen.prototype,

src/plugins/ExecutePipeline/ExecutePipeline.js

@@ -175,6 +175,7 @@ define([
     ExecutePipeline.prototype.resumePipeline = function () {
         var nodes = Object.keys(this.nodes).map(id => this.nodes[id]),
             allJobs = nodes.filter(node => this.core.isTypeOf(node, this.META.Job)),
+            name = this.getAttribute(this.activeNode, 'name'),
             status,
             jobs = {
                 success: [],
@@ -208,6 +209,7 @@ define([
         return Q.all(allJobs.map(job => this.recordOldMetadata(job, true)))
             .then(() => Q.all(jobs.success.map(job => this.getOperation(job))))
             .then(ops => ops.forEach(op => this.updateJobCompletionRecords(op)))
+            .then(() => this.save(`Resuming pipeline execution: ${name}`))
             .then(() => {
                 
                 if (jobs.running.length) {  // Resume all running jobs
@@ -531,10 +533,12 @@ define([
         this.logger.info(`Setting ${jobId} status to "success"`);
         this.logger.info(`There are now ${this.runningJobs} running jobs`);
         this.logger.debug(`Making a commit from ${this.currentHash}`);
+
+        counts = this.updateJobCompletionRecords(opNode);
+
         this.save(`Operation "${name}" in ${this.pipelineName} completed successfully`)
             .then(() => {
 
-                counts = this.updateJobCompletionRecords(opNode);
                 hasReadyOps = counts.indexOf(0) > -1;
 
                 this.logger.debug(`Operation "${name}" completed. ` + 

src/plugins/GenerateJob/GenerateJob.js

@@ -1,29 +1,212 @@
 /*globals define*/
+/*jshint node:true, browser:true*/
+
 define([
     './templates/index',
     'q',
     'underscore',
-    'deepforge/Constants'
+    'deepforge/Constants',
-], function(
+    'deepforge/plugin/Operation',
+    'deepforge/plugin/PtrCodeGen',
+    'text!./metadata.json',
+    'plugin/PluginBase'
+], function (
     Templates,
     Q,
     _,
-    CONSTANTS
+    CONSTANTS,
+    OperationHelpers,
+    PtrCodeGen,
+    pluginMetadata,
+    PluginBase
 ) {
+    'use strict';
 
-    var SKIP_ATTRIBUTES = [
+    pluginMetadata = JSON.parse(pluginMetadata);
-        'code',
+    var OUTPUT_INTERVAL = 1500,
-        'stdout',
+        STDOUT_FILE = 'job_stdout.txt',
-        'execFiles',
+        SKIP_ATTRIBUTES = [
-        'jobId',
+            'code',
-        'secret',
+            'stdout',
-        CONSTANTS.LINE_OFFSET,
+            'execFiles',
-        CONSTANTS.DISPLAY_COLOR
+            'jobId',
-    ];
+            'secret',
-    var ExecuteJob = function() {
+            CONSTANTS.LINE_OFFSET,
+            CONSTANTS.DISPLAY_COLOR
+        ];
+
+    /**
+     * Initializes a new instance of GenerateJob.
+     * @class
+     * @augments {PluginBase}
+     * @classdesc This class represents the plugin GenerateJob.
+     * @constructor
+     */
+    var GenerateJob = function () {
+        // Call base class' constructor.
+        PluginBase.call(this);
+        this.pluginMetadata = pluginMetadata;
     };
 
-    ExecuteJob.prototype.createOperationFiles = function (node) {
+    /**
+     * Metadata associated with the plugin. Contains id, name, version, description, icon, configStructue etc.
+     * This is also available at the instance at this.pluginMetadata.
+     * @type {object}
+     */
+    GenerateJob.metadata = pluginMetadata;
+
+    // Prototypical inheritance from PluginBase.
+    GenerateJob.prototype = Object.create(PluginBase.prototype);
+    GenerateJob.prototype.constructor = GenerateJob;
+
+    /**
+     * Main function for the plugin to execute. This will perform the execution.
+     * Notes:
+     * - Always log with the provided logger.[error,warning,info,debug].
+     * - Do NOT put any user interaction logic UI, etc. inside this method.
+     * - callback always has to be called even if error happened.
+     *
+     * @param {function(string, plugin.PluginResult)} callback - the result callback
+     */
+    GenerateJob.prototype.main = function (callback) {
+        var files,
+            artifactName,
+            artifact,
+            data = {},
+            inputs,
+            name,
+            opId;
+
+        name = this.getAttribute(this.activeNode, 'name');
+        opId = this.core.getPath(this.activeNode);
+
+        return this.createOperationFiles(this.activeNode)
+            .then(results => {
+                this.logger.info('Created operation files!');
+                files = results;
+                artifactName = `${name}_${opId.replace(/\//g, '_')}-execution-files`;
+                artifact = this.blobClient.createArtifact(artifactName);
+
+                // Add the input assets
+                //   - get the metadata (name)
+                //   - add the given inputs
+                inputs = Object.keys(files.inputAssets);
+
+                return Q.all(
+                    inputs.map(input => {  // Get the metadata for each input
+                        var hash = files.inputAssets[input];
+
+                        // data asset for "input"
+                        return this.blobClient.getMetadata(hash)
+                            .fail(() => {
+                                throw Error(`BLOB_FETCH_FAILED:${input}`);
+                            });
+                    })
+                );
+            })
+            .then(mds => {
+                // Record the large files
+                var inputData = {},
+                    runsh = '# Bash script to download data files and run job\n' +
+                    'if [ -z "$DEEPFORGE_URL" ]; then\n  echo "Please set DEEPFORGE_URL and' +
+                    ' re-run:"\n  echo ""  \n  echo "  DEEPFORGE_URL=http://my.' +
+                    'deepforge.server.com:8080 bash run.sh"\n  echo ""\n exit 1\nfi\n';
+
+                mds.forEach((metadata, i) => {
+                    // add the hashes for each input
+                    var input = inputs[i], 
+                        hash = files.inputAssets[input],
+                        dataPath = 'inputs/' + input + '/data',
+                        url = this.blobClient.getRelativeDownloadURL(hash);
+
+                    inputData[dataPath] = {
+                        req: hash,
+                        cache: metadata.content
+                    };
+
+                    // Add to the run.sh file
+                    runsh += `wget $DEEPFORGE_URL${url} -O ${dataPath}\n`;
+                });
+
+                delete files.inputAssets;
+                files['input-data.json'] = JSON.stringify(inputData, null, 2);
+                runsh += 'th init.lua';
+                files['run.sh'] = runsh;
+
+                // Add pointer assets
+                Object.keys(files.ptrAssets)
+                    .forEach(path => data[path] = files.ptrAssets[path]);
+
+                // Add the executor config
+                return this.getOutputs(this.activeNode);
+            })
+            .then(outputArgs => {
+                var config,
+                    outputs,
+                    fileList,
+                    ptrFiles = Object.keys(files.ptrAssets),
+                    file;
+
+                delete files.ptrAssets;
+                fileList = Object.keys(files).concat(ptrFiles);
+
+                outputs = outputArgs.map(pair => pair[0])
+                    .map(name => {
+                        return {
+                            name: name,
+                            resultPatterns: [`outputs/${name}`]
+                        };
+                    });
+
+                outputs.push(
+                    {
+                        name: 'stdout',
+                        resultPatterns: [STDOUT_FILE]
+                    },
+                    {
+                        name: name + '-all-files',
+                        resultPatterns: fileList
+                    }
+                );
+
+                config = {
+                    cmd: 'node',
+                    args: ['start.js'],
+                    outputInterval: OUTPUT_INTERVAL,
+                    resultArtifacts: outputs
+                };
+                files['executor_config.json'] = JSON.stringify(config, null, 4);
+
+                // Save the artifact
+                // Remove empty hashes
+                for (file in data) {
+                    if (!data[file]) {
+                        this.logger.warn(`Empty data hash has been found for file "${file}". Removing it...`);
+                        delete data[file];
+                    }
+                }
+                return artifact.addObjectHashes(data);
+            })
+            .then(() => {
+                this.logger.info(`Added ptr/input data hashes for "${artifactName}"`);
+                return artifact.addFiles(files);
+            })
+            .then(() => {
+                this.logger.info(`Added execution files for "${artifactName}"`);
+                return artifact.save();
+            })
+            .then(hash => {
+                this.result.setSuccess(true);
+                this.result.addArtifact(hash);
+                callback(null, this.result);
+            })
+            .fail(err => {
+                this.result.setSuccess(false);
+                callback(err, this.result);
+            });
+    }; 
+
+    GenerateJob.prototype.createOperationFiles = function (node) {
         var files = {};
         // For each operation, generate the output files:
         //   inputs/<arg-name>/init.lua  (respective data deserializer)
@@ -47,10 +230,14 @@ define([
             .then(() => {
                 this.createAttributeFile(node, files);
                 return Q.ninvoke(this, 'createPointers', node, files);
+            })
+            .fail(err => {
+                this.logger.error(err);
+                throw err;
             });
     };
 
-    ExecuteJob.prototype.createEntryFile = function (node, files) {
+    GenerateJob.prototype.createEntryFile = function (node, files) {
         this.logger.info('Creating entry files...');
         return this.getOutputs(node)
             .then(outputs => {
@@ -68,7 +255,7 @@ define([
             });
     };
 
-    ExecuteJob.prototype.createClasses = function (node, files) {
+    GenerateJob.prototype.createClasses = function (node, files) {
         var metaDict = this.core.getAllMetaNodes(this.rootNode),
             isClass,
             metanodes,
@@ -120,7 +307,7 @@ define([
         files['classes/init.lua'] = code;
     };
 
-    ExecuteJob.prototype.getTypeDictFor = function (name, metanodes) {
+    GenerateJob.prototype.getTypeDictFor = function (name, metanodes) {
         var isType = {};
         // Get all the custom layers
         for (var i = metanodes.length; i--;) {
@@ -131,7 +318,7 @@ define([
         return isType;
     };
 
-    ExecuteJob.prototype.createCustomLayers = function (node, files) {
+    GenerateJob.prototype.createCustomLayers = function (node, files) {
         var metaDict = this.core.getAllMetaNodes(this.rootNode),
             isCustomLayer,
             metanodes,
@@ -153,7 +340,29 @@ define([
         files['custom-layers.lua'] = code;
     };
 
-    ExecuteJob.prototype.createInputs = function (node, files) {
+    GenerateJob.prototype.getConnectionContainer = function () {
+        var container = this.core.getParent(this.activeNode);
+
+        if (this.isMetaTypeOf(container, this.META.Job)) {
+            container = this.core.getParent(container);
+        }
+
+        return container;
+    };
+
+    GenerateJob.prototype.getInputPortsFor = function (nodeId) {
+        var container = this.getConnectionContainer();
+
+        // Get the connections to this node
+        return this.core.loadChildren(container)
+            .then(children => {
+                return children.filter(child =>
+                    this.core.getPointerPath(child, 'dst') === nodeId)
+                    .map(conn => this.core.getPointerPath(conn, 'src'))[0];
+            });
+    };
+
+    GenerateJob.prototype.createInputs = function (node, files) {
         var tplContents,
             inputs;
 
@@ -174,15 +383,13 @@ define([
                 return Q.all(inputs.map(pair => {
                     var name = pair[0],
                         node = pair[2],
-                        nodeId = this.core.getPath(node),
+                        nodeId = this.core.getPath(node);
-                        fromNodeId;
 
                     // Get the deserialize function. First, try to get it from
                     // the source method (this guarantees that the correct
                     // deserialize method is used despite any auto-upcasting
-                    fromNodeId = this.inputPortsFor[nodeId][0] || nodeId;
+                    return this.getInputPortsFor(nodeId)
-
+                        .then(fromNodeId => this.core.loadByPath(this.rootNode, fromNodeId || nodeId))
-                    return this.core.loadByPath(this.rootNode, fromNodeId)
                         .then(fromNode => {
                             var deserFn,
                                 base,
@@ -218,7 +425,9 @@ define([
 
                 return Q.all(hashes.map(pair => 
                     this.blobClient.getMetadata(pair.hash)
-                        .fail(err => this.onBlobRetrievalFail(node, pair.name, err))));
+                        .fail(() => {
+                            throw Error(`BLOB_FETCH_FAILED:${pair.name}`);
+                        })));
             })
             .then(metadatas => {
                 // Create the deserializer
@@ -231,7 +440,7 @@ define([
             });
     };
 
-    ExecuteJob.prototype.createOutputs = function (node, files) {
+    GenerateJob.prototype.createOutputs = function (node, files) {
         // For each of the output types, grab their serialization functions and
         // create the `outputs/init.lua` file
         this.logger.info('Creating outputs/init.lua...');
@@ -256,7 +465,7 @@ define([
             });
     };
 
-    ExecuteJob.prototype.createMainFile = function (node, files) {
+    GenerateJob.prototype.createMainFile = function (node, files) {
         this.logger.info('Creating main file...');
         return this.getInputs(node)
             .then(inputs => {
@@ -286,14 +495,14 @@ define([
             });
     };
 
-    ExecuteJob.prototype.getLineOffset = function (main, snippet) {
+    GenerateJob.prototype.getLineOffset = function (main, snippet) {
         var i = main.indexOf(snippet),
             lines = main.substring(0, i).match(/\n/g);
 
         return lines ? lines.length : 0;
     };
 
-    ExecuteJob.prototype.createAttributeFile = function (node, files) {
+    GenerateJob.prototype.createAttributeFile = function (node, files) {
         var numOrBool = /^(-?\d+\.?\d*((e|e-)\d+)?|(true|false))$/,
             table;
 
@@ -313,7 +522,7 @@ define([
         files['attributes.lua'] = `-- attributes of ${this.getAttribute(node, 'name')}\nreturn ${table}`;
     };
 
-    ExecuteJob.prototype.createPointers = function (node, files, cb) {
+    GenerateJob.prototype.createPointers = function (node, files, cb) {
         var pointers,
             nIds;
 
@@ -341,6 +550,19 @@ define([
         });
     };
 
-    return ExecuteJob;
+    GenerateJob.prototype.getAttribute = function (node, attr) {
-});
+        return this.core.getAttribute(node, attr);
+    };
 
+    GenerateJob.prototype.setAttribute = function (node, attr, value) {
+        return this.core.setAttribute(node, attr, value);
+    };
+
+    _.extend(
+        GenerateJob.prototype,
+        OperationHelpers.prototype,
+        PtrCodeGen.prototype
+    );
+
+    return GenerateJob;
+});

src/plugins/GenerateJob/metadata.json

@@ -0,0 +1,14 @@
+{
+  "id": "GenerateJob",
+  "name": "GenerateJob",
+  "version": "0.1.0",
+  "description": "",
+  "icon": {
+    "class": "glyphicon glyphicon-cog",
+    "src": ""
+  },
+  "disableServerSideExecution": false,
+  "disableBrowserSideExecution": false,
+  "writeAccessRequired": false,
+  "configStructure": []
+}

test/plugins/ExecuteJob/ExecuteJob.spec.js

@@ -300,62 +300,6 @@ describe('ExecuteJob', function () {
         });
     });
 
-    describe('exec files', function() {
-        describe('attribute file', function() {
-            var boolString = /['"](true|false)['"]/g;
-
-            beforeEach(preparePlugin);
-
-            it('should not quote true (s) boolean values', function() {
-                var files = {},
-                    content,
-                    matches;
-
-                plugin.setAttribute(node, 'debug', 'true');
-                plugin.createAttributeFile(node, files);
-                content = files['attributes.lua'];
-                matches = content.match(boolString);
-                expect(matches).to.equal(null);
-            });
-
-            it('should not quote true boolean values', function() {
-                var files = {},
-                    content,
-                    matches;
-
-                plugin.setAttribute(node, 'debug', true);
-                plugin.createAttributeFile(node, files);
-                content = files['attributes.lua'];
-                matches = content.match(boolString);
-                expect(matches).to.equal(null);
-            });
-
-            it('should not quote false (s) boolean values', function() {
-                var files = {},
-                    content,
-                    matches;
-
-                plugin.setAttribute(node, 'debug', 'false');
-                plugin.createAttributeFile(node, files);
-                content = files['attributes.lua'];
-                matches = content.match(boolString);
-                expect(matches).to.equal(null);
-            });
-
-            it('should not quote false boolean values', function() {
-                var files = {},
-                    content,
-                    matches;
-
-                plugin.setAttribute(node, 'debug', false);
-                plugin.createAttributeFile(node, files);
-                content = files['attributes.lua'];
-                matches = content.match(boolString);
-                expect(matches).to.equal(null);
-            });
-        });
-    });
-
     describe('resume detection', function() {
         var mockPluginForJobStatus = function(gmeStatus, pulse, originBranch, shouldResume, done) {
             plugin.setAttribute(node, 'status', gmeStatus);

test/plugins/GenerateJob/GenerateJob.spec.js

@@ -0,0 +1,175 @@
+/*jshint node:true, mocha:true*/
+/**
+ * Generated by PluginGenerator 1.7.0 from webgme on Mon Apr 17 2017 07:34:11 GMT-0500 (CDT).
+ */
+
+'use strict';
+var testFixture = require('../../globals');
+
+describe('GenerateJob', function () {
+    var gmeConfig = testFixture.getGmeConfig(),
+        expect = testFixture.expect,
+        logger = testFixture.logger.fork('GenerateJob'),
+        PluginCliManager = testFixture.WebGME.PluginCliManager,
+        manager = new PluginCliManager(null, logger, gmeConfig),
+        projectName = 'testProject',
+        pluginName = 'GenerateJob',
+        project,
+        gmeAuth,
+        storage,
+        commitHash;
+
+    before(function (done) {
+        testFixture.clearDBAndGetGMEAuth(gmeConfig, projectName)
+            .then(function (gmeAuth_) {
+                gmeAuth = gmeAuth_;
+                // This uses in memory storage. Use testFixture.getMongoStorage to persist test to database.
+                storage = testFixture.getMemoryStorage(logger, gmeConfig, gmeAuth);
+                return storage.openDatabase();
+            })
+            .then(function () {
+                var importParam = {
+                    projectSeed: testFixture.path.join(testFixture.DF_SEED_DIR, 'devProject', 'devProject.webgmex'),
+                    projectName: projectName,
+                    branchName: 'master',
+                    logger: logger,
+                    gmeConfig: gmeConfig
+                };
+
+                return testFixture.importProject(storage, importParam);
+            })
+            .then(function (importResult) {
+                project = importResult.project;
+                commitHash = importResult.commitHash;
+                return project.createBranch('test', commitHash);
+            })
+            .nodeify(done);
+    });
+
+    after(function (done) {
+        storage.closeDatabase()
+            .then(function () {
+                return gmeAuth.unload();
+            })
+            .nodeify(done);
+    });
+
+    describe('basic checks', function() {
+        var pluginResult,
+            error;
+
+        before(function(done) {
+            var pluginConfig = {
+                },
+                context = {
+                    project: project,
+                    commitHash: commitHash,
+                    branchName: 'test',
+                    activeNode: '/1',
+                };
+
+            manager.executePlugin(pluginName, pluginConfig, context, (err, result) => {
+                error = err;
+                pluginResult = result;
+
+                done();
+            });
+        });
+
+        it('should run without error', function () {
+            expect(error).to.equal(null);
+            expect(typeof pluginResult).to.equal('object');
+            expect(pluginResult.success).to.equal(true);
+        });
+
+        it('should generate artifacts', function () {
+            expect(pluginResult.artifacts[0]).to.not.equal(undefined);
+        });
+
+        it('should NOT update the branch', function (done) {
+            project.getBranchHash('test')
+                .then(function (branchHash) {
+                    expect(branchHash).to.equal(commitHash);
+                })
+                .nodeify(done);
+        });
+    });
+
+    ////////// Helper Functions //////////
+    var plugin,
+        node,
+        preparePlugin = function(done) {
+            var context = {
+                project: project,
+                commitHash: commitHash,
+                namespace: 'pipeline',
+                branchName: 'test',
+                activeNode: '/K/R/p/m'  // hello world operation
+            };
+
+            return manager.initializePlugin(pluginName)
+                .then(plugin_ => {
+                    plugin = plugin_;
+                    return manager.configurePlugin(plugin, {}, context);
+                })
+                .then(() => node = plugin.activeNode)
+                .nodeify(done);
+        };
+
+    describe('exec files', function() {
+        describe('attribute file', function() {
+            var boolString = /['"](true|false)['"]/g;
+
+            beforeEach(preparePlugin);
+
+            it('should not quote true (s) boolean values', function() {
+                var files = {},
+                    content,
+                    matches;
+
+                plugin.setAttribute(node, 'debug', 'true');
+                plugin.createAttributeFile(node, files);
+                content = files['attributes.lua'];
+                matches = content.match(boolString);
+                expect(matches).to.equal(null);
+            });
+
+            it('should not quote true boolean values', function() {
+                var files = {},
+                    content,
+                    matches;
+
+                plugin.setAttribute(node, 'debug', true);
+                plugin.createAttributeFile(node, files);
+                content = files['attributes.lua'];
+                matches = content.match(boolString);
+                expect(matches).to.equal(null);
+            });
+
+            it('should not quote false (s) boolean values', function() {
+                var files = {},
+                    content,
+                    matches;
+
+                plugin.setAttribute(node, 'debug', 'false');
+                plugin.createAttributeFile(node, files);
+                content = files['attributes.lua'];
+                matches = content.match(boolString);
+                expect(matches).to.equal(null);
+            });
+
+            it('should not quote false boolean values', function() {
+                var files = {},
+                    content,
+                    matches;
+
+                plugin.setAttribute(node, 'debug', false);
+                plugin.createAttributeFile(node, files);
+                content = files['attributes.lua'];
+                matches = content.match(boolString);
+                expect(matches).to.equal(null);
+            });
+        });
+    });
+
+});

webgme-setup.json

@@ -56,6 +56,10 @@
       "ValidateArchitecture": {
         "src": "src/plugins/ValidateArchitecture",
         "test": "test/plugins/ValidateArchitecture"
+      },
+      "GenerateJob": {
+        "src": "src/plugins/GenerateJob",
+        "test": "test/plugins/GenerateJob"
       }
     },
     "layouts": {