diff --git a/src/common/plugin/Operation.js b/src/common/plugin/Operation.js new file mode 100644 index 0000000..a50c297 --- /dev/null +++ b/src/common/plugin/Operation.js @@ -0,0 +1,35 @@ +/*globals define */ +// This is a mixin containing helpers for working with operation nodes +define([],function() { + + var OperationOps = function() { + }; + + OperationOps.prototype.getOutputs = function (node) { + return this.getOperationData(node, this.META.Outputs); + }; + + OperationOps.prototype.getInputs = function (node) { + return this.getOperationData(node, this.META.Inputs); + }; + + OperationOps.prototype.getOperationData = function (node, metaType) { + // Load the children and the output's children + return this.core.loadChildren(node) + .then(containers => { + var outputs = containers.find(c => this.core.isTypeOf(c, metaType)); + return outputs ? this.core.loadChildren(outputs) : []; + }) + .then(outputs => { + var bases = outputs.map(node => this.core.getMetaType(node)); + // return [[arg1, Type1, node1], [arg2, Type2, node2]] + return outputs.map((node, i) => [ + this.getAttribute(node, 'name'), + this.getAttribute(bases[i], 'name'), + node + ]); + }); + }; + + return OperationOps; +}); diff --git a/src/common/plugin/PtrCodeGen.js b/src/common/plugin/PtrCodeGen.js index bf60fa4..5c63e37 100644 --- a/src/common/plugin/PtrCodeGen.js +++ b/src/common/plugin/PtrCodeGen.js @@ -6,27 +6,64 @@ define([ PluginUtils, Q ) { + + var CodeGen = { + Operation: { + pluginId: 'GenerateJob', + namespace: 'pipeline' + } + }; + var PtrCodeGen = function() { }; + PtrCodeGen.prototype.getCodeGenPluginIdFor = function(node) { + var base = this.core.getBase(node), + name = this.core.getAttribute(node, 'name'), + namespace = this.core.getNamespace(node), + pluginId; + + //this.logger.debug(`loaded pointer target of ${ptrId}: ${ptrNode}`); + pluginId = (this.core.getOwnRegistry(node, 'validPlugins') || '').split(' ').shift(); + //this.logger.info(`generating code for ${this.core.getAttribute(ptrNode, 'name')} using ${pluginId}`); + + if (this.core.isMetaNode(node) && CodeGen[name]) { + pluginId = CodeGen[name].pluginId || CodeGen[name]; + namespace = CodeGen[name].namespace; + } + + if (pluginId) { + return { + namespace: namespace, + pluginId: pluginId + }; + } else if (base) { + return this.getCodeGenPluginIdFor(base); + } else { + return null; + } + }; + PtrCodeGen.prototype.getPtrCodeHash = function(ptrId) { return this.core.loadByPath(this.rootNode, ptrId) .then(ptrNode => { // Look up the plugin to use - var metanode = this.core.getMetaType(ptrNode), - pluginId; + var genInfo = this.getCodeGenPluginIdFor(ptrNode); - this.logger.debug(`loaded pointer target of ${ptrId}: ${ptrNode}`); - pluginId = this.core.getRegistry(ptrNode, 'validPlugins').split(' ').shift(); - this.logger.info(`generating code for ${this.core.getAttribute(ptrNode, 'name')} using ${pluginId}`); + if (genInfo.pluginId) { + var context = { + namespace: genInfo.namespace, + activeNode: this.core.getPath(ptrNode) + }; - var context = { - namespace: this.core.getNamespace(metanode), - activeNode: this.core.getPath(ptrNode) - }; - - // Load and run the plugin - return this.executePlugin(pluginId, context); + // Load and run the plugin + return this.executePlugin(genInfo.pluginId, context); + } else { + var metanode = this.core.getMetaType(ptrNode), + type = this.core.getAttribute(metanode, 'name'); + this.logger.warn(`Could not find plugin for ${type}. Will try to proceed anyway`); + return null; + } }) .then(hashes => hashes[0]); // Grab the first asset for now }; @@ -56,12 +93,13 @@ define([ return PluginUtils.loadNodesAtCommitHash( this.project, this.core, - this.commitHash, + this.currentHash, this.logger, opts ).then(config => { plugin.initialize(logger, this.blobClient, this.gmeConfig); config.core = this.core; + config.project = this.project; plugin.configure(config); return plugin; }); diff --git a/src/plugins/ExecuteJob/ExecuteJob.js b/src/plugins/ExecuteJob/ExecuteJob.js index 998490f..8fdd824 100644 --- a/src/plugins/ExecuteJob/ExecuteJob.js +++ b/src/plugins/ExecuteJob/ExecuteJob.js @@ -8,10 +8,10 @@ define([ 'plugin/PluginBase', 'deepforge/plugin/LocalExecutor', 'deepforge/plugin/PtrCodeGen', + 'deepforge/plugin/Operation', 'deepforge/api/JobLogsClient', 'deepforge/api/JobOriginClient', 'deepforge/api/ExecPulseClient', - './ExecuteJob.Files', './ExecuteJob.Metadata', './ExecuteJob.SafeSave', 'deepforge/Constants', @@ -26,11 +26,11 @@ define([ PluginBase, LocalExecutor, // DeepForge operation primitives PtrCodeGen, + OperationPlugin, JobLogsClient, JobOriginClient, ExecPulseClient, - ExecuteJobFiles, ExecuteJobMetadata, ExecuteJobSafeSave, @@ -44,8 +44,7 @@ define([ pluginMetadata = JSON.parse(pluginMetadata); - var OUTPUT_INTERVAL = 1500, - STDOUT_FILE = 'job_stdout.txt'; + var STDOUT_FILE = 'job_stdout.txt'; /** * Initializes a new instance of ExecuteJob. @@ -453,9 +452,10 @@ define([ children.find(child => this.isMetaTypeOf(child, this.META.Operation))); }; - ExecuteJob.prototype.onBlobRetrievalFail = function (node, input, err) { + // Handle the blob retrieval failed error + ExecuteJob.prototype.onBlobRetrievalFail = function (node, input) { var job = this.core.getParent(node), - e = `Failed to retrieve "${input}" (${err})`, + e = `Failed to retrieve "${input}" (BLOB_FETCH_FAILED)`, consoleErr = `Failed to execute operation: ${e}`; consoleErr += [ @@ -473,14 +473,8 @@ define([ ExecuteJob.prototype.executeJob = function (job) { return this.getOperation(job).then(node => { - var jobId = this.core.getPath(job), - name = this.getAttribute(node, 'name'), - localTypeId = this.getLocalOperationType(node), - artifact, - artifactName, - files, - data = {}, - inputs; + var name = this.getAttribute(node, 'name'), + localTypeId = this.getLocalOperationType(node); // Execute any special operation types here - not on an executor this.logger.debug(`Executing operation "${name}"`); @@ -488,126 +482,22 @@ define([ return this.executeLocalOperation(localTypeId, node); } else { // Generate all execution files - return this.createOperationFiles(node).then(results => { - this.logger.info('Created operation files!'); - files = results; - artifactName = `${name}_${jobId.replace(/\//g, '_')}-execution-files`; - artifact = this.blobClient.createArtifact(artifactName); - - // Add the input assets - // - get the metadata (name) - // - add the given inputs - inputs = Object.keys(files.inputAssets); - - return Q.all( - inputs.map(input => { // Get the metadata for each input - var hash = files.inputAssets[input]; - - // data asset for "input" - return this.blobClient.getMetadata(hash) - .fail(err => this.onBlobRetrievalFail(job, input, err)); - }) - ); - }) - .then(mds => { - // Record the large files - var inputData = {}, - runsh = '# Bash script to download data files and run job\n' + - 'if [ -z "$DEEPFORGE_URL" ]; then\n echo "Please set DEEPFORGE_URL and' + - ' re-run:"\n echo "" \n echo " DEEPFORGE_URL=http://my.' + - 'deepforge.server.com:8080 bash run.sh"\n echo ""\n exit 1\nfi\n'; - - mds.forEach((metadata, i) => { - // add the hashes for each input - var input = inputs[i], - hash = files.inputAssets[input], - dataPath = 'inputs/' + input + '/data', - url = this.blobClient.getRelativeDownloadURL(hash); - - inputData[dataPath] = { - req: hash, - cache: metadata.content - }; - - // Add to the run.sh file - runsh += `wget $DEEPFORGE_URL${url} -O ${dataPath}\n`; + return this.getPtrCodeHash(this.core.getPath(node)) + .fail(err => { + this.logger.error(`Could not generate files: ${err}`); + if (err.message.indexOf('BLOB_FETCH_FAILED') > -1) { + this.onBlobRetrievalFail(node, err.message.split(':')[1]); + } + throw err; + }) + .then(hash => { + this.logger.info(`Saved execution files`); + this.result.addArtifact(hash); // Probably only need this for debugging... + this.executeDistOperation(job, node, hash); + }) + .fail(e => { + this.onOperationFail(node, `Distributed operation "${name}" failed ${e}`); }); - - delete files.inputAssets; - files['input-data.json'] = JSON.stringify(inputData, null, 2); - runsh += 'th init.lua'; - files['run.sh'] = runsh; - - // Add pointer assets - Object.keys(files.ptrAssets) - .forEach(path => data[path] = files.ptrAssets[path]); - - // Add the executor config - return this.getOutputs(node); - }) - .then(outputArgs => { - var config, - outputs, - fileList, - ptrFiles = Object.keys(files.ptrAssets), - file; - - delete files.ptrAssets; - fileList = Object.keys(files).concat(ptrFiles); - - outputs = outputArgs.map(pair => pair[0]) - .map(name => { - return { - name: name, - resultPatterns: [`outputs/${name}`] - }; - }); - - outputs.push( - { - name: 'stdout', - resultPatterns: [STDOUT_FILE] - }, - { - name: name + '-all-files', - resultPatterns: fileList - } - ); - - config = { - cmd: 'node', - args: ['start.js'], - outputInterval: OUTPUT_INTERVAL, - resultArtifacts: outputs - }; - files['executor_config.json'] = JSON.stringify(config, null, 4); - - // Save the artifact - // Remove empty hashes - for (file in data) { - if (!data[file]) { - this.logger.warn(`Empty data hash has been found for file "${file}". Removing it...`); - delete data[file]; - } - } - return artifact.addObjectHashes(data); - }) - .then(() => { - this.logger.info(`Added ptr/input data hashes for "${artifactName}"`); - return artifact.addFiles(files); - }) - .then(() => { - this.logger.info(`Added execution files for "${artifactName}"`); - return artifact.save(); - }) - .then(hash => { - this.logger.info(`Saved execution files "${artifactName}"`); - this.result.addArtifact(hash); // Probably only need this for debugging... - this.executeDistOperation(job, node, hash); - }) - .fail(e => { - this.onOperationFail(node, `Distributed operation "${name}" failed ${e}`); - }); } }); }; @@ -881,32 +771,6 @@ define([ .fail(e => this.onOperationFail(node, `Operation ${nodeId} failed: ${e}`)); }; - ExecuteJob.prototype.getOutputs = function (node) { - return this.getOperationData(node, this.META.Outputs); - }; - - ExecuteJob.prototype.getInputs = function (node) { - return this.getOperationData(node, this.META.Inputs); - }; - - ExecuteJob.prototype.getOperationData = function (node, metaType) { - // Load the children and the output's children - return this.core.loadChildren(node) - .then(containers => { - var outputs = containers.find(c => this.core.isTypeOf(c, metaType)); - return outputs ? this.core.loadChildren(outputs) : []; - }) - .then(outputs => { - var bases = outputs.map(node => this.core.getMetaType(node)); - // return [[arg1, Type1, node1], [arg2, Type2, node2]] - return outputs.map((node, i) => [ - this.getAttribute(node, 'name'), - this.getAttribute(bases[i], 'name'), - node - ]); - }); - }; - //////////////////////////// Special Operations //////////////////////////// ExecuteJob.prototype.executeLocalOperation = function (type, node) { // Retrieve the given LOCAL_OP type @@ -920,7 +784,7 @@ define([ _.extend( ExecuteJob.prototype, - ExecuteJobFiles.prototype, + OperationPlugin.prototype, ExecuteJobMetadata.prototype, ExecuteJobSafeSave.prototype, PtrCodeGen.prototype, diff --git a/src/plugins/ExecutePipeline/ExecutePipeline.js b/src/plugins/ExecutePipeline/ExecutePipeline.js index 084f6bf..e3d7a19 100644 --- a/src/plugins/ExecutePipeline/ExecutePipeline.js +++ b/src/plugins/ExecutePipeline/ExecutePipeline.js @@ -175,6 +175,7 @@ define([ ExecutePipeline.prototype.resumePipeline = function () { var nodes = Object.keys(this.nodes).map(id => this.nodes[id]), allJobs = nodes.filter(node => this.core.isTypeOf(node, this.META.Job)), + name = this.getAttribute(this.activeNode, 'name'), status, jobs = { success: [], @@ -208,6 +209,7 @@ define([ return Q.all(allJobs.map(job => this.recordOldMetadata(job, true))) .then(() => Q.all(jobs.success.map(job => this.getOperation(job)))) .then(ops => ops.forEach(op => this.updateJobCompletionRecords(op))) + .then(() => this.save(`Resuming pipeline execution: ${name}`)) .then(() => { if (jobs.running.length) { // Resume all running jobs @@ -531,10 +533,12 @@ define([ this.logger.info(`Setting ${jobId} status to "success"`); this.logger.info(`There are now ${this.runningJobs} running jobs`); this.logger.debug(`Making a commit from ${this.currentHash}`); + + counts = this.updateJobCompletionRecords(opNode); + this.save(`Operation "${name}" in ${this.pipelineName} completed successfully`) .then(() => { - counts = this.updateJobCompletionRecords(opNode); hasReadyOps = counts.indexOf(0) > -1; this.logger.debug(`Operation "${name}" completed. ` + diff --git a/src/plugins/ExecuteJob/ExecuteJob.Files.js b/src/plugins/GenerateJob/GenerateJob.js similarity index 55% rename from src/plugins/ExecuteJob/ExecuteJob.Files.js rename to src/plugins/GenerateJob/GenerateJob.js index 3d5561f..c1f9658 100644 --- a/src/plugins/ExecuteJob/ExecuteJob.Files.js +++ b/src/plugins/GenerateJob/GenerateJob.js @@ -1,29 +1,212 @@ /*globals define*/ +/*jshint node:true, browser:true*/ + define([ './templates/index', 'q', 'underscore', - 'deepforge/Constants' -], function( + 'deepforge/Constants', + 'deepforge/plugin/Operation', + 'deepforge/plugin/PtrCodeGen', + 'text!./metadata.json', + 'plugin/PluginBase' +], function ( Templates, Q, _, - CONSTANTS + CONSTANTS, + OperationHelpers, + PtrCodeGen, + pluginMetadata, + PluginBase ) { + 'use strict'; - var SKIP_ATTRIBUTES = [ - 'code', - 'stdout', - 'execFiles', - 'jobId', - 'secret', - CONSTANTS.LINE_OFFSET, - CONSTANTS.DISPLAY_COLOR - ]; - var ExecuteJob = function() { + pluginMetadata = JSON.parse(pluginMetadata); + var OUTPUT_INTERVAL = 1500, + STDOUT_FILE = 'job_stdout.txt', + SKIP_ATTRIBUTES = [ + 'code', + 'stdout', + 'execFiles', + 'jobId', + 'secret', + CONSTANTS.LINE_OFFSET, + CONSTANTS.DISPLAY_COLOR + ]; + + /** + * Initializes a new instance of GenerateJob. + * @class + * @augments {PluginBase} + * @classdesc This class represents the plugin GenerateJob. + * @constructor + */ + var GenerateJob = function () { + // Call base class' constructor. + PluginBase.call(this); + this.pluginMetadata = pluginMetadata; }; - ExecuteJob.prototype.createOperationFiles = function (node) { + /** + * Metadata associated with the plugin. Contains id, name, version, description, icon, configStructue etc. + * This is also available at the instance at this.pluginMetadata. + * @type {object} + */ + GenerateJob.metadata = pluginMetadata; + + // Prototypical inheritance from PluginBase. + GenerateJob.prototype = Object.create(PluginBase.prototype); + GenerateJob.prototype.constructor = GenerateJob; + + /** + * Main function for the plugin to execute. This will perform the execution. + * Notes: + * - Always log with the provided logger.[error,warning,info,debug]. + * - Do NOT put any user interaction logic UI, etc. inside this method. + * - callback always has to be called even if error happened. + * + * @param {function(string, plugin.PluginResult)} callback - the result callback + */ + GenerateJob.prototype.main = function (callback) { + var files, + artifactName, + artifact, + data = {}, + inputs, + name, + opId; + + name = this.getAttribute(this.activeNode, 'name'); + opId = this.core.getPath(this.activeNode); + + return this.createOperationFiles(this.activeNode) + .then(results => { + this.logger.info('Created operation files!'); + files = results; + artifactName = `${name}_${opId.replace(/\//g, '_')}-execution-files`; + artifact = this.blobClient.createArtifact(artifactName); + + // Add the input assets + // - get the metadata (name) + // - add the given inputs + inputs = Object.keys(files.inputAssets); + + return Q.all( + inputs.map(input => { // Get the metadata for each input + var hash = files.inputAssets[input]; + + // data asset for "input" + return this.blobClient.getMetadata(hash) + .fail(() => { + throw Error(`BLOB_FETCH_FAILED:${input}`); + }); + }) + ); + }) + .then(mds => { + // Record the large files + var inputData = {}, + runsh = '# Bash script to download data files and run job\n' + + 'if [ -z "$DEEPFORGE_URL" ]; then\n echo "Please set DEEPFORGE_URL and' + + ' re-run:"\n echo "" \n echo " DEEPFORGE_URL=http://my.' + + 'deepforge.server.com:8080 bash run.sh"\n echo ""\n exit 1\nfi\n'; + + mds.forEach((metadata, i) => { + // add the hashes for each input + var input = inputs[i], + hash = files.inputAssets[input], + dataPath = 'inputs/' + input + '/data', + url = this.blobClient.getRelativeDownloadURL(hash); + + inputData[dataPath] = { + req: hash, + cache: metadata.content + }; + + // Add to the run.sh file + runsh += `wget $DEEPFORGE_URL${url} -O ${dataPath}\n`; + }); + + delete files.inputAssets; + files['input-data.json'] = JSON.stringify(inputData, null, 2); + runsh += 'th init.lua'; + files['run.sh'] = runsh; + + // Add pointer assets + Object.keys(files.ptrAssets) + .forEach(path => data[path] = files.ptrAssets[path]); + + // Add the executor config + return this.getOutputs(this.activeNode); + }) + .then(outputArgs => { + var config, + outputs, + fileList, + ptrFiles = Object.keys(files.ptrAssets), + file; + + delete files.ptrAssets; + fileList = Object.keys(files).concat(ptrFiles); + + outputs = outputArgs.map(pair => pair[0]) + .map(name => { + return { + name: name, + resultPatterns: [`outputs/${name}`] + }; + }); + + outputs.push( + { + name: 'stdout', + resultPatterns: [STDOUT_FILE] + }, + { + name: name + '-all-files', + resultPatterns: fileList + } + ); + + config = { + cmd: 'node', + args: ['start.js'], + outputInterval: OUTPUT_INTERVAL, + resultArtifacts: outputs + }; + files['executor_config.json'] = JSON.stringify(config, null, 4); + + // Save the artifact + // Remove empty hashes + for (file in data) { + if (!data[file]) { + this.logger.warn(`Empty data hash has been found for file "${file}". Removing it...`); + delete data[file]; + } + } + return artifact.addObjectHashes(data); + }) + .then(() => { + this.logger.info(`Added ptr/input data hashes for "${artifactName}"`); + return artifact.addFiles(files); + }) + .then(() => { + this.logger.info(`Added execution files for "${artifactName}"`); + return artifact.save(); + }) + .then(hash => { + this.result.setSuccess(true); + this.result.addArtifact(hash); + callback(null, this.result); + }) + .fail(err => { + this.result.setSuccess(false); + callback(err, this.result); + }); + }; + + GenerateJob.prototype.createOperationFiles = function (node) { var files = {}; // For each operation, generate the output files: // inputs//init.lua (respective data deserializer) @@ -47,10 +230,14 @@ define([ .then(() => { this.createAttributeFile(node, files); return Q.ninvoke(this, 'createPointers', node, files); + }) + .fail(err => { + this.logger.error(err); + throw err; }); }; - ExecuteJob.prototype.createEntryFile = function (node, files) { + GenerateJob.prototype.createEntryFile = function (node, files) { this.logger.info('Creating entry files...'); return this.getOutputs(node) .then(outputs => { @@ -68,7 +255,7 @@ define([ }); }; - ExecuteJob.prototype.createClasses = function (node, files) { + GenerateJob.prototype.createClasses = function (node, files) { var metaDict = this.core.getAllMetaNodes(this.rootNode), isClass, metanodes, @@ -120,7 +307,7 @@ define([ files['classes/init.lua'] = code; }; - ExecuteJob.prototype.getTypeDictFor = function (name, metanodes) { + GenerateJob.prototype.getTypeDictFor = function (name, metanodes) { var isType = {}; // Get all the custom layers for (var i = metanodes.length; i--;) { @@ -131,7 +318,7 @@ define([ return isType; }; - ExecuteJob.prototype.createCustomLayers = function (node, files) { + GenerateJob.prototype.createCustomLayers = function (node, files) { var metaDict = this.core.getAllMetaNodes(this.rootNode), isCustomLayer, metanodes, @@ -153,7 +340,29 @@ define([ files['custom-layers.lua'] = code; }; - ExecuteJob.prototype.createInputs = function (node, files) { + GenerateJob.prototype.getConnectionContainer = function () { + var container = this.core.getParent(this.activeNode); + + if (this.isMetaTypeOf(container, this.META.Job)) { + container = this.core.getParent(container); + } + + return container; + }; + + GenerateJob.prototype.getInputPortsFor = function (nodeId) { + var container = this.getConnectionContainer(); + + // Get the connections to this node + return this.core.loadChildren(container) + .then(children => { + return children.filter(child => + this.core.getPointerPath(child, 'dst') === nodeId) + .map(conn => this.core.getPointerPath(conn, 'src'))[0]; + }); + }; + + GenerateJob.prototype.createInputs = function (node, files) { var tplContents, inputs; @@ -174,15 +383,13 @@ define([ return Q.all(inputs.map(pair => { var name = pair[0], node = pair[2], - nodeId = this.core.getPath(node), - fromNodeId; + nodeId = this.core.getPath(node); // Get the deserialize function. First, try to get it from // the source method (this guarantees that the correct // deserialize method is used despite any auto-upcasting - fromNodeId = this.inputPortsFor[nodeId][0] || nodeId; - - return this.core.loadByPath(this.rootNode, fromNodeId) + return this.getInputPortsFor(nodeId) + .then(fromNodeId => this.core.loadByPath(this.rootNode, fromNodeId || nodeId)) .then(fromNode => { var deserFn, base, @@ -218,7 +425,9 @@ define([ return Q.all(hashes.map(pair => this.blobClient.getMetadata(pair.hash) - .fail(err => this.onBlobRetrievalFail(node, pair.name, err)))); + .fail(() => { + throw Error(`BLOB_FETCH_FAILED:${pair.name}`); + }))); }) .then(metadatas => { // Create the deserializer @@ -231,7 +440,7 @@ define([ }); }; - ExecuteJob.prototype.createOutputs = function (node, files) { + GenerateJob.prototype.createOutputs = function (node, files) { // For each of the output types, grab their serialization functions and // create the `outputs/init.lua` file this.logger.info('Creating outputs/init.lua...'); @@ -256,7 +465,7 @@ define([ }); }; - ExecuteJob.prototype.createMainFile = function (node, files) { + GenerateJob.prototype.createMainFile = function (node, files) { this.logger.info('Creating main file...'); return this.getInputs(node) .then(inputs => { @@ -286,14 +495,14 @@ define([ }); }; - ExecuteJob.prototype.getLineOffset = function (main, snippet) { + GenerateJob.prototype.getLineOffset = function (main, snippet) { var i = main.indexOf(snippet), lines = main.substring(0, i).match(/\n/g); return lines ? lines.length : 0; }; - ExecuteJob.prototype.createAttributeFile = function (node, files) { + GenerateJob.prototype.createAttributeFile = function (node, files) { var numOrBool = /^(-?\d+\.?\d*((e|e-)\d+)?|(true|false))$/, table; @@ -313,7 +522,7 @@ define([ files['attributes.lua'] = `-- attributes of ${this.getAttribute(node, 'name')}\nreturn ${table}`; }; - ExecuteJob.prototype.createPointers = function (node, files, cb) { + GenerateJob.prototype.createPointers = function (node, files, cb) { var pointers, nIds; @@ -341,6 +550,19 @@ define([ }); }; - return ExecuteJob; -}); + GenerateJob.prototype.getAttribute = function (node, attr) { + return this.core.getAttribute(node, attr); + }; + GenerateJob.prototype.setAttribute = function (node, attr, value) { + return this.core.setAttribute(node, attr, value); + }; + + _.extend( + GenerateJob.prototype, + OperationHelpers.prototype, + PtrCodeGen.prototype + ); + + return GenerateJob; +}); diff --git a/src/plugins/GenerateJob/metadata.json b/src/plugins/GenerateJob/metadata.json new file mode 100644 index 0000000..9470545 --- /dev/null +++ b/src/plugins/GenerateJob/metadata.json @@ -0,0 +1,14 @@ +{ + "id": "GenerateJob", + "name": "GenerateJob", + "version": "0.1.0", + "description": "", + "icon": { + "class": "glyphicon glyphicon-cog", + "src": "" + }, + "disableServerSideExecution": false, + "disableBrowserSideExecution": false, + "writeAccessRequired": false, + "configStructure": [] +} \ No newline at end of file diff --git a/src/plugins/ExecuteJob/templates/deepforge.ejs b/src/plugins/GenerateJob/templates/deepforge.ejs similarity index 100% rename from src/plugins/ExecuteJob/templates/deepforge.ejs rename to src/plugins/GenerateJob/templates/deepforge.ejs diff --git a/src/plugins/ExecuteJob/templates/deserialize.ejs b/src/plugins/GenerateJob/templates/deserialize.ejs similarity index 100% rename from src/plugins/ExecuteJob/templates/deserialize.ejs rename to src/plugins/GenerateJob/templates/deserialize.ejs diff --git a/src/plugins/ExecuteJob/templates/entry.ejs b/src/plugins/GenerateJob/templates/entry.ejs similarity index 100% rename from src/plugins/ExecuteJob/templates/entry.ejs rename to src/plugins/GenerateJob/templates/entry.ejs diff --git a/src/plugins/ExecuteJob/templates/index.js b/src/plugins/GenerateJob/templates/index.js similarity index 100% rename from src/plugins/ExecuteJob/templates/index.js rename to src/plugins/GenerateJob/templates/index.js diff --git a/src/plugins/ExecuteJob/templates/main.ejs b/src/plugins/GenerateJob/templates/main.ejs similarity index 100% rename from src/plugins/ExecuteJob/templates/main.ejs rename to src/plugins/GenerateJob/templates/main.ejs diff --git a/src/plugins/ExecuteJob/templates/serialize.ejs b/src/plugins/GenerateJob/templates/serialize.ejs similarity index 100% rename from src/plugins/ExecuteJob/templates/serialize.ejs rename to src/plugins/GenerateJob/templates/serialize.ejs diff --git a/src/plugins/ExecuteJob/templates/start.ejs b/src/plugins/GenerateJob/templates/start.ejs similarity index 100% rename from src/plugins/ExecuteJob/templates/start.ejs rename to src/plugins/GenerateJob/templates/start.ejs diff --git a/test/plugins/ExecuteJob/ExecuteJob.spec.js b/test/plugins/ExecuteJob/ExecuteJob.spec.js index 34bd9cc..ce9cd4b 100644 --- a/test/plugins/ExecuteJob/ExecuteJob.spec.js +++ b/test/plugins/ExecuteJob/ExecuteJob.spec.js @@ -300,62 +300,6 @@ describe('ExecuteJob', function () { }); }); - describe('exec files', function() { - describe('attribute file', function() { - var boolString = /['"](true|false)['"]/g; - - beforeEach(preparePlugin); - - it('should not quote true (s) boolean values', function() { - var files = {}, - content, - matches; - - plugin.setAttribute(node, 'debug', 'true'); - plugin.createAttributeFile(node, files); - content = files['attributes.lua']; - matches = content.match(boolString); - expect(matches).to.equal(null); - }); - - it('should not quote true boolean values', function() { - var files = {}, - content, - matches; - - plugin.setAttribute(node, 'debug', true); - plugin.createAttributeFile(node, files); - content = files['attributes.lua']; - matches = content.match(boolString); - expect(matches).to.equal(null); - }); - - it('should not quote false (s) boolean values', function() { - var files = {}, - content, - matches; - - plugin.setAttribute(node, 'debug', 'false'); - plugin.createAttributeFile(node, files); - content = files['attributes.lua']; - matches = content.match(boolString); - expect(matches).to.equal(null); - }); - - it('should not quote false boolean values', function() { - var files = {}, - content, - matches; - - plugin.setAttribute(node, 'debug', false); - plugin.createAttributeFile(node, files); - content = files['attributes.lua']; - matches = content.match(boolString); - expect(matches).to.equal(null); - }); - }); - }); - describe('resume detection', function() { var mockPluginForJobStatus = function(gmeStatus, pulse, originBranch, shouldResume, done) { plugin.setAttribute(node, 'status', gmeStatus); diff --git a/test/plugins/GenerateJob/GenerateJob.spec.js b/test/plugins/GenerateJob/GenerateJob.spec.js new file mode 100644 index 0000000..e1cb94c --- /dev/null +++ b/test/plugins/GenerateJob/GenerateJob.spec.js @@ -0,0 +1,175 @@ +/*jshint node:true, mocha:true*/ +/** + * Generated by PluginGenerator 1.7.0 from webgme on Mon Apr 17 2017 07:34:11 GMT-0500 (CDT). + */ + +'use strict'; +var testFixture = require('../../globals'); + +describe('GenerateJob', function () { + var gmeConfig = testFixture.getGmeConfig(), + expect = testFixture.expect, + logger = testFixture.logger.fork('GenerateJob'), + PluginCliManager = testFixture.WebGME.PluginCliManager, + manager = new PluginCliManager(null, logger, gmeConfig), + projectName = 'testProject', + pluginName = 'GenerateJob', + project, + gmeAuth, + storage, + commitHash; + + before(function (done) { + testFixture.clearDBAndGetGMEAuth(gmeConfig, projectName) + .then(function (gmeAuth_) { + gmeAuth = gmeAuth_; + // This uses in memory storage. Use testFixture.getMongoStorage to persist test to database. + storage = testFixture.getMemoryStorage(logger, gmeConfig, gmeAuth); + return storage.openDatabase(); + }) + .then(function () { + var importParam = { + projectSeed: testFixture.path.join(testFixture.DF_SEED_DIR, 'devProject', 'devProject.webgmex'), + projectName: projectName, + branchName: 'master', + logger: logger, + gmeConfig: gmeConfig + }; + + return testFixture.importProject(storage, importParam); + }) + .then(function (importResult) { + project = importResult.project; + commitHash = importResult.commitHash; + return project.createBranch('test', commitHash); + }) + .nodeify(done); + }); + + after(function (done) { + storage.closeDatabase() + .then(function () { + return gmeAuth.unload(); + }) + .nodeify(done); + }); + + describe('basic checks', function() { + var pluginResult, + error; + + before(function(done) { + var pluginConfig = { + }, + context = { + project: project, + commitHash: commitHash, + branchName: 'test', + activeNode: '/1', + }; + + manager.executePlugin(pluginName, pluginConfig, context, (err, result) => { + error = err; + pluginResult = result; + + done(); + }); + }); + + it('should run without error', function () { + expect(error).to.equal(null); + expect(typeof pluginResult).to.equal('object'); + expect(pluginResult.success).to.equal(true); + }); + + it('should generate artifacts', function () { + expect(pluginResult.artifacts[0]).to.not.equal(undefined); + }); + + it('should NOT update the branch', function (done) { + project.getBranchHash('test') + .then(function (branchHash) { + expect(branchHash).to.equal(commitHash); + }) + .nodeify(done); + }); + }); + + ////////// Helper Functions ////////// + var plugin, + node, + preparePlugin = function(done) { + var context = { + project: project, + commitHash: commitHash, + namespace: 'pipeline', + branchName: 'test', + activeNode: '/K/R/p/m' // hello world operation + }; + + return manager.initializePlugin(pluginName) + .then(plugin_ => { + plugin = plugin_; + return manager.configurePlugin(plugin, {}, context); + }) + .then(() => node = plugin.activeNode) + .nodeify(done); + }; + + describe('exec files', function() { + describe('attribute file', function() { + var boolString = /['"](true|false)['"]/g; + + beforeEach(preparePlugin); + + it('should not quote true (s) boolean values', function() { + var files = {}, + content, + matches; + + plugin.setAttribute(node, 'debug', 'true'); + plugin.createAttributeFile(node, files); + content = files['attributes.lua']; + matches = content.match(boolString); + expect(matches).to.equal(null); + }); + + it('should not quote true boolean values', function() { + var files = {}, + content, + matches; + + plugin.setAttribute(node, 'debug', true); + plugin.createAttributeFile(node, files); + content = files['attributes.lua']; + matches = content.match(boolString); + expect(matches).to.equal(null); + }); + + it('should not quote false (s) boolean values', function() { + var files = {}, + content, + matches; + + plugin.setAttribute(node, 'debug', 'false'); + plugin.createAttributeFile(node, files); + content = files['attributes.lua']; + matches = content.match(boolString); + expect(matches).to.equal(null); + }); + + it('should not quote false boolean values', function() { + var files = {}, + content, + matches; + + plugin.setAttribute(node, 'debug', false); + plugin.createAttributeFile(node, files); + content = files['attributes.lua']; + matches = content.match(boolString); + expect(matches).to.equal(null); + }); + }); + }); + +}); diff --git a/webgme-setup.json b/webgme-setup.json index 8365211..5ca9a10 100644 --- a/webgme-setup.json +++ b/webgme-setup.json @@ -56,6 +56,10 @@ "ValidateArchitecture": { "src": "src/plugins/ValidateArchitecture", "test": "test/plugins/ValidateArchitecture" + }, + "GenerateJob": { + "src": "src/plugins/GenerateJob", + "test": "test/plugins/GenerateJob" } }, "layouts": {