wellton/deepforge
1
0
Fork 0
Código Issues Pull Requests Ações Pacotes Projetos Versões Wiki Atividade

Refactored code gen out of ExecuteJob. Fixes #1014 (#1018)

Ver Código-Fonte 
* WIP created GenerateJob plugin for generating exec code

* WIP moved templates to GenerateJob

* WIP #1014 GenerateJob tests passing

* WIP #1014 Added attribute file test

* WIP #1014 Added more GenerateJob tests

* WIP #1014 removed extra logs

* WIP #1014 Follow base for plugin, namespace

* WIP #1014 Added CodeGen component settings

* WIP #1014 Removed component settings

* WIP #1014 Fixed the loading of inputs

* WIP #1014 Fixed namespace detection

* WIP #1014 Fixed code gen of job refs

* WIP #1014 Fixed 2nd degree code gen

* WIP #1014 Removed old code generation file

* WIP #1014 Fixed code linting issues

* WIP #1014 Fixed blob fetch error handling

* WIP #1014 fixed error handling for bad blob fetch

* WIP #1014 Updated the commit hash for the ptr code gen

* WIP #1014 save after forwarding data from the successful job

* WIP #1014 updated error msg on blob retrieval fail

* WIP #1014 fixed code linting issues
Esse commit está contido em:
Brian Broll
2017-04-26 21:46:17 -05:00
commit de GitHub
pai ff541be7a0
commit 73f7ba38ba
16 arquivos alterados com 562 adições e 262 exclusões
Baixar Arquivo de Patch Baixar Arquivo de Diff Expandir todos os arquivos Colapsar todos os arquivos
src/common/plugin/Operation.js
+35
Ver Arquivo
@@ -0,0 +1,35 @@
/*globals define */
// This is a mixin containing helpers for working with operation nodes
define([],function() {
var OperationOps = function() {
};
OperationOps.prototype.getOutputs = function (node) {
return this.getOperationData(node, this.META.Outputs);
};
OperationOps.prototype.getInputs = function (node) {
return this.getOperationData(node, this.META.Inputs);
};
OperationOps.prototype.getOperationData = function (node, metaType) {
// Load the children and the output's children
return this.core.loadChildren(node)
.then(containers => {
var outputs = containers.find(c => this.core.isTypeOf(c, metaType));
return outputs ? this.core.loadChildren(outputs) : [];
})
.then(outputs => {
var bases = outputs.map(node => this.core.getMetaType(node));
// return [[arg1, Type1, node1], [arg2, Type2, node2]]
return outputs.map((node, i) => [
this.getAttribute(node, 'name'),
this.getAttribute(bases[i], 'name'),
node
]);
});
};
return OperationOps;
});
src/common/plugin/PtrCodeGen.js
+51 -13
Ver Arquivo
@@ -6,27 +6,64 @@ define([
PluginUtils,
Q
) {
var CodeGen = {
Operation: {
pluginId: 'GenerateJob',
namespace: 'pipeline'
}
};
var PtrCodeGen = function() {
};
PtrCodeGen.prototype.getCodeGenPluginIdFor = function(node) {
var base = this.core.getBase(node),
name = this.core.getAttribute(node, 'name'),
namespace = this.core.getNamespace(node),
pluginId;
//this.logger.debug(`loaded pointer target of ${ptrId}: ${ptrNode}`);
pluginId = (this.core.getOwnRegistry(node, 'validPlugins') || '').split(' ').shift();
//this.logger.info(`generating code for ${this.core.getAttribute(ptrNode, 'name')} using ${pluginId}`);
if (this.core.isMetaNode(node) && CodeGen[name]) {
pluginId = CodeGen[name].pluginId || CodeGen[name];
namespace = CodeGen[name].namespace;
}
if (pluginId) {
return {
namespace: namespace,
pluginId: pluginId
};
} else if (base) {
return this.getCodeGenPluginIdFor(base);
} else {
return null;
}
};
PtrCodeGen.prototype.getPtrCodeHash = function(ptrId) {
return this.core.loadByPath(this.rootNode, ptrId)
.then(ptrNode => {
// Look up the plugin to use
var metanode = this.core.getMetaType(ptrNode),
pluginId;
var genInfo = this.getCodeGenPluginIdFor(ptrNode);
this.logger.debug(`loaded pointer target of ${ptrId}: ${ptrNode}`);
pluginId = this.core.getRegistry(ptrNode, 'validPlugins').split(' ').shift();
this.logger.info(`generating code for ${this.core.getAttribute(ptrNode, 'name')} using ${pluginId}`);
if (genInfo.pluginId) {
var context = {
namespace: genInfo.namespace,
activeNode: this.core.getPath(ptrNode)
};
var context = {
namespace: this.core.getNamespace(metanode),
activeNode: this.core.getPath(ptrNode)
};
// Load and run the plugin
return this.executePlugin(pluginId, context);
// Load and run the plugin
return this.executePlugin(genInfo.pluginId, context);
} else {
var metanode = this.core.getMetaType(ptrNode),
type = this.core.getAttribute(metanode, 'name');
this.logger.warn(`Could not find plugin for ${type}. Will try to proceed anyway`);
return null;
}
})
.then(hashes => hashes[0]); // Grab the first asset for now
};
@@ -56,12 +93,13 @@ define([
return PluginUtils.loadNodesAtCommitHash(
this.project,
this.core,
this.commitHash,
this.currentHash,
this.logger,
opts
).then(config => {
plugin.initialize(logger, this.blobClient, this.gmeConfig);
config.core = this.core;
config.project = this.project;
plugin.configure(config);
return plugin;
});
src/plugins/ExecuteJob/ExecuteJob.js
+24 -160
Ver Arquivo
@@ -8,10 +8,10 @@ define([
'plugin/PluginBase',
'deepforge/plugin/LocalExecutor',
'deepforge/plugin/PtrCodeGen',
'deepforge/plugin/Operation',
'deepforge/api/JobLogsClient',
'deepforge/api/JobOriginClient',
'deepforge/api/ExecPulseClient',
'./ExecuteJob.Files',
'./ExecuteJob.Metadata',
'./ExecuteJob.SafeSave',
'deepforge/Constants',
@@ -26,11 +26,11 @@ define([
PluginBase,
LocalExecutor, // DeepForge operation primitives
PtrCodeGen,
OperationPlugin,
JobLogsClient,
JobOriginClient,
ExecPulseClient,
ExecuteJobFiles,
ExecuteJobMetadata,
ExecuteJobSafeSave,
@@ -44,8 +44,7 @@ define([
pluginMetadata = JSON.parse(pluginMetadata);
var OUTPUT_INTERVAL = 1500,
STDOUT_FILE = 'job_stdout.txt';
var STDOUT_FILE = 'job_stdout.txt';
/**
* Initializes a new instance of ExecuteJob.
@@ -453,9 +452,10 @@ define([
children.find(child => this.isMetaTypeOf(child, this.META.Operation)));
};
ExecuteJob.prototype.onBlobRetrievalFail = function (node, input, err) {
// Handle the blob retrieval failed error
ExecuteJob.prototype.onBlobRetrievalFail = function (node, input) {
var job = this.core.getParent(node),
e = `Failed to retrieve "${input}" (${err})`,
e = `Failed to retrieve "${input}" (BLOB_FETCH_FAILED)`,
consoleErr = `Failed to execute operation: ${e}`;
consoleErr += [
@@ -473,14 +473,8 @@ define([
ExecuteJob.prototype.executeJob = function (job) {
return this.getOperation(job).then(node => {
var jobId = this.core.getPath(job),
name = this.getAttribute(node, 'name'),
localTypeId = this.getLocalOperationType(node),
artifact,
artifactName,
files,
data = {},
inputs;
var name = this.getAttribute(node, 'name'),
localTypeId = this.getLocalOperationType(node);
// Execute any special operation types here - not on an executor
this.logger.debug(`Executing operation "${name}"`);
@@ -488,126 +482,22 @@ define([
return this.executeLocalOperation(localTypeId, node);
} else {
// Generate all execution files
return this.createOperationFiles(node).then(results => {
this.logger.info('Created operation files!');
files = results;
artifactName = `${name}_${jobId.replace(/\//g, '_')}-execution-files`;
artifact = this.blobClient.createArtifact(artifactName);
// Add the input assets
// - get the metadata (name)
// - add the given inputs
inputs = Object.keys(files.inputAssets);
return Q.all(
inputs.map(input => { // Get the metadata for each input
var hash = files.inputAssets[input];
// data asset for "input"
return this.blobClient.getMetadata(hash)
.fail(err => this.onBlobRetrievalFail(job, input, err));
})
);
})
.then(mds => {
// Record the large files
var inputData = {},
runsh = '# Bash script to download data files and run job\n' +
'if [ -z "$DEEPFORGE_URL" ]; then\n echo "Please set DEEPFORGE_URL and' +
' re-run:"\n echo "" \n echo " DEEPFORGE_URL=http://my.' +
'deepforge.server.com:8080 bash run.sh"\n echo ""\n exit 1\nfi\n';
mds.forEach((metadata, i) => {
// add the hashes for each input
var input = inputs[i],
hash = files.inputAssets[input],
dataPath = 'inputs/' + input + '/data',
url = this.blobClient.getRelativeDownloadURL(hash);
inputData[dataPath] = {
req: hash,
cache: metadata.content
};
// Add to the run.sh file
runsh += `wget $DEEPFORGE_URL${url} -O ${dataPath}\n`;
return this.getPtrCodeHash(this.core.getPath(node))
.fail(err => {
this.logger.error(`Could not generate files: ${err}`);
if (err.message.indexOf('BLOB_FETCH_FAILED') > -1) {
this.onBlobRetrievalFail(node, err.message.split(':')[1]);
}
throw err;
})
.then(hash => {
this.logger.info(`Saved execution files`);
this.result.addArtifact(hash); // Probably only need this for debugging...
this.executeDistOperation(job, node, hash);
})
.fail(e => {
this.onOperationFail(node, `Distributed operation "${name}" failed ${e}`);
});
delete files.inputAssets;
files['input-data.json'] = JSON.stringify(inputData, null, 2);
runsh += 'th init.lua';
files['run.sh'] = runsh;
// Add pointer assets
Object.keys(files.ptrAssets)
.forEach(path => data[path] = files.ptrAssets[path]);
// Add the executor config
return this.getOutputs(node);
})
.then(outputArgs => {
var config,
outputs,
fileList,
ptrFiles = Object.keys(files.ptrAssets),
file;
delete files.ptrAssets;
fileList = Object.keys(files).concat(ptrFiles);
outputs = outputArgs.map(pair => pair[0])
.map(name => {
return {
name: name,
resultPatterns: [`outputs/${name}`]
};
});
outputs.push(
{
name: 'stdout',
resultPatterns: [STDOUT_FILE]
},
{
name: name + '-all-files',
resultPatterns: fileList
}
);
config = {
cmd: 'node',
args: ['start.js'],
outputInterval: OUTPUT_INTERVAL,
resultArtifacts: outputs
};
files['executor_config.json'] = JSON.stringify(config, null, 4);
// Save the artifact
// Remove empty hashes
for (file in data) {
if (!data[file]) {
this.logger.warn(`Empty data hash has been found for file "${file}". Removing it...`);
delete data[file];
}
}
return artifact.addObjectHashes(data);
})
.then(() => {
this.logger.info(`Added ptr/input data hashes for "${artifactName}"`);
return artifact.addFiles(files);
})
.then(() => {
this.logger.info(`Added execution files for "${artifactName}"`);
return artifact.save();
})
.then(hash => {
this.logger.info(`Saved execution files "${artifactName}"`);
this.result.addArtifact(hash); // Probably only need this for debugging...
this.executeDistOperation(job, node, hash);
})
.fail(e => {
this.onOperationFail(node, `Distributed operation "${name}" failed ${e}`);
});
}
});
};
@@ -881,32 +771,6 @@ define([
.fail(e => this.onOperationFail(node, `Operation ${nodeId} failed: ${e}`));
};
ExecuteJob.prototype.getOutputs = function (node) {
return this.getOperationData(node, this.META.Outputs);
};
ExecuteJob.prototype.getInputs = function (node) {
return this.getOperationData(node, this.META.Inputs);
};
ExecuteJob.prototype.getOperationData = function (node, metaType) {
// Load the children and the output's children
return this.core.loadChildren(node)
.then(containers => {
var outputs = containers.find(c => this.core.isTypeOf(c, metaType));
return outputs ? this.core.loadChildren(outputs) : [];
})
.then(outputs => {
var bases = outputs.map(node => this.core.getMetaType(node));
// return [[arg1, Type1, node1], [arg2, Type2, node2]]
return outputs.map((node, i) => [
this.getAttribute(node, 'name'),
this.getAttribute(bases[i], 'name'),
node
]);
});
};
//////////////////////////// Special Operations ////////////////////////////
ExecuteJob.prototype.executeLocalOperation = function (type, node) {
// Retrieve the given LOCAL_OP type
@@ -920,7 +784,7 @@ define([
_.extend(
ExecuteJob.prototype,
ExecuteJobFiles.prototype,
OperationPlugin.prototype,
ExecuteJobMetadata.prototype,
ExecuteJobSafeSave.prototype,
PtrCodeGen.prototype,
src/plugins/ExecutePipeline/ExecutePipeline.js
+5 -1
Ver Arquivo
@@ -175,6 +175,7 @@ define([
ExecutePipeline.prototype.resumePipeline = function () {
var nodes = Object.keys(this.nodes).map(id => this.nodes[id]),
allJobs = nodes.filter(node => this.core.isTypeOf(node, this.META.Job)),
name = this.getAttribute(this.activeNode, 'name'),
status,
jobs = {
success: [],
@@ -208,6 +209,7 @@ define([
return Q.all(allJobs.map(job => this.recordOldMetadata(job, true)))
.then(() => Q.all(jobs.success.map(job => this.getOperation(job))))
.then(ops => ops.forEach(op => this.updateJobCompletionRecords(op)))
.then(() => this.save(`Resuming pipeline execution: ${name}`))
.then(() => {
if (jobs.running.length) { // Resume all running jobs
@@ -531,10 +533,12 @@ define([
this.logger.info(`Setting ${jobId} status to "success"`);
this.logger.info(`There are now ${this.runningJobs} running jobs`);
this.logger.debug(`Making a commit from ${this.currentHash}`);
counts = this.updateJobCompletionRecords(opNode);
this.save(`Operation "${name}" in ${this.pipelineName} completed successfully`)
.then(() => {
counts = this.updateJobCompletionRecords(opNode);
hasReadyOps = counts.indexOf(0) > -1;
this.logger.debug(`Operation "${name}" completed. ` +
src/plugins/ExecuteJob/ExecuteJob.Files.js → src/plugins/GenerateJob/GenerateJob.js
+254 -32
Ver Arquivo
@@ -1,29 +1,212 @@
/*globals define*/
/*jshint node:true, browser:true*/
define([
'./templates/index',
'q',
'underscore',
'deepforge/Constants'
], function(
'deepforge/Constants',
'deepforge/plugin/Operation',
'deepforge/plugin/PtrCodeGen',
'text!./metadata.json',
'plugin/PluginBase'
], function (
Templates,
Q,
_,
CONSTANTS
CONSTANTS,
OperationHelpers,
PtrCodeGen,
pluginMetadata,
PluginBase
) {
'use strict';
var SKIP_ATTRIBUTES = [
'code',
'stdout',
'execFiles',
'jobId',
'secret',
CONSTANTS.LINE_OFFSET,
CONSTANTS.DISPLAY_COLOR
];
var ExecuteJob = function() {
pluginMetadata = JSON.parse(pluginMetadata);
var OUTPUT_INTERVAL = 1500,
STDOUT_FILE = 'job_stdout.txt',
SKIP_ATTRIBUTES = [
'code',
'stdout',
'execFiles',
'jobId',
'secret',
CONSTANTS.LINE_OFFSET,
CONSTANTS.DISPLAY_COLOR
];
/**
* Initializes a new instance of GenerateJob.
* @class
* @augments {PluginBase}
* @classdesc This class represents the plugin GenerateJob.
* @constructor
*/
var GenerateJob = function () {
// Call base class' constructor.
PluginBase.call(this);
this.pluginMetadata = pluginMetadata;
};
ExecuteJob.prototype.createOperationFiles = function (node) {
/**
* Metadata associated with the plugin. Contains id, name, version, description, icon, configStructue etc.
* This is also available at the instance at this.pluginMetadata.
* @type {object}
*/
GenerateJob.metadata = pluginMetadata;
// Prototypical inheritance from PluginBase.
GenerateJob.prototype = Object.create(PluginBase.prototype);
GenerateJob.prototype.constructor = GenerateJob;
/**
* Main function for the plugin to execute. This will perform the execution.
* Notes:
* - Always log with the provided logger.[error,warning,info,debug].
* - Do NOT put any user interaction logic UI, etc. inside this method.
* - callback always has to be called even if error happened.
*
* @param {function(string, plugin.PluginResult)} callback - the result callback
*/
GenerateJob.prototype.main = function (callback) {
var files,
artifactName,
artifact,
data = {},
inputs,
name,
opId;
name = this.getAttribute(this.activeNode, 'name');
opId = this.core.getPath(this.activeNode);
return this.createOperationFiles(this.activeNode)
.then(results => {
this.logger.info('Created operation files!');
files = results;
artifactName = `${name}_${opId.replace(/\//g, '_')}-execution-files`;
artifact = this.blobClient.createArtifact(artifactName);
// Add the input assets
// - get the metadata (name)
// - add the given inputs
inputs = Object.keys(files.inputAssets);
return Q.all(
inputs.map(input => { // Get the metadata for each input
var hash = files.inputAssets[input];
// data asset for "input"
return this.blobClient.getMetadata(hash)
.fail(() => {
throw Error(`BLOB_FETCH_FAILED:${input}`);
});
})
);
})
.then(mds => {
// Record the large files
var inputData = {},
runsh = '# Bash script to download data files and run job\n' +
'if [ -z "$DEEPFORGE_URL" ]; then\n echo "Please set DEEPFORGE_URL and' +
' re-run:"\n echo "" \n echo " DEEPFORGE_URL=http://my.' +
'deepforge.server.com:8080 bash run.sh"\n echo ""\n exit 1\nfi\n';
mds.forEach((metadata, i) => {
// add the hashes for each input
var input = inputs[i],
hash = files.inputAssets[input],
dataPath = 'inputs/' + input + '/data',
url = this.blobClient.getRelativeDownloadURL(hash);
inputData[dataPath] = {
req: hash,
cache: metadata.content
};
// Add to the run.sh file
runsh += `wget $DEEPFORGE_URL${url} -O ${dataPath}\n`;
});
delete files.inputAssets;
files['input-data.json'] = JSON.stringify(inputData, null, 2);
runsh += 'th init.lua';
files['run.sh'] = runsh;
// Add pointer assets
Object.keys(files.ptrAssets)
.forEach(path => data[path] = files.ptrAssets[path]);
// Add the executor config
return this.getOutputs(this.activeNode);
})
.then(outputArgs => {
var config,
outputs,
fileList,
ptrFiles = Object.keys(files.ptrAssets),
file;
delete files.ptrAssets;
fileList = Object.keys(files).concat(ptrFiles);
outputs = outputArgs.map(pair => pair[0])
.map(name => {
return {
name: name,
resultPatterns: [`outputs/${name}`]
};
});
outputs.push(
{
name: 'stdout',
resultPatterns: [STDOUT_FILE]
},
{
name: name + '-all-files',
resultPatterns: fileList
}
);
config = {
cmd: 'node',
args: ['start.js'],
outputInterval: OUTPUT_INTERVAL,
resultArtifacts: outputs
};
files['executor_config.json'] = JSON.stringify(config, null, 4);
// Save the artifact
// Remove empty hashes
for (file in data) {
if (!data[file]) {
this.logger.warn(`Empty data hash has been found for file "${file}". Removing it...`);
delete data[file];
}
}
return artifact.addObjectHashes(data);
})
.then(() => {
this.logger.info(`Added ptr/input data hashes for "${artifactName}"`);
return artifact.addFiles(files);
})
.then(() => {
this.logger.info(`Added execution files for "${artifactName}"`);
return artifact.save();
})
.then(hash => {
this.result.setSuccess(true);
this.result.addArtifact(hash);
callback(null, this.result);
})
.fail(err => {
this.result.setSuccess(false);
callback(err, this.result);
});
};
GenerateJob.prototype.createOperationFiles = function (node) {
var files = {};
// For each operation, generate the output files:
// inputs/<arg-name>/init.lua (respective data deserializer)
@@ -47,10 +230,14 @@ define([
.then(() => {
this.createAttributeFile(node, files);
return Q.ninvoke(this, 'createPointers', node, files);
})
.fail(err => {
this.logger.error(err);
throw err;
});
};
ExecuteJob.prototype.createEntryFile = function (node, files) {
GenerateJob.prototype.createEntryFile = function (node, files) {
this.logger.info('Creating entry files...');
return this.getOutputs(node)
.then(outputs => {
@@ -68,7 +255,7 @@ define([
});
};
ExecuteJob.prototype.createClasses = function (node, files) {
GenerateJob.prototype.createClasses = function (node, files) {
var metaDict = this.core.getAllMetaNodes(this.rootNode),
isClass,
metanodes,
@@ -120,7 +307,7 @@ define([
files['classes/init.lua'] = code;
};
ExecuteJob.prototype.getTypeDictFor = function (name, metanodes) {
GenerateJob.prototype.getTypeDictFor = function (name, metanodes) {
var isType = {};
// Get all the custom layers
for (var i = metanodes.length; i--;) {
@@ -131,7 +318,7 @@ define([
return isType;
};
ExecuteJob.prototype.createCustomLayers = function (node, files) {
GenerateJob.prototype.createCustomLayers = function (node, files) {
var metaDict = this.core.getAllMetaNodes(this.rootNode),
isCustomLayer,
metanodes,
@@ -153,7 +340,29 @@ define([
files['custom-layers.lua'] = code;
};
ExecuteJob.prototype.createInputs = function (node, files) {
GenerateJob.prototype.getConnectionContainer = function () {
var container = this.core.getParent(this.activeNode);
if (this.isMetaTypeOf(container, this.META.Job)) {
container = this.core.getParent(container);
}
return container;
};
GenerateJob.prototype.getInputPortsFor = function (nodeId) {
var container = this.getConnectionContainer();
// Get the connections to this node
return this.core.loadChildren(container)
.then(children => {
return children.filter(child =>
this.core.getPointerPath(child, 'dst') === nodeId)
.map(conn => this.core.getPointerPath(conn, 'src'))[0];
});
};
GenerateJob.prototype.createInputs = function (node, files) {
var tplContents,
inputs;
@@ -174,15 +383,13 @@ define([
return Q.all(inputs.map(pair => {
var name = pair[0],
node = pair[2],
nodeId = this.core.getPath(node),
fromNodeId;
nodeId = this.core.getPath(node);
// Get the deserialize function. First, try to get it from
// the source method (this guarantees that the correct
// deserialize method is used despite any auto-upcasting
fromNodeId = this.inputPortsFor[nodeId][0] || nodeId;
return this.core.loadByPath(this.rootNode, fromNodeId)
return this.getInputPortsFor(nodeId)
.then(fromNodeId => this.core.loadByPath(this.rootNode, fromNodeId || nodeId))
.then(fromNode => {
var deserFn,
base,
@@ -218,7 +425,9 @@ define([
return Q.all(hashes.map(pair =>
this.blobClient.getMetadata(pair.hash)
.fail(err => this.onBlobRetrievalFail(node, pair.name, err))));
.fail(() => {
throw Error(`BLOB_FETCH_FAILED:${pair.name}`);
})));
})
.then(metadatas => {
// Create the deserializer
@@ -231,7 +440,7 @@ define([
});
};
ExecuteJob.prototype.createOutputs = function (node, files) {
GenerateJob.prototype.createOutputs = function (node, files) {
// For each of the output types, grab their serialization functions and
// create the `outputs/init.lua` file
this.logger.info('Creating outputs/init.lua...');
@@ -256,7 +465,7 @@ define([
});
};
ExecuteJob.prototype.createMainFile = function (node, files) {
GenerateJob.prototype.createMainFile = function (node, files) {
this.logger.info('Creating main file...');
return this.getInputs(node)
.then(inputs => {
@@ -286,14 +495,14 @@ define([
});
};
ExecuteJob.prototype.getLineOffset = function (main, snippet) {
GenerateJob.prototype.getLineOffset = function (main, snippet) {
var i = main.indexOf(snippet),
lines = main.substring(0, i).match(/\n/g);
return lines ? lines.length : 0;
};
ExecuteJob.prototype.createAttributeFile = function (node, files) {
GenerateJob.prototype.createAttributeFile = function (node, files) {
var numOrBool = /^(-?\d+\.?\d*((e|e-)\d+)?|(true|false))$/,
table;
@@ -313,7 +522,7 @@ define([
files['attributes.lua'] = `-- attributes of ${this.getAttribute(node, 'name')}\nreturn ${table}`;
};
ExecuteJob.prototype.createPointers = function (node, files, cb) {
GenerateJob.prototype.createPointers = function (node, files, cb) {
var pointers,
nIds;
@@ -341,6 +550,19 @@ define([
});
};
return ExecuteJob;
});
GenerateJob.prototype.getAttribute = function (node, attr) {
return this.core.getAttribute(node, attr);
};
GenerateJob.prototype.setAttribute = function (node, attr, value) {
return this.core.setAttribute(node, attr, value);
};
_.extend(
GenerateJob.prototype,
OperationHelpers.prototype,
PtrCodeGen.prototype
);
return GenerateJob;
});
src/plugins/GenerateJob/metadata.json
+14
Ver Arquivo
@@ -0,0 +1,14 @@
{
"id": "GenerateJob",
"name": "GenerateJob",
"version": "0.1.0",
"description": "",
"icon": {
"class": "glyphicon glyphicon-cog",
"src": ""
},
"disableServerSideExecution": false,
"disableBrowserSideExecution": false,
"writeAccessRequired": false,
"configStructure": []
}
src/plugins/ExecuteJob/templates/deepforge.ejs → src/plugins/GenerateJob/templates/deepforge.ejs
Ver Arquivo
src/plugins/ExecuteJob/templates/deserialize.ejs → src/plugins/GenerateJob/templates/deserialize.ejs
Ver Arquivo
src/plugins/ExecuteJob/templates/entry.ejs → src/plugins/GenerateJob/templates/entry.ejs
Ver Arquivo
src/plugins/ExecuteJob/templates/index.js → src/plugins/GenerateJob/templates/index.js
Ver Arquivo
src/plugins/ExecuteJob/templates/main.ejs → src/plugins/GenerateJob/templates/main.ejs
Ver Arquivo
src/plugins/ExecuteJob/templates/serialize.ejs → src/plugins/GenerateJob/templates/serialize.ejs
Ver Arquivo
src/plugins/ExecuteJob/templates/start.ejs → src/plugins/GenerateJob/templates/start.ejs
Ver Arquivo
test/plugins/ExecuteJob/ExecuteJob.spec.js
-56
Ver Arquivo
@@ -300,62 +300,6 @@ describe('ExecuteJob', function () {
});
});
describe('exec files', function() {
describe('attribute file', function() {
var boolString = /['"](true|false)['"]/g;
beforeEach(preparePlugin);
it('should not quote true (s) boolean values', function() {
var files = {},
content,
matches;
plugin.setAttribute(node, 'debug', 'true');
plugin.createAttributeFile(node, files);
content = files['attributes.lua'];
matches = content.match(boolString);
expect(matches).to.equal(null);
});
it('should not quote true boolean values', function() {
var files = {},
content,
matches;
plugin.setAttribute(node, 'debug', true);
plugin.createAttributeFile(node, files);
content = files['attributes.lua'];
matches = content.match(boolString);
expect(matches).to.equal(null);
});
it('should not quote false (s) boolean values', function() {
var files = {},
content,
matches;
plugin.setAttribute(node, 'debug', 'false');
plugin.createAttributeFile(node, files);
content = files['attributes.lua'];
matches = content.match(boolString);
expect(matches).to.equal(null);
});
it('should not quote false boolean values', function() {
var files = {},
content,
matches;
plugin.setAttribute(node, 'debug', false);
plugin.createAttributeFile(node, files);
content = files['attributes.lua'];
matches = content.match(boolString);
expect(matches).to.equal(null);
});
});
});
describe('resume detection', function() {
var mockPluginForJobStatus = function(gmeStatus, pulse, originBranch, shouldResume, done) {
plugin.setAttribute(node, 'status', gmeStatus);
test/plugins/GenerateJob/GenerateJob.spec.js
+175
Ver Arquivo
@@ -0,0 +1,175 @@
/*jshint node:true, mocha:true*/
/**
* Generated by PluginGenerator 1.7.0 from webgme on Mon Apr 17 2017 07:34:11 GMT-0500 (CDT).
*/
'use strict';
var testFixture = require('../../globals');
describe('GenerateJob', function () {
var gmeConfig = testFixture.getGmeConfig(),
expect = testFixture.expect,
logger = testFixture.logger.fork('GenerateJob'),
PluginCliManager = testFixture.WebGME.PluginCliManager,
manager = new PluginCliManager(null, logger, gmeConfig),
projectName = 'testProject',
pluginName = 'GenerateJob',
project,
gmeAuth,
storage,
commitHash;
before(function (done) {
testFixture.clearDBAndGetGMEAuth(gmeConfig, projectName)
.then(function (gmeAuth_) {
gmeAuth = gmeAuth_;
// This uses in memory storage. Use testFixture.getMongoStorage to persist test to database.
storage = testFixture.getMemoryStorage(logger, gmeConfig, gmeAuth);
return storage.openDatabase();
})
.then(function () {
var importParam = {
projectSeed: testFixture.path.join(testFixture.DF_SEED_DIR, 'devProject', 'devProject.webgmex'),
projectName: projectName,
branchName: 'master',
logger: logger,
gmeConfig: gmeConfig
};
return testFixture.importProject(storage, importParam);
})
.then(function (importResult) {
project = importResult.project;
commitHash = importResult.commitHash;
return project.createBranch('test', commitHash);
})
.nodeify(done);
});
after(function (done) {
storage.closeDatabase()
.then(function () {
return gmeAuth.unload();
})
.nodeify(done);
});
describe('basic checks', function() {
var pluginResult,
error;
before(function(done) {
var pluginConfig = {
},
context = {
project: project,
commitHash: commitHash,
branchName: 'test',
activeNode: '/1',
};
manager.executePlugin(pluginName, pluginConfig, context, (err, result) => {
error = err;
pluginResult = result;
done();
});
});
it('should run without error', function () {
expect(error).to.equal(null);
expect(typeof pluginResult).to.equal('object');
expect(pluginResult.success).to.equal(true);
});
it('should generate artifacts', function () {
expect(pluginResult.artifacts[0]).to.not.equal(undefined);
});
it('should NOT update the branch', function (done) {
project.getBranchHash('test')
.then(function (branchHash) {
expect(branchHash).to.equal(commitHash);
})
.nodeify(done);
});
});
////////// Helper Functions //////////
var plugin,
node,
preparePlugin = function(done) {
var context = {
project: project,
commitHash: commitHash,
namespace: 'pipeline',
branchName: 'test',
activeNode: '/K/R/p/m' // hello world operation
};
return manager.initializePlugin(pluginName)
.then(plugin_ => {
plugin = plugin_;
return manager.configurePlugin(plugin, {}, context);
})
.then(() => node = plugin.activeNode)
.nodeify(done);
};
describe('exec files', function() {
describe('attribute file', function() {
var boolString = /['"](true|false)['"]/g;
beforeEach(preparePlugin);
it('should not quote true (s) boolean values', function() {
var files = {},
content,
matches;
plugin.setAttribute(node, 'debug', 'true');
plugin.createAttributeFile(node, files);
content = files['attributes.lua'];
matches = content.match(boolString);
expect(matches).to.equal(null);
});
it('should not quote true boolean values', function() {
var files = {},
content,
matches;
plugin.setAttribute(node, 'debug', true);
plugin.createAttributeFile(node, files);
content = files['attributes.lua'];
matches = content.match(boolString);
expect(matches).to.equal(null);
});
it('should not quote false (s) boolean values', function() {
var files = {},
content,
matches;
plugin.setAttribute(node, 'debug', 'false');
plugin.createAttributeFile(node, files);
content = files['attributes.lua'];
matches = content.match(boolString);
expect(matches).to.equal(null);
});
it('should not quote false boolean values', function() {
var files = {},
content,
matches;
plugin.setAttribute(node, 'debug', false);
plugin.createAttributeFile(node, files);
content = files['attributes.lua'];
matches = content.match(boolString);
expect(matches).to.equal(null);
});
});
});
});
webgme-setup.json
+4
Ver Arquivo
@@ -56,6 +56,10 @@
"ValidateArchitecture": {
"src": "src/plugins/ValidateArchitecture",
"test": "test/plugins/ValidateArchitecture"
},
"GenerateJob": {
"src": "src/plugins/GenerateJob",
"test": "test/plugins/GenerateJob"
}
},
"layouts": {