added package

Esse commit está contido em:
David Henderson
2013-08-10 15:13:36 +01:00
commit 61412dad39
6 arquivos alterados com 706 adições e 0 exclusões
+18
Ver Arquivo
@@ -0,0 +1,18 @@
lib-cov
*.seed
*.log
*.csv
*.dat
*.out
*.pid
*.gz
pids
logs
results
node_modules
npm-debug.log
.DS_Store
test/test.js
+149
Ver Arquivo
@@ -0,0 +1,149 @@
# Node Nearest Neighbor
[Nearest neighbor](http://en.wikipedia.org/wiki/K-nearest_neighbor_algorithm) algorithm to find the most similar object within an array of objects.
## Installation
```bash
$ npm install nearest-neighbor
```
## Usage
```javascript
var nn = require('nearest-neighbor');
nn.findMostSimilar(query, items, fields, function(nearestNeighbor, probability) {
console.log(nearestNeighbor);
console.log(probability);
});
```
```items``` is an array of objects that acts as the haystack that should be searched for the most similar ```query``` object. The ```fields``` array includes all the keys of the object the similarity should be calculated of and a comparison method to perform the similarity check. The callback returns the most similar object from the ```items``` array and the probability of the match between 1 and 0.
### Example
```javascript
nn = require('nearest-neighbor');
var items = [
{ name: "Bill", age: 10, pc: "Mac", ip: "68.23.13.8" },
{ name: "Alice", age: 22, pc: "Windows", ip: "193.186.11.3" },
{ name: "Bob", age: 12, pc: "Windows", ip: "56.89.22.1" }
];
var query = { name: "Bob", age: 12, pc: "Windows", ip: "68.23.13.10" };
var fields = [
{ name: "name", measure: nn.comparisonMethods.word },
{ name: "age", measure: nn.comparisonMethods.number, max: 100 },
{ name: "pc", measure: nn.comparisonMethods.word },
{ name: "ip", measure: nn.comparisonMethods.ip }
];
nn.findMostSimilar(query, items, fields, function(nearestNeighbor, probability) {
console.log(query);
console.log(nearestNeighbor);
console.log(probability);
});
```
#### Output
The most similar object from the ```items``` array compared to the ```query``` object and the probability ([0..1]) of the match found.
```javascript
{ name: 'Bob', age: 12, pc: 'Windows', ip: '68.23.13.10' }
{ name: 'Bob', age: 12, pc: 'Windows', ip: '56.89.22.1' }
0.9764705882352941
```
### Comparison Methods
There are some basic comparison methods shipped with this module. Every comparison method has two parameters and returns a number between 1 and 0.
a is the current item from the ```items``` array
b is always the ```query``` object
```javascript
// example comparison function
function(a, b) {
return 0.5;
};
```
The predefined comparison functions are available through the ```nn.comparisonMethods``` object. E.g. ```nn.comparisonMethods.exact```.
* **exact**: Checks if a and b have the exact same value (using ```===```).
* **word**: Calculates the similarity of two words.
* **wordArray**: Calculates the similarity of an array of words.
* **ip**: Calculates the similarity of two IP addresses.
* **ipArray**: Calculates the similarity of an array of IP addresses.
* **number**: Calculates the similarity two numbers.
### Add your custom comparison method
You can also roll your own comparison method if you like.
Just add your custom method to the ```nn.comparisonMethods``` object.
```javascript
// define items and query as shown before
nn.comparisonMethods.custom = function(a, b) {
// compare something...
var value = ...
return value; // between 0 and 1
};
// then use your custom method for one of the comparison fields
var fields = [
{ name: "name", measure: nn.comparisonMethods.custom },
{ name: "age", measure: nn.comparisonMethods.number, max: 100 },
{ name: "pc", measure: nn.comparisonMethods.word },
{ name: "ip", measure: nn.comparisonMethods.ip }
];
// use nn.findMostSimilar as shown before
```
## Tests
```bash
$ npm test
{ name: 'Bob', age: 12, pc: 'Windows', ip: '68.23.13.10' }
{ name: 'Bob', age: 12, pc: 'Windows', ip: '56.89.22.1' }
0.9764705882352941
All tests OK
```
## Contributors
Thanks to [Blago](http://stackoverflow.com/users/113999/blago) on StackOverflow for providing the initial draft/idea for the algorithm.
## Contributing
* Create something awesome, make the code better, add some functionality,
whatever (this is the hardest part).
* [Fork it](http://help.github.com/forking/)
* Create new branch to make your changes
* Commit all your changes to your branch
* Submit a [pull request](http://help.github.com/pull-requests/)
## Contact
Feel free to get in touch.
* Website: <http://schuch.me>
* Twitter: [@schuchalexander](http://twitter.com/schuchalexander)
## Licence
Copyright (C) 2013 Alexander Schuch
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+185
Ver Arquivo
@@ -0,0 +1,185 @@
// Generated by CoffeeScript 1.6.2
(function() {
var calculateSum, exactSimilarity, intersect, ipArraySimilarity, ipSimilarity, numberSimilarity, recordSimilarity, tokenize, wordArraySimilarity, wordSimilarity;
recordSimilarity = function(a, b, fields) {
var i, measure, name, similarity, sum, unmatchedFields;
sum = 0;
i = 0;
unmatchedFields = {};
while (i < fields.length) {
name = fields[i].name;
measure = fields[i].measure;
if ('' + measure === '' + exports.comparisonMethods.number) {
if (typeof fields[i].max !== 'undefined' && fields[i].max !== null) {
similarity = measure(a[name], b[name], fields[i].max);
} else {
console.warn("max number missing, falling back to max: 9007199254740992");
similarity = measure(a[name], b[name], 9007199254740992);
}
} else {
similarity = measure(a[name], b[name]);
}
if (similarity < 1.0) {
unmatchedFields[name] = similarity;
}
sum += similarity;
i++;
}
return [sum / fields.length, unmatchedFields];
};
exactSimilarity = function(a, b) {
var _ref;
return (_ref = a === b) != null ? _ref : {
1: 0
};
};
wordSimilarity = function(a, b) {
var left, middle, right;
left = tokenize(a);
right = tokenize(b);
middle = intersect(left, right);
return (2 * middle.length) / (left.length + right.length);
};
wordArraySimilarity = function(a, b) {
var i, similarity;
if (a.length === b.length) {
i = 0;
similarity = 0;
while (i < a.length) {
similarity += wordSimilarity(a[i], b[i]);
i++;
}
return similarity / a.length;
}
return 0;
};
ipSimilarity = function(a, b) {
var diff, diff1, diff2, diffs, distance, i, left, right;
left = a.split(".");
right = b.split(".");
diffs = [];
i = 0;
while (i < 4) {
diff1 = 255 - left[i];
diff2 = 255 - right[i];
diff = Math.abs(diff2 - diff1);
diffs[i] = diff;
i++;
}
distance = calculateSum(diffs) / (255 * 4);
return 1 - distance;
};
ipArraySimilarity = function(a, b) {
var i, num, similarity;
if (a.length === b.length) {
i = 0;
similarity = 0;
num = 0;
while (i < a.length) {
if (a[i].match(/\b(?:\d{1,3}\.){3}\d{1,3}\b/) !== null && b[i].match(/\b(?:\d{1,3}\.){3}\d{1,3}\b/) !== null) {
similarity += ipSimilarity(a[i], b[i]);
num++;
}
i++;
}
return similarity / num;
}
return 0;
};
numberSimilarity = function(a, b, max) {
var diff, diff1, diff2, distance;
diff1 = max - a;
diff2 = max - b;
diff = Math.abs(diff2 - diff1);
distance = diff / max;
return 1 - distance;
};
tokenize = function(string) {
var i, tokens;
tokens = [];
if (typeof string !== 'undefined' && string !== null) {
i = 0;
while (i < string.length - 1) {
tokens.push(string.substr(i, 2).toLowerCase());
i++;
}
}
return tokens.sort();
};
intersect = function(a, b) {
var ai, bi, result;
ai = 0;
bi = 0;
result = new Array();
while (ai < a.length && bi < b.length) {
if (a[ai] < b[bi]) {
ai++;
} else if (a[ai] > b[bi]) {
bi++;
} else {
result.push(a[ai]);
ai++;
bi++;
}
}
return result;
};
calculateSum = function(items) {
var i, sum;
sum = 0;
i = 0;
while (i < items.length) {
sum += items[i];
i++;
}
return sum;
};
exports.comparisonMethods = {
exact: exactSimilarity,
word: wordSimilarity,
wordArray: wordArraySimilarity,
ip: ipSimilarity,
ipArray: ipArraySimilarity,
number: numberSimilarity
};
exports.findMostSimilar = function(query, items, fields, callback) {
var i, item, maxSimilarity, result, similarity, unmatchedFields, _ref;
maxSimilarity = 0;
result = null;
i = 0;
while (i < items.length) {
item = items[i];
_ref = recordSimilarity(item, query, fields), similarity = _ref[0], unmatchedFields = _ref[1];
if (similarity > maxSimilarity) {
maxSimilarity = similarity;
result = item;
}
i++;
}
return callback(result, maxSimilarity, unmatchedFields);
};
}).call(this);
Diff do arquivo suprimido porque uma ou mais linhas são muito longas
+229
Ver Arquivo
@@ -0,0 +1,229 @@
#
# Nearest Neighbour algorithm
#
# ///////////////////////////////////////////
# Similarity comparison
# ///////////////////////////////////////////
#
# Calculates similarity of two records (a,b)
# based on comparison fields
#
recordSimilarity = (a, b, fields) ->
sum = 0
i = 0
unmatchedFields = {} # all fields whose similarity is not exactly 1.0
while i < fields.length
name = fields[i].name
measure = fields[i].measure
if ''+measure == ''+exports.comparisonMethods.number
if typeof(fields[i].max) != 'undefined' && fields[i].max != null
similarity = measure(a[name], b[name], fields[i].max)
else
console.warn "max number missing, falling back to max: 9007199254740992"
similarity = measure(a[name], b[name], 9007199254740992) # see http://stackoverflow.com/a/307200/1000569
else
similarity = measure(a[name], b[name])
if similarity < 1.0
unmatchedFields[name] = similarity
#console.log name + " | " + a[name] + " " + b[name] + " | " + similarity
sum += similarity
i++
[sum/fields.length, unmatchedFields]
# ///////////////////////////////////////////
# Similarity functions
# ///////////////////////////////////////////
#
# Calculates the exact similarity of two words
# returns a percentage of similarity 0||1
#
exactSimilarity = (a, b) ->
a == b ? 1 : 0
# //////////////////////////////////////
#
# Calculates the similarity of two words
# returns a percentage of similarity 0..1
#
wordSimilarity = (a, b) ->
left = tokenize(a)
right = tokenize(b)
middle = intersect(left, right)
(2 * middle.length) / (left.length + right.length)
# //////////////////////////////////////
#
# Calculates the similarity of an array of words
# returns a percentage of similarity 0..1
#
wordArraySimilarity = (a, b) ->
if a.length == b.length
i = 0
similarity = 0
while i < a.length
similarity += wordSimilarity(a[i], b[i])
i++
return similarity/a.length
0
# //////////////////////////////////////
#
# Calculates the similarity of two IP addresses
# returns a percentage of similarity 0..1
#
ipSimilarity = (a, b) ->
left = a.split(".")
right = b.split(".")
diffs = []
i = 0
while i < 4
diff1 = 255 - left[i]
diff2 = 255 - right[i]
diff = Math.abs(diff2 - diff1)
diffs[i] = diff
i++
distance = calculateSum(diffs) / (255 * 4)
1 - distance
# //////////////////////////////////////
#
# Calculates the similarity of an array of IP addresses
# returns a percentage of similarity 0..1
#
ipArraySimilarity = (a, b) ->
if a.length == b.length
i = 0
similarity = 0
num = 0
while i < a.length
if a[i].match(/\b(?:\d{1,3}\.){3}\d{1,3}\b/) != null && b[i].match(/\b(?:\d{1,3}\.){3}\d{1,3}\b/) != null
similarity += ipSimilarity(a[i], b[i])
num++
i++
return similarity/num
0
# //////////////////////////////////////
#
# Calculates the similarity of two numbers
# returns a percentage of similarity 0..1
#
numberSimilarity = (a, b, max) ->
diff1 = max - a
diff2 = max - b
diff = Math.abs(diff2 - diff1)
distance = diff / max
1 - distance
# ///////////////////////////////////////////
# Helper methods
# ///////////////////////////////////////////
#
# Returns tokens in pairs of two
# from the given string
#
tokenize = (string) ->
tokens = []
if typeof(string) != 'undefined' && string != null
i = 0
while i < string.length - 1
tokens.push string.substr(i, 2).toLowerCase()
i++
tokens.sort()
# //////////////////////////////////////
#
# Calculates the intersection of two
# given arrays of strings
#
intersect = (a, b) ->
ai = 0
bi = 0
result = new Array()
while ai < a.length and bi < b.length
if a[ai] < b[bi]
ai++
else if a[ai] > b[bi]
bi++
else
# they're equal
result.push a[ai]
ai++
bi++
result
# //////////////////////////////////////
#
# Calculates the sum of an array of values
#
calculateSum = (items) ->
sum = 0
i = 0
while i < items.length
sum += items[i]
i++
sum
# ///////////////////////////////////////////
# Public methods
# ///////////////////////////////////////////
#
#
#
exports.comparisonMethods =
exact: exactSimilarity
word: wordSimilarity
wordArray: wordArraySimilarity
ip: ipSimilarity
ipArray: ipArraySimilarity
number: numberSimilarity
#
# Calculates most similar record given a query object
# and multiple items that should be checked for similarity
# to the query object
#
exports.findMostSimilar = (query, items, fields, callback) ->
maxSimilarity = 0
result = null
i = 0
while i < items.length
item = items[i]
[similarity, unmatchedFields] = recordSimilarity(item, query, fields)
if similarity > maxSimilarity
maxSimilarity = similarity
result = item
i++
callback result, maxSimilarity, unmatchedFields
+85
Ver Arquivo
@@ -0,0 +1,85 @@
nn = require('../src/nearest-neighbor')
should = require('should')
# ///////////////////////////////
items = [
name: "Bill"
age: 10
pc: "Mac"
ip: "68.23.13.8"
,
name: "Alice"
age: 22
pc: "Windows"
ip: "193.186.11.3"
,
name: "Bob"
age: 12
pc: "Windows"
ip: "56.89.22.1"
]
query =
name: "Bob"
age: 12
pc: "Windows"
ip: "68.23.13.10"
nn.comparisonMethods.custom = (a, b) ->
return 0.5
fields = [
name: "name"
measure: nn.comparisonMethods.word
,
name: "age"
measure: nn.comparisonMethods.number
max: 100
,
name: "pc"
measure: nn.comparisonMethods.word
,
name: "ip"
measure: nn.comparisonMethods.ip
]
nn.findMostSimilar query, items, fields, (nearestNeighbor, probability, unmatchedFields) ->
console.log query
console.log nearestNeighbor
console.log probability
console.log unmatchedFields
console.log "====================="
query =
name: "Bob"
age: 12
pc: "Windows"
ip: "56.89.22.1"
nn.findMostSimilar query, items, fields, (nearestNeighbor, probability, unmatchedFields) ->
console.log query
console.log nearestNeighbor
console.log probability
console.log unmatchedFields
console.log "====================="
query =
name: "Max"
age: 14
pc: "Windows XP"
ip: "56.89.22.1"
nn.findMostSimilar query, items, fields, (nearestNeighbor, probability, unmatchedFields) ->
console.log query
console.log nearestNeighbor
console.log probability
console.log unmatchedFields
console.log "====================="
# //////////////////////////////////////
console.log "\nAll tests OK"