mi

mapred.io

mapreduce based on socket.io

Showing:

Popularity

Downloads/wk

0

GitHub Stars

3

Maintenance

Last Commit

8yrs ago

Contributors

3

Package

Dependencies

1

License

Type Definitions

Tree-Shakeable

No?

Categories

Readme

mapred.io

mapreduce based on socket.io
You can use this module, use the browser to easily build mapreduce programming model based on parallel computing platform. On this platform, you can submit the page using javascript mapreduce tasks.

##install

npm install mapred.io

Server

var app = require('http').createServer(handler)
  , fs = require('fs')
  , io = require('mapred.io').listen(app)
  , parse = require('url').parse;
  
  app.listen(80);
  io.set('log level', 2);

function handler (req, res) {
  var info = parse(req.url, true);

  var hander = function (err, data) {
    if (err) {
      res.writeHead(500);
      return res.end('Error loading file');
    }

    res.writeHead(200);
    res.end(data);
  }
  if (info.pathname.match(/\/public/))
    fs.readFile(__dirname + info.pathname, hander )
  else if (info.pathname.match(/\/client/))
    fs.readFile(__dirname + '/client.html', hander )
  else fs.readFile(__dirname + '/job.html', hander )
}



Client

<script type="text/javascript" src="/socket.io/socket.io.js"></script>
<script type="text/javascript" src="/socket.io/mapred.io.js"></script>
<script type="text/javascript" >
var socket = io.connect();
var MapredClient = new MapredClient(socket);
</script>

Submit Job

<script type="text/javascript" src="/socket.io/socket.io.js"></script>
<script type="text/javascript" src="/socket.io/mapred.io.js"></script>
<script type="text/javascript" >

var input = [
    ['frase primera', 'primer trozo de informacion para procesado primer trozo'],
    ['segunda frase', 'segundo trozo de informacion trozo de'],
    ['cacho 3', 'otro trozo para ser procesado otro otro otro trozo'],
    ['cuarta frase', 'primer trozo de informacion para procesado primer trozo'],
    ['frase 5', 'segundo trozo de informacion trozo de']
], inputs = input, i;
//for ( i = 0 ; i &lt; 20000 ; i ++ ) inputs = inputs.concat(input);

socket.emit('job', { map: (function(key, value){
    var list = [], aux = {};
        value = value.split(' ');
        value.forEach(function(w){
            aux[w] = (aux[w] || 0) + 1;
        });
        for(var k in aux){
            list.push([k, aux[k]]);
        }
        return list;
    }).toString(), reduce: (function(key, values){
        var sum = 0;
        values.forEach(function(e){
            sum += e;
        });
        return sum;
    }).toString(), inputs:inputs)

change log

2013-11-29 update mapred.io-client.js can be using in browser and command line, add client.js can run as a node.

To do

  1. Performance Test
  2. Client program (the standalone client not on the browser).
  3. Stored the datas (the server do not have enough memory to handle large data, Need a fast persistent storage framework. Just using the store of socket.io default is "MemoryStore", you can using "RedisStore")

screenshot screenshot update the screeshot

Rate & Review

Great Documentation0
Easy to Use0
Performant0
Highly Customizable0
Bleeding Edge0
Responsive Maintainers0
Poor Documentation0
Hard to Use0
Slow0
Buggy0
Abandoned0
Unwelcoming Community0
100
No reviews found
Be the first to rate

Alternatives

No alternatives found

Tutorials

No tutorials found
Add a tutorial