streaming view rendering

Allows for a view engine to return a stream upon being asked to render a
path. This allows for the engine to simply stream out data as it is
loaded from disk (or other IO location) without having to collect it all
first and then send it.

Fallback mode for all existing engines that call `fn` upon completion
takes over if no stream is returned. This works for both app.render and
res.render

History.md

@@ -1,44 +1,3 @@
-4.0.0 /
-==================
-
- * remove:
-   - express(1) - moved to [express-generator](https://github.com/expressjs/generator)
-   - `req.accepted*` - use `req.accepts*()` instead
-   - `app.configure` - use logic in your own app code
- * change:
-   - `req.accepts*` -> `req.accepts*s` - i.e. `req.acceptsEncoding` -> `req.acceptsEncodings`
-   - `req.params` is now an object instead of an array
-   - `json spaces` no longer enabled by default in development
- * refactor:
-   - `req.accepts*` with [accepts](https://github.com/expressjs/accepts)
-   - `req.is` with [type-is](https://github.com/expressjs/type-is)
-
-3.4.8 / 2014-01-13
-==================
-
- * prevent incorrect automatic OPTIONS responses #1868 @dpatti
- * update binary and examples for jade 1.0 #1876 @yossi, #1877 @reqshark, #1892 @matheusazzi
- * throw 400 in case of malformed paths @rlidwka
-
-3.4.7 / 2013-12-10
-==================
-
- * update connect
-
-3.4.6 / 2013-12-01
-==================
-
- * update connect (raw-body)
-
-3.4.5 / 2013-11-27
-==================
-
- * update connect
- * res.location: remove leading ./ #1802 @kapouer
- * res.redirect: fix `res.redirect('toString') #1829 @michaelficarra
- * res.send: always send ETag when content-length > 0
- * router: add Router.all() method
-
 3.4.4 / 2013-10-29
 ==================
 

Makefile

@@ -24,11 +24,11 @@ test-cov: lib-cov
 lib-cov:
 	@jscoverage lib lib-cov
 
-bench:
-	@$(MAKE) -C benchmarks
+benchmark:
+	@./support/bench
 
 clean:
 	rm -f coverage.html
 	rm -fr lib-cov
 
-.PHONY: test test-unit test-acceptance bench clean
+.PHONY: test test-unit test-acceptance benchmark clean

Readme.md

@@ -50,14 +50,14 @@ app.listen(3000);
 
 ## Philosophy
 
-  The Express philosophy is to provide small, robust tooling for HTTP servers, making
+  The Express philosophy is to provide small, robust tooling for HTTP servers. Making
   it a great solution for single page applications, web sites, hybrids, or public
   HTTP APIs.
 
-  Built on Connect, you can use _only_ what you need, and nothing more. Applications
+  Built on Connect you can use _only_ what you need, and nothing more, applications
   can be as big or as small as you like, even a single file. Express does
   not force you to use any specific ORM or template engine. With support for over
-  14 template engines via [Consolidate.js](http://github.com/visionmedia/consolidate.js),
+  14 template engines via [Consolidate.js](http://github.com/visionmedia/consolidate.js)
   you can quickly craft your perfect framework.
 
 ## More Information
@@ -72,27 +72,27 @@ app.listen(3000);
 
 ## Viewing Examples
 
-Clone the Express repo, then install the dev dependencies to install all the example / test suite dependencies:
+Clone the Express repo, then install the dev dependencies to install all the example / test suite deps:
 
     $ git clone git://github.com/visionmedia/express.git --depth 1
     $ cd express
     $ npm install
 
-Then run whichever tests you want:
+then run whichever tests you want:
 
     $ node examples/content-negotiation
 
-You can also view live examples here:
+You can also view live examples here
 
 <a href="https://runnable.com/express" target="_blank"><img src="https://runnable.com/external/styles/assets/runnablebtn.png" style="width:67px;height:25px;"></a>
 
 ## Running Tests
 
-To run the test suite, first invoke the following command within the repo, installing the development dependencies:
+To run the test suite first invoke the following command within the repo, installing the development dependencies:
 
     $ npm install
 
-Then run the tests:
+then run the tests:
 
     $ make test
 

benchmarks/Makefile

@@ -1,13 +0,0 @@
-
-all:
-	@./run 1 middleware
-	@./run 5 middleware
-	@./run 10 middleware
-	@./run 15 middleware
-	@./run 20 middleware
-	@./run 30 middleware
-	@./run 50 middleware
-	@./run 100 middleware
-	@echo
-
-.PHONY: all

benchmarks/middleware.js

@@ -1,23 +0,0 @@
-
-var http = require('http');
-var express = require('..');
-var app = express();
-
-// number of middleware
-
-var n = parseInt(process.env.MW || '1', 10);
-console.log('  %s middleware', n);
-
-while (n--) {
-  app.use(function(req, res, next){
-    next();
-  });
-}
-
-var body = new Buffer('Hello World');
-
-app.use(function(req, res, next){
-  res.send(body);
-});
-
-app.listen(3333);

benchmarks/run

@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-
-echo
-MW=$1 node $2 &
-pid=$!
-
-sleep 2
-
-wrk 'http://localhost:3333/?foo[bar]=baz' \
-  -d 3 \
-  -c 50 \
-  -t 8 \
-  | grep 'Requests/sec' \
-  | awk '{ print "  " $2 }'
-
-kill $pid

bin/express

@@ -74,7 +74,7 @@ var users = [
  */
 
 var jadeLayout = [
-    'doctype html'
+    'doctype 5'
   , 'html'
   , '  head'
   , '    title= title'

examples/auth/pass.js

@@ -32,7 +32,7 @@ var iterations = 12000;
 exports.hash = function (pwd, salt, fn) {
   if (3 == arguments.length) {
     crypto.pbkdf2(pwd, salt, iterations, len, function(err, hash){
-      fn(err, hash.toString('base64'));
+      fn(err, (new Buffer(hash, 'binary')).toString('base64'));
     });
   } else {
     fn = salt;
@@ -41,8 +41,8 @@ exports.hash = function (pwd, salt, fn) {
       salt = salt.toString('base64');
       crypto.pbkdf2(pwd, salt, iterations, len, function(err, hash){
         if (err) return fn(err);
-        fn(null, salt, hash.toString('base64'));
+        fn(null, salt, (new Buffer(hash, 'binary')).toString('base64'));
       });
     });
   }
-};
+};

examples/content-negotiation/index.js

@@ -28,10 +28,10 @@ app.get('/', function(req, res){
 // this to add a layer of abstraction
 // and make things a bit more declarative:
 
-function format(path) {
-  var obj = require(path);
+function format(requestHandlerName) {
+  var requestHandler = require(requestHandlerName);
   return function(req, res){
-    res.format(obj);
+    res.format(requestHandler);
   }
 }
 

examples/jade/views/layout.jade

@@ -1,5 +1,5 @@
-doctype html
+!!! 5
 html
   include header
   body
-    block content
+    block content

examples/view-locals/layout.jade

@@ -1,4 +1,4 @@
-doctype html
+doctype 5
 html
   head
     title= title

lib/application.js

@@ -490,7 +490,7 @@ app.render = function(name, options, fn){
     });
 
     if (!view.path) {
-      var err = new Error('Failed to lookup view "' + name + '" in views directory "' + view.root + '"');
+      var err = new Error('Failed to lookup view "' + name + '"');
       err.view = view;
       return fn(err);
     }
@@ -501,7 +501,26 @@ app.render = function(name, options, fn){
 
   // render
   try {
-    view.render(opts, fn);
+    var view_stream = view.render(opts, fn);
+
+    // if the engine returned a stream AND user specified a function
+    // then we will capture data for user
+    if (view_stream && fn) {
+      var body = '';
+      view_stream.on('data', function(chunk) {
+        body += chunk;
+      });
+
+      view_stream.once('error', function(err) {
+        fn(err);
+      });
+
+      view_stream.once('end', function() {
+        fn(null, body);
+      });
+    }
+
+    return view_stream;
   } catch (err) {
     fn(err);
   }

lib/express.js

@@ -2,7 +2,6 @@
  * Module dependencies.
  */
 
-var merge = require('merge-descriptors');
 var connect = require('connect')
   , proto = require('./application')
   , Route = require('./router/route')
@@ -44,7 +43,12 @@ function createApplication() {
  * for example `express.logger` etc.
  */
 
-merge(exports, connect.middleware);
+for (var key in connect.middleware) {
+  Object.defineProperty(
+      exports
+    , key
+    , Object.getOwnPropertyDescriptor(connect.middleware, key));
+}
 
 /**
  * Error on createServer().

lib/response.js

@@ -605,7 +605,8 @@ res.cookie = function(name, val, options){
 /**
  * Set the location header to `url`.
  *
- * The given `url` can also be "back", which redirects
+ * The given `url` can also be the name of a mapped url, for
+ * example by default express supports "back" which redirects
  * to the _Referrer_ or _Referer_ headers or "/".
  *
  * Examples:
@@ -636,8 +637,11 @@ res.location = function(url){
     , req = this.req
     , path;
 
-  // "back" is an alias for the referrer
-  if ('back' == url) url = req.get('Referrer') || '/';
+  // setup redirect map
+  var map = { back: req.get('Referrer') || '/' };
+
+  // perform redirect
+  url = map[url] || url;
 
   // relative
   if (!~url.indexOf('://') && 0 != url.indexOf('//')) {
@@ -788,12 +792,49 @@ res.render = function(view, options, fn){
   // merge res.locals
   options._locals = self.locals;
 
-  // default callback to respond
-  fn = fn || function(err, str){
+  // support for non streaming view rendering
+  // if renderer uses streams, this will not be called
+  // see below for how streams are handled
+  var respond = fn || function(err, str){
+    if (view_stream) return;
+
     if (err) return req.next(err);
     self.send(str);
   };
 
-  // render
-  app.render(view, options, fn);
+  var view_stream = app.render(view, options, respond);
+
+  // old style, will have already called respond for us
+  if (!view_stream) {
+    return;
+  }
+
+  // user wants to handle sending themselves
+  if (fn) {
+    var body = '';
+    view_stream.on('data', function(chunk) {
+      body += chunk;
+    });
+
+    view_stream.once('error', function(err) {
+      fn(err);
+    });
+
+    view_stream.once('end', function() {
+      fn(null, body);
+    });
+
+    return;
+  }
+
+  // set response headers
+  self.statusCode = 200;
+  if (!self.get('Content-Type')) {
+    self.charset = self.charset || 'utf-8';
+    self.type('html');
+  }
+  self.setHeader('Transfer-Encoding', 'chunked');
+
+  // start streaming the response
+  view_stream.pipe(self);
 };

lib/router/index.js

@@ -104,7 +104,7 @@ Router.prototype._dispatch = function(req, res, next){
     req.route = route = self.matchRequest(req, i);
 
     // implied OPTIONS
-    if (!route && 'OPTIONS' == req.method) return self._options(req, res, next);
+    if (!route && 'OPTIONS' == req.method) return self._options(req, res);
 
     // no route
     if (!route) return next(err);
@@ -181,10 +181,9 @@ Router.prototype._dispatch = function(req, res, next){
  * @api private
  */
 
-Router.prototype._options = function(req, res, next){
+Router.prototype._options = function(req, res){
   var path = parse(req).pathname
     , body = this._optionsFor(path).join(',');
-  if (!body) return next();
   res.set('Allow', body).send(body);
 };
 
@@ -303,15 +302,6 @@ Router.prototype.route = function(method, path, callbacks){
   return this;
 };
 
-Router.prototype.all = function(path) {
-  var self = this;
-  var args = [].slice.call(arguments);
-  methods.forEach(function(method){
-      self.route.apply(self, [method].concat(args));
-  });
-  return this;
-};
-
 methods.forEach(function(method){
   Router.prototype[method] = function(path){
     var args = [method].concat([].slice.call(arguments));

lib/router/route.js

@@ -57,15 +57,9 @@ Route.prototype.match = function(path){
   for (var i = 1, len = m.length; i < len; ++i) {
     var key = keys[i - 1];
 
-    try {
-      var val = 'string' == typeof m[i]
-        ? decodeURIComponent(m[i])
-        : m[i];
-    } catch(e) {
-      var err = new Error("Failed to decode param '" + m[i] + "'");
-      err.status = 400;
-      throw err;
-    }
+    var val = 'string' == typeof m[i]
+      ? utils.decode(m[i])
+      : m[i];
 
     if (key) {
       params[key.name] = val;

lib/utils.js

@@ -312,3 +312,22 @@ exports.pathRegexp = function(path, keys, sensitive, strict) {
     .replace(/\*/g, '(.*)');
   return new RegExp('^' + path + '$', sensitive ? '' : 'i');
 }
+
+
+/**
+ * Decodes a URI component. Returns
+ * the original string if the component
+ * is malformed.
+ *
+ * @param  {String} str
+ * @return {String}
+ * @api private
+ */
+
+exports.decode = function(str) {
+  try {
+    return decodeURIComponent(str);
+  } catch (e) {
+    return str;
+  }
+}

lib/view.js

@@ -73,5 +73,5 @@ View.prototype.lookup = function(path){
  */
 
 View.prototype.render = function(options, fn){
-  this.engine(this.path, options, fn);
+  return this.engine(this.path, options, fn);
 };

package.json

@@ -1,7 +1,7 @@
 {
   "name": "express",
   "description": "Sinatra inspired web development framework",
-  "version": "3.4.8",
+  "version": "3.4.4",
   "author": "TJ Holowaychuk <tj@vision-media.ca>",
   "contributors": [
     {
@@ -22,29 +22,28 @@
     }
   ],
   "dependencies": {
-    "connect": "2.12.0",
+    "connect": "2.11.0",
     "commander": "1.3.2",
-    "range-parser": "1.0.0",
+    "range-parser": "0.0.4",
     "mkdirp": "0.3.5",
     "cookie": "0.1.0",
     "buffer-crc32": "0.2.1",
-    "fresh": "0.2.1",
+    "fresh": "0.2.0",
     "methods": "0.1.0",
-    "send": "0.2.0",
-    "cookie-signature": "1.0.3",
-    "merge-descriptors": "0.0.2",
+    "send": "0.1.4",
+    "cookie-signature": "1.0.1",
     "debug": ">= 0.7.3 < 1"
   },
   "devDependencies": {
-    "ejs": "~0.8.4",
-    "mocha": "~1.15.1",
-    "jade": "~0.30.0",
-    "hjs": "~0.0.6",
-    "stylus": "~0.40.0",
-    "should": "~2.1.1",
-    "connect-redis": "~1.4.5",
-    "marked": "0.2.10",
-    "supertest": "~0.8.1"
+    "ejs": ">= 0.8.4 < 1",
+    "mocha": ">= 1.13.0 < 2",
+    "jade": "0.30.0",
+    "hjs": ">= 0.0.6 < 1",
+    "stylus": ">= 0.39.1 < 1",
+    "should": ">= 2.0.2 < 3",
+    "connect-redis": ">= 1.4.5 < 2",
+    "marked": ">= 0.2.9 < 1",
+    "supertest": ">= 0.8.1 < 1"
   },
   "keywords": [
     "express",
@@ -68,6 +67,5 @@
   },
   "engines": {
     "node": ">= 0.8.0"
-  },
-  "license": "MIT"
+  }
 }

support/bench

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+NODE_ENV=production node ./support/app &
+pid=$!
+
+bench() {
+  ab -n 5000 -c 50 -k -q http://127.0.0.1:8000$1 \
+    | grep "Requests per" \
+    | cut -d ' ' -f 7 \
+    | xargs echo "$2:"
+}
+
+bench_conditional() {
+  ab -n 5000 -c 50 -H "If-None-Match: $3" -k -q http://127.0.0.1:8000$1 \
+    | grep "Requests per" \
+    | cut -d ' ' -f 7 \
+    | xargs echo "$2:"
+}
+
+sleep .5
+bench / "Hello World"
+bench /blog "Mounted Hello World"
+bench /blog/admin "Mounted 2 Hello World"
+bench /middleware "Middleware"
+bench /match "Router"
+bench /render "Render"
+bench /json "JSON tiny"
+bench /json/15 "JSON small"
+bench /json/50 "JSON medium"
+bench /json/150 "JSON large"
+
+kill -9 $pid

test/Router.js

@@ -2,7 +2,6 @@
 var express = require('../')
   , Router = express.Router
   , request = require('./support/http')
-  , methods = require('methods')
   , assert = require('assert');
 
 describe('Router', function(){
@@ -101,21 +100,4 @@ describe('Router', function(){
       router.route('get', '/foo', function(){}, function(){});
     })
   })
-
-  describe('.all', function() {
-    it('should support using .all to capture all http verbs', function() {
-      var router = new Router();
-
-      router.all('/foo', function(){});
-
-      var url = '/foo?bar=baz';
-
-      methods.forEach(function testMethod(method) {
-        var route = router.match(method, url);
-        route.constructor.name.should.equal('Route');
-        route.method.should.equal(method);
-        route.path.should.equal('/foo');
-      });
-    })
-  })
 })

test/acceptance/markdown.js

@@ -7,7 +7,7 @@ describe('markdown', function(){
     it('should respond with html', function(done){
       request(app)
         .get('/')
-        .expect(/<h1[^>]*>Markdown Example<\/h1>/,done)
+        .expect(/<h1>Markdown Example<\/h1>/,done)
     })
   })
 

test/app.engine.js

@@ -1,5 +1,6 @@
 
 var express = require('../')
+  , request = require('./support/http')
   , fs = require('fs');
 
 function render(path, options, fn) {
@@ -10,6 +11,10 @@ function render(path, options, fn) {
   });
 }
 
+function streaming_render(path, options, fn) {
+  return fs.createReadStream(path, { encoding: 'utf-8' });
+}
+
 describe('app', function(){
   describe('.engine(ext, fn)', function(){
     it('should map a template engine', function(done){
@@ -76,5 +81,42 @@ describe('app', function(){
         done();
       })
     })
+
+    it('should support streaming engines', function(done) {
+      var app = express();
+      app.set('views', __dirname + '/fixtures');
+      app.engine('.html', streaming_render);
+      app.set('view engine', '.html');
+      app.locals.user = { name: 'tobi' };
+
+      // using function should just call the function when done
+      app.render('user', function(err, str){
+        if (err) return done(err);
+        str.should.equal('<p>{{user.name}}</p>');
+        done();
+      })
+    });
+
+    it('should render a response using the streaming engine', function(done) {
+      var app = express();
+      app.set('views', __dirname + '/fixtures');
+      app.engine('.html', streaming_render);
+      app.set('view engine', '.html');
+      app.locals.user = { name: 'tobi' };
+
+      app.get('/', function(req, res) {
+        res.render('user');
+      });
+
+      request(app)
+      .get('/')
+      .expect(200)
+      .end(function(err, res) {
+        if (err) return done(err);
+        res.header['transfer-encoding'].should.equal('chunked');
+        res.text.should.equal('<p>{{user.name}}</p>');
+        done();
+      });
+    });
   })
 })

test/app.options.js

@@ -15,30 +15,6 @@ describe('OPTIONS', function(){
     .expect('GET,PUT')
     .expect('Allow', 'GET,PUT', done);
   })
-
-  it('should not respond if the path is not defined', function(done){
-    var app = express();
-
-    app.get('/users', function(req, res){});
-
-    request(app)
-    .options('/other')
-    .expect(404, done);
-  })
-
-  it('should forward requests down the middleware chain', function(done){
-    var app = express();
-    var router = new express.Router();
-
-    router.get('/users', function(req, res){});
-    app.use(router.middleware);
-    app.get('/other', function(req, res){});
-
-    request(app)
-    .options('/other')
-    .expect('GET')
-    .expect('Allow', 'GET', done);
-  })
 })
 
 describe('app.options()', function(){

test/app.render.js

@@ -59,7 +59,7 @@ describe('app', function(){
         var app = express();
         app.set('views', __dirname + '/fixtures');
         app.render('rawr.jade', function(err){
-          err.message.should.equal('Failed to lookup view "rawr.jade" in views directory "' + __dirname + '/fixtures"');
+          err.message.should.equal('Failed to lookup view "rawr.jade"');
           done();
         });
       })

test/app.router.js

@@ -27,54 +27,28 @@ describe('app.router', function(){
     });
   })
 
-  describe('decode querystring', function(){
-    it('should decode correct params', function(done){
-      var app = express();
+  it('should decode params', function(done){
+    var app = express();
 
-      app.get('/:name', function(req, res, next){
-        res.send(req.params.name);
-      });
+    app.get('/:name', function(req, res, next){
+      res.send(req.params.name);
+    });
 
-      request(app)
-      .get('/foo%2Fbar')
-      .expect('foo/bar', done);
-    })
+    request(app)
+    .get('/foo%2Fbar')
+    .expect('foo/bar', done);
+  })
 
-    it('should not accept params in malformed paths', function(done) {
-      var app = express();
+  it('should accept params in malformed paths', function(done) {
+    var app = express();
 
-      app.get('/:name', function(req, res, next){
-        res.send(req.params.name);
-      });
+    app.get('/:name', function(req, res, next){
+      res.send(req.params.name);
+    });
 
-      request(app)
-      .get('/%foobar')
-      .expect(400, done);
-    })
-
-    it('should not decode spaces', function(done) {
-      var app = express();
-
-      app.get('/:name', function(req, res, next){
-        res.send(req.params.name);
-      });
-
-      request(app)
-      .get('/foo+bar')
-      .expect('foo+bar', done);
-    })
-
-    it('should work with unicode', function(done) {
-      var app = express();
-
-      app.get('/:name', function(req, res, next){
-        res.send(req.params.name);
-      });
-
-      request(app)
-      .get('/%ce%b1')
-      .expect('\u03b1', done);
-    })
+    request(app)
+    .get('/%foobar')
+    .expect('%foobar', done);
   })
 
   it('should be .use()able', function(done){