streaming view rendering

Allows for a view engine to return a stream upon being asked to render a
path. This allows for the engine to simply stream out data as it is
loaded from disk (or other IO location) without having to collect it all
first and then send it.

Fallback mode for all existing engines that call `fn` upon completion
takes over if no stream is returned. This works for both app.render and
res.render

.travis.yml

@@ -1,4 +1,4 @@
 language: node_js
 node_js:
   - "0.8"
-  - "0.10"
+  - "0.10"

History.md

@@ -1,10 +1,25 @@
+3.4.4 / 2013-10-29
+==================
+
+ * update connect
+ * update supertest
+ * update methods
+ * express(1): replace bodyParser() with urlencoded() and json() #1795 @chirag04
+
+3.4.3 / 2013-10-23
+==================
+
+ * update connect
+
 3.4.2 / 2013-10-18
 ==================
+
  * update connect
  * downgrade commander
 
 3.4.1 / 2013-10-15
 ==================
+
  * update connect
  * update commander
  * jsonp: check if callback is a function

bin/express

@@ -222,7 +222,8 @@ var app = [
   , 'app.set(\'view engine\', \':TEMPLATE\');'
   , 'app.use(express.favicon());'
   , 'app.use(express.logger(\'dev\'));'
-  , 'app.use(express.bodyParser());'
+  , 'app.use(express.json());'
+  , 'app.use(express.urlencoded());'
   , 'app.use(express.methodOverride());{sess}'
   , 'app.use(app.router);{css}'
   , 'app.use(express.static(path.join(__dirname, \'public\')));'

lib/application.js

@@ -501,7 +501,26 @@ app.render = function(name, options, fn){
 
   // render
   try {
-    view.render(opts, fn);
+    var view_stream = view.render(opts, fn);
+
+    // if the engine returned a stream AND user specified a function
+    // then we will capture data for user
+    if (view_stream && fn) {
+      var body = '';
+      view_stream.on('data', function(chunk) {
+        body += chunk;
+      });
+
+      view_stream.once('error', function(err) {
+        fn(err);
+      });
+
+      view_stream.once('end', function() {
+        fn(null, body);
+      });
+    }
+
+    return view_stream;
   } catch (err) {
     fn(err);
   }

lib/response.js

@@ -14,6 +14,7 @@ var http = require('http')
   , cookie = require('cookie')
   , send = require('send')
   , mime = connect.mime
+  , resolve = require('url').resolve
   , basename = path.basename
   , extname = path.extname;
 
@@ -132,7 +133,7 @@ res.send = function(body){
 
   // ETag support
   // TODO: W/ support
-  if (app.settings.etag && len > 1024 && 'GET' == req.method) {
+  if (app.settings.etag && len && 'GET' == req.method) {
     if (!this.get('ETag')) {
       this.set('ETag', etag(body));
     }
@@ -633,7 +634,8 @@ res.cookie = function(name, val, options){
 
 res.location = function(url){
   var app = this.app
-    , req = this.req;
+    , req = this.req
+    , path;
 
   // setup redirect map
   var map = { back: req.get('Referrer') || '/' };
@@ -643,12 +645,11 @@ res.location = function(url){
 
   // relative
   if (!~url.indexOf('://') && 0 != url.indexOf('//')) {
-    var path
-
     // relative to path
     if ('.' == url[0]) {
-      path = req.originalUrl.split('?')[0]
-      url =  path + ('/' == path[path.length - 1] ? '' : '/') + url;
+      path = req.originalUrl.split('?')[0];
+      path = path + ('/' == path[path.length - 1] ? '' : '/');
+      url = resolve(path, url);
       // relative to mount-point
     } else if ('/' != url[0]) {
       path = app.path();
@@ -791,12 +792,49 @@ res.render = function(view, options, fn){
   // merge res.locals
   options._locals = self.locals;
 
-  // default callback to respond
-  fn = fn || function(err, str){
+  // support for non streaming view rendering
+  // if renderer uses streams, this will not be called
+  // see below for how streams are handled
+  var respond = fn || function(err, str){
+    if (view_stream) return;
+
     if (err) return req.next(err);
     self.send(str);
   };
 
-  // render
-  app.render(view, options, fn);
+  var view_stream = app.render(view, options, respond);
+
+  // old style, will have already called respond for us
+  if (!view_stream) {
+    return;
+  }
+
+  // user wants to handle sending themselves
+  if (fn) {
+    var body = '';
+    view_stream.on('data', function(chunk) {
+      body += chunk;
+    });
+
+    view_stream.once('error', function(err) {
+      fn(err);
+    });
+
+    view_stream.once('end', function() {
+      fn(null, body);
+    });
+
+    return;
+  }
+
+  // set response headers
+  self.statusCode = 200;
+  if (!self.get('Content-Type')) {
+    self.charset = self.charset || 'utf-8';
+    self.type('html');
+  }
+  self.setHeader('Transfer-Encoding', 'chunked');
+
+  // start streaming the response
+  view_stream.pipe(self);
 };

lib/view.js

@@ -73,5 +73,5 @@ View.prototype.lookup = function(path){
  */
 
 View.prototype.render = function(options, fn){
-  this.engine(this.path, options, fn);
+  return this.engine(this.path, options, fn);
 };

package.json

@@ -1,7 +1,7 @@
 {
   "name": "express",
   "description": "Sinatra inspired web development framework",
-  "version": "3.4.2",
+  "version": "3.4.4",
   "author": "TJ Holowaychuk <tj@vision-media.ca>",
   "contributors": [
     {
@@ -22,28 +22,28 @@
     }
   ],
   "dependencies": {
-    "connect": "2.9.2",
+    "connect": "2.11.0",
     "commander": "1.3.2",
     "range-parser": "0.0.4",
     "mkdirp": "0.3.5",
     "cookie": "0.1.0",
     "buffer-crc32": "0.2.1",
     "fresh": "0.2.0",
-    "methods": "0.0.1",
+    "methods": "0.1.0",
     "send": "0.1.4",
     "cookie-signature": "1.0.1",
-    "debug": "*"
+    "debug": ">= 0.7.3 < 1"
   },
   "devDependencies": {
-    "ejs": "*",
-    "mocha": "*",
+    "ejs": ">= 0.8.4 < 1",
+    "mocha": ">= 1.13.0 < 2",
     "jade": "0.30.0",
-    "hjs": "*",
-    "stylus": "*",
-    "should": "2",
-    "connect-redis": "*",
-    "marked": "*",
-    "supertest": "0.6.0"
+    "hjs": ">= 0.0.6 < 1",
+    "stylus": ">= 0.39.1 < 1",
+    "should": ">= 2.0.2 < 3",
+    "connect-redis": ">= 1.4.5 < 2",
+    "marked": ">= 0.2.9 < 1",
+    "supertest": ">= 0.8.1 < 1"
   },
   "keywords": [
     "express",
@@ -66,6 +66,6 @@
     "test": "make test"
   },
   "engines": {
-    "node": "*"
+    "node": ">= 0.8.0"
   }
 }

test/app.engine.js

@@ -1,5 +1,6 @@
 
 var express = require('../')
+  , request = require('./support/http')
   , fs = require('fs');
 
 function render(path, options, fn) {
@@ -10,6 +11,10 @@ function render(path, options, fn) {
   });
 }
 
+function streaming_render(path, options, fn) {
+  return fs.createReadStream(path, { encoding: 'utf-8' });
+}
+
 describe('app', function(){
   describe('.engine(ext, fn)', function(){
     it('should map a template engine', function(done){
@@ -76,5 +81,42 @@ describe('app', function(){
         done();
       })
     })
+
+    it('should support streaming engines', function(done) {
+      var app = express();
+      app.set('views', __dirname + '/fixtures');
+      app.engine('.html', streaming_render);
+      app.set('view engine', '.html');
+      app.locals.user = { name: 'tobi' };
+
+      // using function should just call the function when done
+      app.render('user', function(err, str){
+        if (err) return done(err);
+        str.should.equal('<p>{{user.name}}</p>');
+        done();
+      })
+    });
+
+    it('should render a response using the streaming engine', function(done) {
+      var app = express();
+      app.set('views', __dirname + '/fixtures');
+      app.engine('.html', streaming_render);
+      app.set('view engine', '.html');
+      app.locals.user = { name: 'tobi' };
+
+      app.get('/', function(req, res) {
+        res.render('user');
+      });
+
+      request(app)
+      .get('/')
+      .expect(200)
+      .end(function(err, res) {
+        if (err) return done(err);
+        res.header['transfer-encoding'].should.equal('chunked');
+        res.text.should.equal('<p>{{user.name}}</p>');
+        done();
+      });
+    });
   })
 })

test/req.acceptedEncodings.js

@@ -29,6 +29,7 @@ describe('req', function(){
 
         request(app)
         .get('/')
+        .set('Accept-Encoding', '')
         .expect(200, done);
       })
     })

test/res.location.js

@@ -64,7 +64,7 @@ describe('res', function(){
         request(app)
         .get('/post/1')
         .end(function(err, res){
-          res.headers.should.have.property('location', '/post/1/./edit');
+          res.headers.should.have.property('location', '/post/1/edit');
           done();
         })
       })
@@ -81,7 +81,24 @@ describe('res', function(){
         request(app)
         .get('/post/1')
         .end(function(err, res){
-          res.headers.should.have.property('location', '/post/1/../new');
+          res.headers.should.have.property('location', '/post/new');
+          done();
+        })
+      })
+    })
+
+    describe('with leading ./ and containing ..', function(){
+      it('should construct path-relative urls', function(done){
+        var app = express();
+
+        app.use(function(req, res){
+          res.location('./skip/../../new').end();
+        });
+
+        request(app)
+        .get('/post/1')
+        .end(function(err, res){
+          res.headers.should.have.property('location', '/post/new');
           done();
         })
       })

test/res.redirect.js

@@ -210,17 +210,17 @@ describe('res', function(){
       request(root)
       .get('/depth1')
       .end(function(err, res){
-        res.headers.should.have.property('location', '/depth1/./index');
+        res.headers.should.have.property('location', '/depth1/index');
 
         request(root)
         .get('/depth1/depth2')
         .end(function(err, res){
-          res.headers.should.have.property('location', '/depth1/depth2/./index');
+          res.headers.should.have.property('location', '/depth1/depth2/index');
 
           request(root)
           .get('/depth1/depth2/depth3')
           .end(function(err, res){
-            res.headers.should.have.property('location', '/depth1/depth2/depth3/./index');
+            res.headers.should.have.property('location', '/depth1/depth2/depth3/index');
             done();
           })
         })
@@ -231,11 +231,11 @@ describe('res', function(){
       request(root)
       .get('/depth2')
       .end(function(err, res){
-        res.headers.should.have.property('location', '/depth2/./index');
+        res.headers.should.have.property('location', '/depth2/index');
         request(root)
         .get('/depth3')
         .end(function(err, res){
-          res.headers.should.have.property('location', '/depth3/./index');
+          res.headers.should.have.property('location', '/depth3/index');
           done();
         })
       })

test/res.send.js

@@ -321,6 +321,21 @@ describe('res', function(){
 
   describe('"etag" setting', function(){
     describe('when enabled', function(){
+      it('should send ETag even when content-length < 1024', function(done){
+        var app = express();
+
+        app.use(function(req, res){
+          res.send('kajdslfkasdf');
+        });
+
+        request(app)
+        .get('/')
+        .end(function(err, res){
+          res.headers.should.have.property('etag');
+          done();
+        });
+      })
+
       it('should send ETag ', function(done){
         var app = express();
 

test/res.sendfile.js

@@ -51,7 +51,7 @@ describe('res', function(){
       .get('/')
       .end(function(err, res){
         assert(1 == calls, 'called too many times');
-        res.text.should.equal("ENOENT, stat 'test/fixtures/nope.html'");
+        res.text.should.startWith("ENOENT, stat");
         res.statusCode.should.equal(200);
         done();
       });