Prepare 1.0.6 release

* Convnet aliases

Not very important, but kinda handy.

* Update convolutional.py

* pep8

.travis.yml

@@ -49,9 +49,9 @@ install:
 
   # install TensorFlow
   - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
-      pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.7.1-cp27-none-linux_x86_64.whl;
+      pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.9.0-cp27-none-linux_x86_64.whl;
     elif [[ "$TRAVIS_PYTHON_VERSION" == "3.4" ]]; then
-      pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.7.1-cp34-none-linux_x86_64.whl;
+      pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.9.0-cp34-cp34m-linux_x86_64.whl;
     fi
 # command to run tests
 script:

README.md

@@ -2,6 +2,7 @@
 
 [![Build Status](https://travis-ci.org/fchollet/keras.svg?branch=master)](https://travis-ci.org/fchollet/keras)
 [![PyPI version](https://badge.fury.io/py/keras.svg)](https://badge.fury.io/py/keras)
+[![license](https://img.shields.io/github/license/mashape/apistatus.svg?maxAge=2592000)](https://github.com/fchollet/keras/blob/master/LICENSE)
 
 ## You have just found Keras.
 

docs/autogen.py

@@ -146,13 +146,8 @@ PAGES = [
         'classes': [
             convolutional.Convolution1D,
             convolutional.Convolution2D,
+            convolutional.AtrousConv2D,
             convolutional.Convolution3D,
-            convolutional.MaxPooling1D,
-            convolutional.MaxPooling2D,
-            convolutional.MaxPooling3D,
-            convolutional.AveragePooling1D,
-            convolutional.AveragePooling2D,
-            convolutional.AveragePooling3D,
             convolutional.UpSampling1D,
             convolutional.UpSampling2D,
             convolutional.UpSampling3D,
@@ -161,6 +156,17 @@ PAGES = [
             convolutional.ZeroPadding3D,
         ],
     },
+    {
+        'page': 'layers/pooling.md',
+        'classes': [
+            convolutional.MaxPooling1D,
+            convolutional.MaxPooling2D,
+            convolutional.MaxPooling3D,
+            convolutional.AveragePooling1D,
+            convolutional.AveragePooling2D,
+            convolutional.AveragePooling3D,
+        ],
+    },
     {
         'page': 'layers/recurrent.md',
         'classes': [

docs/mkdocs.yml

@@ -24,6 +24,7 @@ pages:
   - About Keras layers: layers/about-keras-layers.md
   - Core Layers: layers/core.md
   - Convolutional Layers: layers/convolutional.md
+  - Pooling Layers: layers/pooling.md
   - Recurrent Layers: layers/recurrent.md
   - Embedding Layers: layers/embeddings.md
   - Advanced Activations Layers: layers/advanced-activations.md

docs/templates/getting-started/faq.md

@@ -12,6 +12,8 @@
 - [How can I record the training / validation loss / accuracy at each epoch?](#how-can-i-record-the-training-validation-loss-accuracy-at-each-epoch)
 - [How can I "freeze" layers?](#how-can-i-freeze-keras-layers)
 - [How can I use stateful RNNs?](#how-can-i-use-stateful-rnns)
+- [How can I remove a layer from a Sequential model?](#how-can-i-remove-a-layer-from-a-sequential-model)
+- [How can I use pre-trained models in Keras?](#how-can-i-use-pre-trained-models-in-keras)
 
 ---
 
@@ -296,3 +298,37 @@ model.layers[0].reset_states()
 
 Notes that the methods `predict`, `fit`, `train_on_batch`, `predict_classes`, etc. will *all* update the states of the stateful layers in a model. This allows you to do not only stateful training, but also stateful prediction.
 
+---
+
+### How can I remove a layer from a Sequential model?
+
+You can remove the last added layer in a Sequential model by calling `.pop()`:
+
+```python
+model = Sequential()
+model.add(Dense(32, activation='relu', input_dim=784))
+model.add(Dense(32, activation='relu'))
+
+print(len(model.layers))  # "2"
+
+model.pop()
+print(len(model.layers))  # "1"
+```
+
+---
+
+### How can I use pre-trained models in Keras?
+
+Code and pre-trained weights are available for the following image classification models:
+
+- [VGG-16](https://gist.github.com/baraldilorenzo/07d7802847aaad0a35d3)
+- [VGG-19](https://gist.github.com/baraldilorenzo/8d096f48a1be4a2d660d)
+- [AlexNet](https://github.com/heuritech/convnets-keras)
+
+For an example of how to use such a pre-trained model for feature extraction or for fine-tuning, see [this blog post](http://blog.keras.io/building-powerful-image-classification-models-using-very-little-data.html).
+
+The VGG-16 model is also the basis for several Keras example scripts:
+
+- [Style transfer](https://github.com/fchollet/keras/blob/master/examples/neural_style_transfer.py)
+- [Feature visualization](https://github.com/fchollet/keras/blob/master/examples/conv_filter_visualization.py)
+- [Deep dream](https://github.com/fchollet/keras/blob/master/examples/deep_dream.py)

docs/templates/getting-started/sequential-model-guide.md

@@ -381,7 +381,7 @@ image_model.load_weights('weight_file.h5')
 language_model = Sequential()
 language_model.add(Embedding(vocab_size, 256, input_length=max_caption_len))
 language_model.add(GRU(output_dim=128, return_sequences=True))
-language_model.add(TimeDistributedDense(128))
+language_model.add(TimeDistributed(Dense(128)))
 
 # let's repeat the image vector to turn it into a sequence.
 image_model.add(RepeatVector(max_caption_len))

docs/templates/initializations.md

@@ -1,7 +1,7 @@
 
 ## Usage of initializations
 
-Initializations define the probability distribution used to set the initial random weights of Keras layers.
+Initializations define the way to set the initial random weights of Keras layers.
 
 The keyword arguments used for passing initializations to layers will depend on the layer. Usually it is simply `init`:
 
@@ -21,3 +21,30 @@ model.add(Dense(64, init='uniform'))
 - __glorot_uniform__
 - __he_normal__: Gaussian initialization scaled by fan_in (He et al., 2014)
 - __he_uniform__
+
+
+An initialization may be passed as a string (must match one of the available initializations above), or as a callable.
+If a callable, then it must take two arguments: `shape` (shape of the variable to initialize) and `name` (name of the variable),
+and it must return a variable (e.g. output of `K.variable()`):
+
+```python
+from keras import backend as K
+import numpy as np
+
+def my_init(shape, name=None):
+    value = np.random.random(shape)
+    return K.variable(value, name=name)
+
+model.add(Dense(64, init=my_init))
+```
+
+You could also use functions from `keras.initializations` in this way:
+
+```python
+from keras import initializations
+
+def my_init(shape, name=None):
+    return initializations.normal(shape, scale=0.01, name=name)
+
+model.add(Dense(64, init=my_init))
+```

keras/__init__.py

@@ -15,4 +15,4 @@ from . import objectives
 from . import optimizers
 from . import regularizers
 
-__version__ = '1.0.5'
+__version__ = '1.0.6'

keras/activations.py

@@ -48,4 +48,6 @@ def linear(x):
 
 from .utils.generic_utils import get_from_module
 def get(identifier):
+    if identifier is None:
+        return linear
     return get_from_module(identifier, globals(), 'activation function')

keras/backend/__init__.py

@@ -35,6 +35,7 @@ if os.path.exists(_config_path):
 
     set_floatx(_floatx)
     set_epsilon(_epsilon)
+    set_image_dim_ordering(_image_dim_ordering)
     _BACKEND = _backend
 
 # save config file

keras/backend/common.py

@@ -1,9 +1,11 @@
 import numpy as np
 
+from collections import defaultdict
+
 # the type of float to use throughout the session.
 _FLOATX = 'float32'
 _EPSILON = 10e-8
-_UID_PREFIXES = {}
+_UID_PREFIXES = defaultdict(int)
 _IMAGE_DIM_ORDERING = 'th'
 
 
@@ -60,9 +62,5 @@ def set_image_dim_ordering(dim_ordering):
 
 
 def get_uid(prefix=''):
-    if prefix not in _UID_PREFIXES:
-        _UID_PREFIXES[prefix] = 1
-        return 1
-    else:
-        _UID_PREFIXES[prefix] += 1
-        return _UID_PREFIXES[prefix]
+    _UID_PREFIXES[prefix] += 1
+    return _UID_PREFIXES[prefix]

keras/backend/tensorflow_backend.py

@@ -3,12 +3,23 @@ import numpy as np
 import os
 import copy
 import warnings
-from .common import _FLOATX, _EPSILON
+from .common import _FLOATX, _EPSILON, _IMAGE_DIM_ORDERING
 
 # INTERNAL UTILS
 
 _SESSION = None
 _LEARNING_PHASE = tf.placeholder(dtype='uint8', name='keras_learning_phase')  # 0 = test, 1 = train
+_MANUAL_VAR_INIT = False
+
+
+def manual_variable_initialization(value):
+    '''Whether variables should be initialized
+    as they are instantiated (default), or if
+    the user should handle the initialization
+    (e.g. via tf.initialize_all_variables()).
+    '''
+    global _MANUAL_VAR_INIT
+    _MANUAL_VAR_INIT = value
 
 
 def learning_phase():
@@ -61,6 +72,23 @@ def set_session(session):
 
 # VARIABLE MANIPULATION
 
+def _convert_string_dtype(dtype):
+    if dtype == 'float16':
+        return tf.float16
+    if dtype == 'float32':
+        return tf.float32
+    elif dtype == 'float64':
+        return tf.float64
+    elif dtype == 'int32':
+        return tf.int32
+    elif dtype == 'int64':
+        return tf.int64
+    elif dtype == 'uint8':
+        return tf.int8
+    else:
+        raise ValueError('Unsupported dtype:', dtype)
+
+
 def variable(value, dtype=_FLOATX, name=None):
     '''Instantiates a tensor.
 
@@ -72,7 +100,9 @@ def variable(value, dtype=_FLOATX, name=None):
     # Returns
         Tensor variable instance.
     '''
-    v = tf.Variable(np.asarray(value, dtype=dtype), name=name)
+    v = tf.Variable(value, dtype=_convert_string_dtype(dtype), name=name)
+    if _MANUAL_VAR_INIT:
+        return v
     if tf.get_default_graph() is get_session().graph:
         try:
             get_session().run(v.initializer)
@@ -154,13 +184,15 @@ def eval(x):
 def zeros(shape, dtype=_FLOATX, name=None):
     '''Instantiates an all-zeros tensor variable.
     '''
-    return variable(np.zeros(shape), dtype, name)
+    return variable(lambda: tf.cast(tf.constant_initializer(0.)(shape), dtype),
+                    dtype, name)
 
 
 def ones(shape, dtype=_FLOATX, name=None):
     '''Instantiates an all-ones tensor variable.
     '''
-    return variable(np.ones(shape), dtype, name)
+    return variable(lambda: tf.cast(tf.constant_initializer(1.)(shape), dtype),
+                    dtype, name)
 
 
 def eye(size, dtype=_FLOATX, name=None):
@@ -224,8 +256,7 @@ def batch_dot(x, y, axes=None):
     make sure that ndim is at least 2.
 
     # Example
-        Assume x = [[1, 2]   and y = [[5, 6]
-                    [3, 4]]           [7, 8]]
+        Assume x = [[1, 2], [3, 4]]   and y = [[5, 6], [7, 8]]
         batch_dot(x, y, axes=1) = [[17, 53]] which is the main diagonal
         of x.dot(y.T), although we never have to calculate the off-diagonal
         elements.
@@ -480,6 +511,42 @@ def cos(x):
     return tf.cos(x)
 
 
+def normalize_batch_in_training(x, gamma, beta,
+                                reduction_axes, epsilon=0.0001):
+    '''Compute mean and std for batch then apply batch_normalization on batch.
+    '''
+    mean, std = tf.nn.moments(x, reduction_axes,
+                              shift=None, name=None, keep_dims=False)
+    if sorted(reduction_axes) == range(ndim(x))[:-1]:
+        normed = tf.nn.batch_normalization(x, mean, std,
+                                           beta, gamma,
+                                           epsilon)
+    else:
+        # need broadcasting
+        target_shape = []
+        for axis in range(ndim(x)):
+            if axis in reduction_axes:
+                target_shape.append(1)
+            else:
+                target_shape.append(tf.shape(x)[axis])
+        target_shape = tf.pack(target_shape)
+
+        broadcast_mean = tf.reshape(mean, target_shape)
+        broadcast_std = tf.reshape(std, target_shape)
+        broadcast_gamma = tf.reshape(gamma, target_shape)
+        broadcast_beta = tf.reshape(beta, target_shape)
+        normed = tf.nn.batch_normalization(x, broadcast_mean, broadcast_std,
+                                           broadcast_beta, broadcast_gamma,
+                                           epsilon)
+    return normed, mean, std
+
+
+def batch_normalization(x, mean, std, beta, gamma, epsilon=0.0001):
+    '''Apply batch normalization on x given mean, std, beta and gamma.
+    '''
+    return tf.nn.batch_normalization(x, mean, std, beta, gamma, epsilon)
+
+
 # SHAPE OPERATIONS
 
 def concatenate(tensors, axis=-1):
@@ -656,6 +723,7 @@ def batch_set_value(tuples):
         ops = [tf.assign(x, np.asarray(value)) for x, value in tuples]
         get_session().run(ops)
 
+
 # GRAPH MANIPULATION
 
 class Function(object):
@@ -702,6 +770,13 @@ def gradients(loss, variables):
     return tf.gradients(loss, variables)
 
 
+def stop_gradient(variables):
+    '''Returns `variables` but with zero gradient with respect to every other
+    variables.
+    '''
+    return tf.stop_gradient(variables)
+
+
 # CONTROL FLOW
 
 def rnn(step_function, inputs, initial_states,
@@ -998,55 +1073,205 @@ def l2_normalize(x, axis):
 
 # CONVOLUTIONS
 
+def _preprocess_conv2d_input(x, dim_ordering):
+    if _FLOATX == 'float64':
+        x = tf.cast(x, 'float32')
+    if dim_ordering == 'th':
+        # TF uses the last dimension as channel dimension,
+        # instead of the 2nd one.
+        # TH input shape: (samples, input_depth, rows, cols)
+        # TF input shape: (samples, rows, cols, input_depth)
+        x = tf.transpose(x, (0, 2, 3, 1))
+    return x
 
-def conv2d(x, kernel, strides=(1, 1), border_mode='valid', dim_ordering='th',
-           image_shape=None, filter_shape=None):
-    '''2D convolution.
 
-    # Arguments
-        kernel: kernel tensor.
-        strides: strides tuple.
-        border_mode: string, "same" or "valid".
-        dim_ordering: "tf" or "th". Whether to use Theano or TensorFlow dimension ordering
-        in inputs/kernels/ouputs.
-    '''
+def _preprocess_conv3d_input(x, dim_ordering):
+    if _FLOATX == 'float64':
+        x = tf.cast(x, 'float32')
+    if dim_ordering == 'th':
+        # TF uses the last dimension as channel dimension,
+        # instead of the 2nd one.
+        # TH input shape: (samples, input_depth, conv_dim1, conv_dim2, conv_dim3)
+        # TF input shape: (samples, conv_dim1, conv_dim2, conv_dim3, input_depth)
+        x = tf.transpose(x, (0, 2, 3, 4, 1))
+    return x
+
+
+def _preprocess_conv2d_kernel(kernel, dim_ordering):
+    if _FLOATX == 'float64':
+        kernel = tf.cast(kernel, 'float32')
+    if dim_ordering == 'th':
+        # TF uses the last dimension as channel dimension,
+        # instead of the 2nd one.
+        # TH kernel shape: (depth, input_depth, rows, cols)
+        # TF kernel shape: (rows, cols, input_depth, depth)
+        kernel = tf.transpose(kernel, (2, 3, 1, 0))
+    return kernel
+
+
+def _preprocess_conv3d_kernel(kernel, dim_ordering):
+    if _FLOATX == 'float64':
+        kernel = tf.cast(kernel, 'float32')
+    if dim_ordering == 'th':
+        # TF uses the last dimension as channel dimension,
+        # instead of the 2nd one.
+        # TH kernel shape: (out_depth, input_depth, kernel_dim1, kernel_dim2, kernel_dim3)
+        # TF kernel shape: (kernel_dim1, kernel_dim2, kernel_dim3, input_depth, out_depth)
+        kernel = tf.transpose(kernel, (2, 3, 4, 1, 0))
+    return kernel
+
+
+def _preprocess_border_mode(border_mode):
     if border_mode == 'same':
         padding = 'SAME'
     elif border_mode == 'valid':
         padding = 'VALID'
     else:
         raise Exception('Invalid border mode: ' + str(border_mode))
+    return padding
 
-    strides = (1,) + strides + (1,)
-
-    if _FLOATX == 'float64':
-        # tf conv2d only supports float32
-        x = tf.cast(x, 'float32')
-        kernel = tf.cast(kernel, 'float32')
 
+def _postprocess_conv2d_output(x, dim_ordering):
     if dim_ordering == 'th':
-        # TF uses the last dimension as channel dimension,
-        # instead of the 2nd one.
-        # TH input shape: (samples, input_depth, rows, cols)
-        # TF input shape: (samples, rows, cols, input_depth)
-        # TH kernel shape: (depth, input_depth, rows, cols)
-        # TF kernel shape: (rows, cols, input_depth, depth)
-        x = tf.transpose(x, (0, 2, 3, 1))
-        kernel = tf.transpose(kernel, (2, 3, 1, 0))
-        x = tf.nn.conv2d(x, kernel, strides, padding=padding)
         x = tf.transpose(x, (0, 3, 1, 2))
-    elif dim_ordering == 'tf':
-        x = tf.nn.conv2d(x, kernel, strides, padding=padding)
-    else:
-        raise Exception('Unknown dim_ordering: ' + str(dim_ordering))
 
     if _FLOATX == 'float64':
         x = tf.cast(x, 'float64')
     return x
 
 
+def _postprocess_conv3d_output(x, dim_ordering):
+    if dim_ordering == 'th':
+        x = tf.transpose(x, (0, 4, 1, 2, 3))
+
+    if _FLOATX == 'float64':
+        x = tf.cast(x, 'float64')
+    return x
+
+
+def conv2d(x, kernel, strides=(1, 1), border_mode='valid',
+           dim_ordering=_IMAGE_DIM_ORDERING,
+           image_shape=None, filter_shape=None, filter_dilation=(1, 1)):
+    '''2D convolution.
+
+    # Arguments
+        kernel: kernel tensor.
+        strides: strides tuple.
+        border_mode: string, "same" or "valid".
+        dim_ordering: "tf" or "th".
+            Whether to use Theano or TensorFlow dimension ordering
+            for inputs/kernels/ouputs.
+    '''
+    if dim_ordering not in {'th', 'tf'}:
+        raise Exception('Unknown dim_ordering ' + str(dim_ordering))
+
+    x = _preprocess_conv2d_input(x, dim_ordering)
+    kernel = _preprocess_conv2d_kernel(kernel, dim_ordering)
+    padding = _preprocess_border_mode(border_mode)
+    if filter_dilation == (1, 1):
+        strides = (1,) + strides + (1,)
+        x = tf.nn.conv2d(x, kernel, strides, padding=padding)
+    else:
+        assert filter_dilation[0] == filter_dilation[1]
+        assert strides == (1, 1), 'Invalid strides for dilated convolution'
+        x = tf.nn.atrous_conv2d(x, kernel, filter_dilation[0], padding=padding)
+    return _postprocess_conv2d_output(x, dim_ordering)
+
+
+def deconv2d(x, kernel, output_shape, strides=(1, 1),
+             border_mode='valid',
+             dim_ordering=_IMAGE_DIM_ORDERING,
+             image_shape=None, filter_shape=None):
+    '''2D deconvolution (i.e. transposed convolution).
+
+    # Arguments
+        x: input tensor.
+        kernel: kernel tensor.
+        output_shape: 1D int tensor for the output shape.
+        strides: strides tuple.
+        border_mode: string, "same" or "valid".
+        dim_ordering: "tf" or "th".
+            Whether to use Theano or TensorFlow dimension ordering
+            for inputs/kernels/ouputs.
+    '''
+    if dim_ordering not in {'th', 'tf'}:
+        raise Exception('Unknown dim_ordering ' + str(dim_ordering))
+
+    x = _preprocess_conv2d_input(x, dim_ordering)
+    kernel = _preprocess_conv2d_kernel(kernel, dim_ordering)
+    padding = _preprocess_border_mode(border_mode)
+    strides = (1,) + strides + (1,)
+
+    # TODO: pre-process output_shape if dim_ordering == th
+    x = tf.nn.conv2d_transpose(x, kernel, output_shape, strides,
+                               padding=padding)
+    return _postprocess_conv2d_output(x, dim_ordering)
+
+
+def atrous_conv2d(x, kernel, rate=1,
+                  border_mode='valid',
+                  dim_ordering=_IMAGE_DIM_ORDERING,
+                  image_shape=None, filter_shape=None):
+    if dim_ordering not in {'th', 'tf'}:
+        raise Exception('Unknown dim_ordering ' + str(dim_ordering))
+    if rate == 1:
+        return conv2d(x, kernel, strides=(1, 1), border_mode=border_mode,
+                      dim_ordering=dim_ordering)
+
+    x = _preprocess_conv2d_input(x, dim_ordering)
+    kernel = _preprocess_conv2d_kernel(kernel, dim_ordering)
+    padding = _preprocess_border_mode(border_mode)
+
+    x = tf.nn.atrous_conv2d(x, kernel, rate, padding)
+    return _postprocess_conv2d_output(x, dim_ordering)
+
+
+def separable_conv2d(x, depthwise_kernel, pointwise_kernel, strides=(1, 1),
+                     border_mode='valid', dim_ordering=_IMAGE_DIM_ORDERING):
+    if dim_ordering not in {'th', 'tf'}:
+        raise Exception('Unknown dim_ordering ' + str(dim_ordering))
+
+    x = _preprocess_conv2d_input(x, dim_ordering)
+    depthwise_kernel = _preprocess_conv2d_kernel(depthwise_kernel,
+                                                 dim_ordering)
+    pointwise_kernel = _preprocess_conv2d_kernel(pointwise_kernel,
+                                                 dim_ordering)
+    padding = _preprocess_border_mode(border_mode)
+    strides = (1,) + strides + (1,)
+
+    x = tf.nn.separable_conv2d(x, depthwise_kernel, pointwise_kernel,
+                               strides, padding)
+    return _postprocess_conv2d_output(x, dim_ordering)
+
+
+def conv3d(x, kernel, strides=(1, 1, 1),
+           border_mode='valid', dim_ordering=_IMAGE_DIM_ORDERING,
+           volume_shape=None, filter_shape=None):
+    '''3D convolution.
+
+    # Arguments
+        kernel: kernel tensor.
+        strides: strides tuple.
+        border_mode: string, "same" or "valid".
+        dim_ordering: "tf" or "th".
+            Whether to use Theano or TensorFlow dimension ordering
+            for inputs/kernels/ouputs.
+    '''
+    if dim_ordering not in {'th', 'tf'}:
+        raise Exception('Unknown dim_ordering ' + str(dim_ordering))
+
+    x = _preprocess_conv3d_input(x, dim_ordering)
+    kernel = _preprocess_conv3d_kernel(kernel, dim_ordering)
+    padding = _preprocess_border_mode(border_mode)
+    strides = (1,) + strides + (1,)
+
+    x = tf.nn.conv3d(x, kernel, strides, padding)
+    return _postprocess_conv3d_output(x, dim_ordering)
+
+
 def pool2d(x, pool_size, strides=(1, 1),
-           border_mode='valid', dim_ordering='th', pool_mode='max'):
+           border_mode='valid', dim_ordering=_IMAGE_DIM_ORDERING,
+           pool_mode='max'):
     '''2D Pooling.
 
     # Arguments
@@ -1056,43 +1281,53 @@ def pool2d(x, pool_size, strides=(1, 1),
         dim_ordering: one of "th", "tf".
         pool_mode: one of "max", "avg".
     '''
-    if border_mode == 'same':
-        padding = 'SAME'
-    elif border_mode == 'valid':
-        padding = 'VALID'
-    else:
-        raise Exception('Invalid border mode: ' + str(border_mode))
+    if dim_ordering not in {'th', 'tf'}:
+        raise Exception('Unknown dim_ordering ' + str(dim_ordering))
 
+    padding = _preprocess_border_mode(border_mode)
     strides = (1,) + strides + (1,)
     pool_size = (1,) + pool_size + (1,)
 
-    if _FLOATX == 'float64':
-        # tf max_pool only supports float32
-        x = tf.cast(x, 'float32')
+    x = _preprocess_conv2d_input(x, dim_ordering)
 
-    if dim_ordering in {'tf', 'th'}:
-        if dim_ordering == 'th':
-            # TF uses the last dimension as channel dimension,
-            # instead of the 2nd one.
-            # TH input shape: (samples, input_depth, rows, cols)
-            # TF input shape: (samples, rows, cols, input_depth)
-            # TH kernel shape: (depth, input_depth, rows, cols)
-            # TF kernel shape: (rows, cols, input_depth, depth)
-            x = tf.transpose(x, (0, 2, 3, 1))
-        if pool_mode == 'max':
-            x = tf.nn.max_pool(x, pool_size, strides, padding=padding)
-        elif pool_mode == 'avg':
-            x = tf.nn.avg_pool(x, pool_size, strides, padding=padding)
-        else:
-            raise Exception('Invalid pooling mode: ' + str(pool_mode))
-        if dim_ordering == 'th':
-            x = tf.transpose(x, (0, 3, 1, 2))
+    if pool_mode == 'max':
+        x = tf.nn.max_pool(x, pool_size, strides, padding=padding)
+    elif pool_mode == 'avg':
+        x = tf.nn.avg_pool(x, pool_size, strides, padding=padding)
     else:
-        raise Exception('Unknown dim_ordering: ' + str(dim_ordering))
+        raise Exception('Invalid pooling mode: ' + str(pool_mode))
 
-    if _FLOATX == 'float64':
-        x = tf.cast(x, 'float64')
-    return x
+    return _postprocess_conv2d_output(x, dim_ordering)
+
+
+def pool3d(x, pool_size, strides=(1, 1, 1), border_mode='valid',
+           dim_ordering=_IMAGE_DIM_ORDERING, pool_mode='max'):
+    '''3D Pooling.
+
+    # Arguments
+        pool_size: tuple of 3 integers.
+        strides: tuple of 3 integers.
+        border_mode: one of "valid", "same".
+        dim_ordering: one of "th", "tf".
+        pool_mode: one of "max", "avg".
+    '''
+    if dim_ordering not in {'th', 'tf'}:
+        raise Exception('Unknown dim_ordering ' + str(dim_ordering))
+
+    padding = _preprocess_border_mode(border_mode)
+    strides = (1,) + strides + (1,)
+    pool_size = (1,) + pool_size + (1,)
+
+    x = _preprocess_conv3d_input(x, dim_ordering)
+
+    if pool_mode == 'max':
+        x = tf.nn.max_pool3d(x, pool_size, strides, padding=padding)
+    elif pool_mode == 'avg':
+        x = tf.nn.avg_pool3d(x, pool_size, strides, padding=padding)
+    else:
+        raise Exception('Invalid pooling mode: ' + str(pool_mode))
+
+    return _postprocess_conv3d_output(x, dim_ordering)
 
 
 # RANDOMNESS
@@ -1115,4 +1350,5 @@ def random_binomial(shape, p=0.0, dtype=_FLOATX, seed=None):
     if seed is None:
         seed = np.random.randint(10e6)
     return tf.select(tf.random_uniform(shape, dtype=dtype, seed=seed) <= p,
-                     tf.ones(shape), tf.zeros(shape))
+                     tf.ones(shape, dtype=dtype),
+                     tf.zeros(shape, dtype=dtype))

keras/backend/theano_backend.py

@@ -9,7 +9,7 @@ except ImportError:
     from theano.sandbox.softsign import softsign as T_softsign
 import inspect
 import numpy as np
-from .common import _FLOATX, _EPSILON
+from .common import _FLOATX, _EPSILON, _IMAGE_DIM_ORDERING
 
 
 # INTERNAL UTILS
@@ -128,8 +128,7 @@ def batch_dot(x, y, axes=None):
     make sure that ndim is at least 2.
 
     # Example
-        Assume x = [[1, 2]   and y = [[5, 6]
-                    [3, 4]]           [7, 8]]
+        Assume x = [[1, 2], [3, 4]]   and y = [[5, 6], [7, 8]]
         batch_dot(x, y, axes=1) = [[17, 53]] which is the main diagonal
         of x.dot(y.T), although we never have to calculate the off-diagonal
         elements.
@@ -287,6 +286,38 @@ def cos(x):
     return T.cos(x)
 
 
+def normalize_batch_in_training(x, gamma, beta,
+                                reduction_axes, epsilon=0.0001):
+    '''Compute mean and std for batch then apply batch_normalization on batch.
+    '''
+    std = T.sqrt(x.var(reduction_axes) + epsilon)
+    mean = x.mean(reduction_axes)
+
+    target_shape = []
+    for axis in range(ndim(x)):
+        if axis in reduction_axes:
+            target_shape.append(1)
+        else:
+            target_shape.append(x.shape[axis])
+    target_shape = T.stack(*target_shape)
+
+    broadcast_mean = T.reshape(mean, target_shape)
+    broadcast_std = T.reshape(std, target_shape)
+    broadcast_beta = T.reshape(beta, target_shape)
+    broadcast_gamma = T.reshape(gamma, target_shape)
+    normed = batch_normalization(x, broadcast_mean, broadcast_std,
+                                 broadcast_beta, broadcast_gamma,
+                                 epsilon)
+    return normed, mean, std
+
+
+def batch_normalization(x, mean, std, beta, gamma, epsilon=0.0001):
+    '''Apply batch normalization on x given mean, std, beta and gamma.
+    '''
+    normed = (x - mean) * (gamma * T.inv(std + epsilon)) + beta
+    return normed
+
+
 # SHAPE OPERATIONS
 
 def concatenate(tensors, axis=-1):
@@ -545,6 +576,13 @@ def gradients(loss, variables):
     return T.grad(loss, variables)
 
 
+def stop_gradient(variables):
+    '''Returns `variables` but with zero gradient with respect to every other
+    variables.
+    '''
+    return theano.gradient.disconnected_grad(variables)
+
+
 # CONTROL FLOW
 
 def rnn(step_function, inputs, initial_states,
@@ -810,10 +848,18 @@ def l2_normalize(x, axis):
 
 # CONVOLUTIONS
 
-def conv2d(x, kernel, strides=(1, 1), border_mode='valid', dim_ordering='th',
-           image_shape=None, filter_shape=None):
-    '''
-    border_mode: string, "same" or "valid".
+def conv2d(x, kernel, strides=(1, 1), border_mode='valid',
+           dim_ordering=_IMAGE_DIM_ORDERING, image_shape=None,
+           filter_shape=None, filter_dilation=(1, 1)):
+    '''2D convolution.
+
+    # Arguments
+        kernel: kernel tensor.
+        strides: strides tuple.
+        border_mode: string, "same" or "valid".
+        dim_ordering: "tf" or "th".
+            Whether to use Theano or TensorFlow dimension ordering
+        in inputs/kernels/ouputs.
     '''
     if dim_ordering not in {'th', 'tf'}:
         raise Exception('Unknown dim_ordering ' + str(dim_ordering))
@@ -855,11 +901,20 @@ def conv2d(x, kernel, strides=(1, 1), border_mode='valid', dim_ordering='th',
     if filter_shape is not None:
         filter_shape = tuple(int_or_none(v) for v in filter_shape)
 
-    conv_out = T.nnet.conv2d(x, kernel,
-                             border_mode=th_border_mode,
-                             subsample=strides,
-                             input_shape=image_shape,
-                             filter_shape=filter_shape)
+    # TODO: remove the if statement when theano with no filter dilation is deprecated.
+    if filter_dilation == (1, 1):
+        conv_out = T.nnet.conv2d(x, kernel,
+                                 border_mode=th_border_mode,
+                                 subsample=strides,
+                                 input_shape=image_shape,
+                                 filter_shape=filter_shape)
+    else:
+        conv_out = T.nnet.conv2d(x, kernel,
+                                 border_mode=th_border_mode,
+                                 subsample=strides,
+                                 input_shape=image_shape,
+                                 filter_shape=filter_shape,
+                                 filter_dilation=filter_dilation)
 
     if border_mode == 'same':
         if np_kernel.shape[2] % 2 == 0:
@@ -872,6 +927,25 @@ def conv2d(x, kernel, strides=(1, 1), border_mode='valid', dim_ordering='th',
     return conv_out
 
 
+def deconv2d(x, kernel, output_shape, strides=(1, 1),
+             border_mode='valid',
+             dim_ordering=_IMAGE_DIM_ORDERING,
+             image_shape=None, filter_shape=None):
+    raise NotImplementedError
+
+
+def atrous_conv2d(x, kernel, rate=1,
+                  border_mode='valid',
+                  dim_ordering=_IMAGE_DIM_ORDERING,
+                  image_shape=None, filter_shape=None):
+    raise NotImplementedError
+
+
+def separable_conv2d(x, depthwise_kernel, pointwise_kernel, strides=(1, 1),
+                     border_mode='valid', dim_ordering=_IMAGE_DIM_ORDERING):
+    raise NotImplementedError
+
+
 def conv3d(x, kernel, strides=(1, 1, 1),
            border_mode='valid', dim_ordering='th',
            volume_shape=None, filter_shape=None):

keras/callbacks.py

@@ -361,13 +361,19 @@ class RemoteMonitor(Callback):
     # Arguments
         root: root url to which the events will be sent (at the end
             of every epoch). Events are sent to
-            `root + '/publish/epoch/end/'`. Calls are HTTP POST,
-            with a `data` argument which is a JSON-encoded dictionary
-            of event data.
+            `root + '/publish/epoch/end/'` by default. Calls are 
+            HTTP POST, with a `data` argument which is a 
+            JSON-encoded dictionary of event data.
     '''
-    def __init__(self, root='http://localhost:9000'):
+
+    def __init__(self,
+                 root='http://localhost:9000',
+                 path='/publish/epoch/end/',
+                 field='data'):
         super(RemoteMonitor, self).__init__()
         self.root = root
+        self.path = path
+        self.field = field
 
     def on_epoch_end(self, epoch, logs={}):
         import requests
@@ -375,10 +381,9 @@ class RemoteMonitor(Callback):
         send['epoch'] = epoch
         for k, v in logs.items():
             send[k] = v
-
         try:
-            requests.post(self.root + '/publish/epoch/end/',
-                          {'data': json.dumps(send)})
+            requests.post(self.root + self.path,
+                          {self.field: json.dumps(send)})
         except:
             print('Warning: could not reach RemoteMonitor '
                   'root server at ' + str(self.root))

keras/engine/topology.py

@@ -914,7 +914,7 @@ class InputLayer(Layer):
     '''TODO: dosctring
     '''
     def __init__(self, input_shape=None, batch_input_shape=None,
-                 input_dtype=None, name=None):
+                 input_dtype=None, input_tensor=None, name=None):
         self.input_spec = None
         self.supports_masking = False
         self.uses_learning_phase = False
@@ -934,25 +934,48 @@ class InputLayer(Layer):
             name = prefix + '_' + str(K.get_uid(prefix))
         self.name = name
 
+        if input_shape and batch_input_shape:
+            raise ValueError('Only provide the input_shape OR '
+                             'batch_input_shape argument to '
+                             'InputLayer, not both at the same time.')
+        if input_tensor is not None:
+            if not input_shape and not batch_input_shape:
+                # attempt automatic input shape inference
+                try:
+                    batch_input_shape = K.int_shape(input_tensor)
+                except:
+                    raise ValueError('InputLayer was provided an input_tensor argument, '
+                                     'but its input shape cannot be automatically inferred. '
+                                     'You should pass an input_shape or batch_input_shape '
+                                     'argument.')
         if not batch_input_shape:
-            assert input_shape, 'An Input layer should be passed either a `batch_input_shape` or an `input_shape`.'
-            batch_input_shape = (None,) + tuple(input_shape)
+            if not input_shape:
+                raise ValueError('An Input layer should be passed either '
+                                 'a `batch_input_shape` or an `input_shape`.')
+            else:
+                batch_input_shape = (None,) + tuple(input_shape)
         else:
             batch_input_shape = tuple(batch_input_shape)
+
         if not input_dtype:
-            input_dtype = K.floatx()
+            if input_tensor is None:
+                input_dtype = K.floatx()
+            else:
+                input_dtype = K.dtype(input_tensor)
 
         self.batch_input_shape = batch_input_shape
         self.input_dtype = input_dtype
 
-        input_tensor = K.placeholder(shape=batch_input_shape,
-                                     dtype=input_dtype,
-                                     name=self.name)
+        if input_tensor is None:
+            input_tensor = K.placeholder(shape=batch_input_shape,
+                                         dtype=input_dtype,
+                                         name=self.name)
+        else:
+            input_tensor._keras_shape = batch_input_shape
         # create an input node to add to self.outbound_node
         # and set output_tensors' _keras_history
         input_tensor._uses_learning_phase = False
         input_tensor._keras_history = (self, 0, 0)
-        shape = input_tensor._keras_shape
         Node(self,
              inbound_layers=[],
              node_indices=[],
@@ -961,8 +984,8 @@ class InputLayer(Layer):
              output_tensors=[input_tensor],
              input_masks=[None],
              output_masks=[None],
-             input_shapes=[shape],
-             output_shapes=[shape])
+             input_shapes=[batch_input_shape],
+             output_shapes=[batch_input_shape])
 
     def get_config(self):
         config = {'batch_input_shape': self.batch_input_shape,
@@ -972,7 +995,8 @@ class InputLayer(Layer):
 
 
 def Input(shape=None, batch_shape=None,
-          name=None, dtype=K.floatx()):
+          name=None, dtype=K.floatx(),
+          tensor=None):
     '''`Input()` is used to instantiate a Keras tensor.
     A Keras tensor is a tensor object from the underlying backend
     (Theano or TensorFlow), which we augment with certain
@@ -1014,14 +1038,15 @@ def Input(shape=None, batch_shape=None,
         model = Model(input=a, output=b)
         ```
     '''
-    if not batch_shape:
+    if not batch_shape and tensor is None:
         assert shape, ('Please provide to Input either a `shape`' +
                        ' or a `batch_shape` argument. Note that ' +
                        '`shape` does not include the batch '
                        'dimension.')
         batch_shape = (None,) + tuple(shape)
     input_layer = InputLayer(batch_input_shape=batch_shape,
-                             name=name, input_dtype=dtype)
+                             name=name, input_dtype=dtype,
+                             input_tensor=tensor)
     # return tensor including _keras_shape and _keras_history
     # note that in this case train_output and test_output are the same pointer.
     outputs = input_layer.inbound_nodes[0].output_tensors
@@ -1055,7 +1080,7 @@ class Merge(Layer):
             a list of layer instances. Must be more
             than one layer/tensor.
         mode: string or lambda/function. If string, must be one
-            of: 'sum', 'mul', 'concat', 'ave', 'cos', 'dot'.
+            of: 'sum', 'mul', 'concat', 'ave', 'cos', 'dot', 'max'.
             If lambda/function, it should take as input a list of tensors
             and return a single tensor.
         concat_axis: integer, axis to use in mode `concat`.
@@ -1130,7 +1155,7 @@ class Merge(Layer):
         as appropriate.
         '''
         if not hasattr(mode, '__call__'):
-            if mode not in {'sum', 'mul', 'concat', 'ave', 'cos', 'dot'}:
+            if mode not in {'sum', 'mul', 'concat', 'ave', 'cos', 'dot', 'max'}:
                 raise Exception('Invalid merge mode: ' + str(mode))
         if type(layers) not in {list, tuple} or len(layers) < 2:
             raise Exception('A Merge should only be applied to a list of '
@@ -1148,7 +1173,7 @@ class Merge(Layer):
                 layer_output_shape = layer_output_shape[tensor_indices[i]]
             input_shapes.append(layer_output_shape)
 
-        if mode in {'sum', 'mul', 'ave', 'cos'}:
+        if mode in {'sum', 'mul', 'ave', 'cos', 'max'}:
             input_shapes_set = set(input_shapes)
             if len(input_shapes_set) > 1:
                 raise Exception('Only layers of same output shape can '
@@ -1161,22 +1186,21 @@ class Merge(Layer):
             shape2 = input_shapes[1]
             n1 = len(shape1)
             n2 = len(shape2)
-            if mode == 'dot':
-                if type(dot_axes) == int:
-                    if dot_axes < 0:
-                        dot_axes = [dot_axes % n1, dot_axes % n2]
-                    else:
-                        dot_axes = [n1 - dot_axes, n2-dot_axes]
-                if type(dot_axes) not in [list, tuple]:
-                    raise Exception('Invalid type for dot_axes - should be a list.')
-                if len(dot_axes) != 2:
-                    raise Exception('Invalid format for dot_axes - should contain two elements.')
-                if type(dot_axes[0]) is not int or type(dot_axes[1]) is not int:
-                    raise Exception('Invalid format for dot_axes - list elements should be "int".')
-                if shape1[dot_axes[0]] != shape2[dot_axes[1]]:
-                    raise Exception('Dimension incompatibility using dot mode: ' +
-                                    '%s != %s. ' % (shape1[dot_axes[0]], shape2[dot_axes[1]]) +
-                                    'Layer shapes: %s, %s' % (shape1, shape2))
+            if type(dot_axes) == int:
+                if dot_axes < 0:
+                    dot_axes = [dot_axes % n1, dot_axes % n2]
+                else:
+                    dot_axes = [n1 - dot_axes, n2-dot_axes]
+            if type(dot_axes) not in [list, tuple]:
+                raise Exception('Invalid type for dot_axes - should be a list.')
+            if len(dot_axes) != 2:
+                raise Exception('Invalid format for dot_axes - should contain two elements.')
+            if type(dot_axes[0]) is not int or type(dot_axes[1]) is not int:
+                raise Exception('Invalid format for dot_axes - list elements should be "int".')
+            if shape1[dot_axes[0]] != shape2[dot_axes[1]]:
+                raise Exception('Dimension incompatibility using dot mode: ' +
+                                '%s != %s. ' % (shape1[dot_axes[0]], shape2[dot_axes[1]]) +
+                                'Layer shapes: %s, %s' % (shape1, shape2))
         elif mode == 'concat':
             reduced_inputs_shapes = [list(shape) for shape in input_shapes]
             shape_set = set()
@@ -1215,7 +1239,11 @@ class Merge(Layer):
             for i in range(1, len(inputs)):
                 s *= inputs[i]
             return s
-
+        elif self.mode == 'max':
+            s = inputs[0]
+            for i in range(1, len(inputs)):
+                s = K.maximum(s, inputs[i])
+            return s
         elif self.mode == 'dot':
             l1 = inputs[0]
             l2 = inputs[1]
@@ -1283,7 +1311,7 @@ class Merge(Layer):
                 output_shape = self._output_shape(input_shape)
                 return output_shape
             elif self._output_shape is not None:
-                return (input_shape[0],) + tuple(self._output_shape)
+                return (input_shape[0][0],) + tuple(self._output_shape)
             else:
                 # TODO: consider shape auto-inference with TF
                 raise Exception('The Merge layer ' + self.name +
@@ -1294,7 +1322,7 @@ class Merge(Layer):
                                 '`output_shape` to Merge.')
         # pre-defined merge modes
         input_shapes = input_shape
-        if self.mode in ['sum', 'mul', 'ave']:
+        if self.mode in ['sum', 'mul', 'ave', 'max']:
             # all tuples in input_shapes should be the same
             return input_shapes[0]
         elif self.mode == 'concat':
@@ -1305,7 +1333,7 @@ class Merge(Layer):
                     break
                 output_shape[self.concat_axis] += shape[self.concat_axis]
             return tuple(output_shape)
-        elif self.mode == 'dot':
+        elif self.mode in ['dot', 'cos']:
             shape1 = list(input_shapes[0])
             shape2 = list(input_shapes[1])
             dot_axes = [a - 1 for a in self.dot_axes]
@@ -1317,11 +1345,9 @@ class Merge(Layer):
             else:
                 shape = tensordot_output.shape
             return (shape1[0],) + shape
-        elif self.mode == 'cos':
-            return (input_shapes[0][0], 1)
 
     def compute_mask(self, inputs, mask=None):
-        if mask is None or not any([m is not None for m in mask]):
+        if mask is None or all([m is None for m in mask]):
             return None
 
         assert hasattr(mask, '__len__') and len(mask) == len(inputs)
@@ -1593,21 +1619,27 @@ class Container(Layer):
                 raise Exception('Output tensors to a ' + cls_name + ' must be '
                                 'Keras tensors. Found: ' + str(x))
         # build self.output_layers:
-        masks = []
         for x in self.outputs:
             layer, node_index, tensor_index = x._keras_history
             self.output_layers.append(layer)
             self.output_layers_node_indices.append(node_index)
             self.output_layers_tensor_indices.append(tensor_index)
 
-            # also fill in the output mask cache
+        # fill in the output mask cache
+        masks = []
+        for x in self.inputs:
+            layer, node_index, tensor_index = x._keras_history
             node = layer.inbound_nodes[node_index]
             mask = node.output_masks[tensor_index]
             masks.append(mask)
-
-        # output mask cache
         mask_cache_key = ','.join([str(id(x)) for x in self.inputs])
         mask_cache_key += '_' + ','.join([str(id(x)) for x in masks])
+        masks = []
+        for x in self.outputs:
+            layer, node_index, tensor_index = x._keras_history
+            node = layer.inbound_nodes[node_index]
+            mask = node.output_masks[tensor_index]
+            masks.append(mask)
         if len(masks) == 1:
             mask = masks[0]
         else:
@@ -2105,6 +2137,8 @@ class Container(Layer):
                         for x, s in zip(output_tensors, shapes):
                             x._keras_shape = s
                             x._uses_learning_phase = uses_learning_phase
+
+                    # update tensor_map
                     for x, y, mask in zip(reference_output_tensors, output_tensors, output_masks):
                         tensor_map[str(id(x))] = (y, mask)
 

keras/engine/training.py

@@ -5,6 +5,7 @@ import warnings
 import copy
 import time
 import numpy as np
+import multiprocessing
 import threading
 try:
     import queue
@@ -205,12 +206,12 @@ def check_loss_and_target_compatibility(targets, losses, output_shapes):
                                 '`sparse_categorical_crossentropy` instead, '
                                 'which does expect integer targets.')
         if loss.__name__ in key_losses and shape[1] is not None and y.shape[1] != shape[1]:
-                raise Exception('A target array with shape ' + str(y.shape) +
-                                ' was passed for an output of shape ' + str(shape) +
-                                ' while using as loss `' + loss.__name__ + '`. '
-                                'This loss expects '
-                                'targets to have the same shape '
-                                'as the output.')
+            raise Exception('A target array with shape ' + str(y.shape) +
+                            ' was passed for an output of shape ' + str(shape) +
+                            ' while using as loss `' + loss.__name__ + '`. '
+                            'This loss expects '
+                            'targets to have the same shape '
+                            'as the output.')
 
 
 def collect_metrics(metrics, output_names):
@@ -395,40 +396,64 @@ def standardize_weights(y, sample_weight=None, class_weight=None,
         return weights
     else:
         if sample_weight_mode is None:
-            return np.ones((y.shape[0],))
+            return np.ones((y.shape[0],), dtype=K.floatx())
         else:
-            return np.ones((y.shape[0], y.shape[1]))
+            return np.ones((y.shape[0], y.shape[1]), dtype=K.floatx())
 
 
 def generator_queue(generator, max_q_size=10,
-                    wait_time=0.05, nb_worker=1):
-    '''Builds a threading queue out of a data generator.
+                    wait_time=0.05, nb_worker=1, pickle_safe=False):
+    '''Builds a queue out of a data generator.
+    If pickle_safe, use a multiprocessing approach. Else, use threading.
     Used in `fit_generator`, `evaluate_generator`, `predict_generator`.
+
     '''
-    q = queue.Queue()
-    _stop = threading.Event()
 
-    def data_generator_task():
-        while not _stop.is_set():
-            try:
-                if q.qsize() < max_q_size:
-                    try:
-                        generator_output = next(generator)
-                    except ValueError:
-                        continue
-                    q.put(generator_output)
-                else:
-                    time.sleep(wait_time)
-            except Exception:
-                _stop.set()
-                raise
+    generator_threads = []
+    if pickle_safe:
+        q = multiprocessing.Queue(maxsize=max_q_size)
+        _stop = multiprocessing.Event()
+    else:
+        q = queue.Queue()
+        _stop = threading.Event()
 
-    generator_threads = [threading.Thread(target=data_generator_task)
-                         for _ in range(nb_worker)]
+    try:
 
-    for thread in generator_threads:
-        thread.daemon = True
-        thread.start()
+        def data_generator_task():
+            while not _stop.is_set():
+                try:
+                    if q.qsize() < max_q_size:
+                        try:
+                            generator_output = next(generator)
+                        except ValueError:
+                            continue
+                        q.put(generator_output)
+                    else:
+                        time.sleep(wait_time)
+                except Exception:
+                    _stop.set()
+                    raise
+
+        for i in range(nb_worker):
+            if pickle_safe:
+                # Reset random seed else all children processes share the same seed
+                np.random.seed()
+                thread = multiprocessing.Process(target=data_generator_task)
+            else:
+                thread = threading.Thread(target=data_generator_task)
+            generator_threads.append(thread)
+            thread.daemon = True
+            thread.start()
+
+    except:
+        _stop.set()
+        if pickle_safe:
+            # Terminate all daemon processes
+            for p in generator_threads:
+                if p.is_alive():
+                    p.terminate()
+            q.close()
+        raise
 
     return q, _stop
 
@@ -485,7 +510,7 @@ class Model(Container):
                                 'it should have one entry per model outputs. '
                                 'The model has ' + str(len(self.outputs)) +
                                 ' outputs, but you passed loss_weights=' +
-                                str(loss))
+                                str(loss_weights))
             loss_weights_list = loss_weights
         else:
             raise Exception('Could not interpret loss_weights argument: ' +
@@ -623,7 +648,7 @@ class Model(Container):
                 if metric == 'accuracy' or metric == 'acc':
                     # custom handling of accuracy (because of class mode duality)
                     output_shape = self.internal_output_shapes[i]
-                    if output_shape[-1] == 1:
+                    if output_shape[-1] == 1 or self.loss_functions[i] == objectives.binary_crossentropy:
                         # case: binary accuracy
                         self.metrics.append(metrics_module.binary_accuracy(y_true, y_pred))
                     elif self.loss_functions[i] == objectives.sparse_categorical_crossentropy:
@@ -858,7 +883,7 @@ class Model(Container):
             if batch_index == 0:
                 for batch_out in batch_outs:
                     shape = (nb_sample,) + batch_out.shape[1:]
-                    outs.append(np.zeros(shape))
+                    outs.append(np.zeros(shape, dtype=K.floatx()))
 
             for i, batch_out in enumerate(batch_outs):
                 outs[i][batch_start:batch_end] = batch_out
@@ -1035,7 +1060,8 @@ class Model(Container):
             split_at = int(len(x[0]) * (1. - validation_split))
             x, val_x = (slice_X(x, 0, split_at), slice_X(x, split_at))
             y, val_y = (slice_X(y, 0, split_at), slice_X(y, split_at))
-            sample_weights, val_sample_weights = (slice_X(sample_weights, 0, split_at), slice_X(sample_weights, split_at))
+            sample_weights, val_sample_weights = (
+                slice_X(sample_weights, 0, split_at), slice_X(sample_weights, split_at))
             self._make_test_function()
             val_f = self.test_function
             if self.uses_learning_phase:
@@ -1255,7 +1281,7 @@ class Model(Container):
     def fit_generator(self, generator, samples_per_epoch, nb_epoch,
                       verbose=1, callbacks=[],
                       validation_data=None, nb_val_samples=None,
-                      class_weight={}, max_q_size=10):
+                      class_weight={}, max_q_size=10, nb_worker=1, pickle_safe=False):
         '''Fits the model on data generated batch-by-batch by
         a Python generator.
         The generator is run in parallel to the model, for efficiency.
@@ -1286,6 +1312,11 @@ class Model(Container):
             class_weight: dictionary mapping class indices to a weight
                 for the class.
             max_q_size: maximum size for the generator queue
+            nb_worker: maximum number of processes to spin up when using process based threading
+            pickle_safe: if True, use process based threading. Note that because
+                this implementation relies on multiprocessing, you should not pass non
+                non picklable arguments to the generator as they can't be passed
+                easily to children processes.
 
         # Returns
             A `History` object.
@@ -1364,7 +1395,8 @@ class Model(Container):
             self.validation_data = None
 
         # start generator thread storing batches into a queue
-        data_gen_queue, _stop = generator_queue(generator, max_q_size=max_q_size)
+        data_gen_queue, _stop = generator_queue(generator, max_q_size=max_q_size, nb_worker=nb_worker,
+                                                pickle_safe=pickle_safe)
 
         callback_model.stop_training = False
         while epoch < nb_epoch:
@@ -1457,10 +1489,12 @@ class Model(Container):
                 break
 
         _stop.set()
+        if pickle_safe:
+            data_gen_queue.close()
         callbacks.on_train_end()
         return self.history
 
-    def evaluate_generator(self, generator, val_samples, max_q_size=10):
+    def evaluate_generator(self, generator, val_samples, max_q_size=10, nb_worker=1, pickle_safe=False):
         '''Evaluates the model on a data generator. The generator should
         return the same kind of data as accepted by `test_on_batch`.
 
@@ -1472,6 +1506,11 @@ class Model(Container):
                 total number of samples to generate from `generator`
                 before returning.
             max_q_size: maximum size for the generator queue
+            nb_worker: maximum number of processes to spin up when using process based threading
+            pickle_safe: if True, use process based threading. Note that because
+                this implementation relies on multiprocessing, you should not pass non
+                non picklable arguments to the generator as they can't be passed
+                easily to children processes.
 
         # Returns
             Scalar test loss (if the model has a single output and no metrics)
@@ -1485,7 +1524,8 @@ class Model(Container):
         wait_time = 0.01
         all_outs = []
         weights = []
-        data_gen_queue, _stop = generator_queue(generator, max_q_size=max_q_size)
+        data_gen_queue, _stop = generator_queue(generator, max_q_size=max_q_size, nb_worker=nb_worker,
+                                                pickle_safe=pickle_safe)
 
         while processed_samples < val_samples:
             generator_output = None
@@ -1529,6 +1569,8 @@ class Model(Container):
             weights.append(nb_samples)
 
         _stop.set()
+        if pickle_safe:
+            data_gen_queue.close()
         if type(outs) is not list:
             return np.average(np.asarray(all_outs),
                               weights=weights)
@@ -1536,10 +1578,10 @@ class Model(Container):
             averages = []
             for i in range(len(outs)):
                 averages.append(np.average([out[i] for out in all_outs],
-                                weights=weights))
+                                           weights=weights))
             return averages
 
-    def predict_generator(self, generator, val_samples, max_q_size=10):
+    def predict_generator(self, generator, val_samples, max_q_size=10, nb_worker=1, pickle_safe=False):
         '''Generates predictions for the input samples from a data generator.
         The generator should return the same kind of data as accepted by
         `predict_on_batch`.
@@ -1549,6 +1591,11 @@ class Model(Container):
             val_samples: total number of samples to generate from `generator`
                 before returning.
             max_q_size: maximum size for the generator queue
+            nb_worker: maximum number of processes to spin up when using process based threading
+            pickle_safe: if True, use process based threading. Note that because
+                this implementation relies on multiprocessing, you should not pass non
+                non picklable arguments to the generator as they can't be passed
+                easily to children processes.
 
         # Returns
             Numpy array(s) of predictions.
@@ -1558,7 +1605,8 @@ class Model(Container):
         processed_samples = 0
         wait_time = 0.01
         all_outs = []
-        data_gen_queue, _stop = generator_queue(generator, max_q_size=max_q_size)
+        data_gen_queue, _stop = generator_queue(generator, max_q_size=max_q_size, nb_worker=nb_worker,
+                                                pickle_safe=pickle_safe)
 
         while processed_samples < val_samples:
             generator_output = None
@@ -1602,7 +1650,7 @@ class Model(Container):
             if len(all_outs) == 0:
                 for out in outs:
                     shape = (val_samples,) + out.shape[1:]
-                    all_outs.append(np.zeros(shape))
+                    all_outs.append(np.zeros(shape, dtype=K.floatx()))
 
             for i, out in enumerate(outs):
                 all_outs[i][processed_samples:(processed_samples + nb_samples)] = out
@@ -1610,6 +1658,8 @@ class Model(Container):
             processed_samples += nb_samples
 
         _stop.set()
+        if pickle_safe:
+            data_gen_queue.close()
         if len(all_outs) == 1:
             return all_outs[0]
         return all_outs

keras/layers/__init__.py

@@ -2,6 +2,8 @@ from __future__ import absolute_import
 from ..engine import Layer, Input, InputLayer, Merge, merge, InputSpec
 from .core import *
 from .convolutional import *
+from .pooling import *
+from .local import *
 from .recurrent import *
 from .normalization import *
 from .embeddings import *

keras/layers/convolutional.py

@@ -4,17 +4,11 @@ from __future__ import absolute_import
 from .. import backend as K
 from .. import activations, initializations, regularizers, constraints
 from ..engine import Layer, InputSpec
+from ..utils.np_utils import conv_output_length
 
-
-def conv_output_length(input_length, filter_size, border_mode, stride):
-    if input_length is None:
-        return None
-    assert border_mode in {'same', 'valid'}
-    if border_mode == 'same':
-        output_length = input_length
-    elif border_mode == 'valid':
-        output_length = input_length - filter_size + 1
-    return (output_length + stride - 1) // stride
+# imports for backwards namespace compatibility
+from .pooling import AveragePooling1D, AveragePooling2D, AveragePooling3D
+from .pooling import MaxPooling1D, MaxPooling2D, MaxPooling3D
 
 
 class Convolution1D(Layer):
@@ -65,7 +59,8 @@ class Convolution1D(Layer):
             (eg. maxnorm, nonneg), applied to the main weights matrix.
         b_constraint: instance of the [constraints](../constraints.md) module,
             applied to the bias.
-        bias: whether to include a bias (i.e. make the layer affine rather than linear).
+        bias: whether to include a bias
+            (i.e. make the layer affine rather than linear).
         input_dim: Number of channels/dimensions in the input.
             Either this argument or the keyword argument `input_shape`must be
             provided when using this layer as the first layer in a model.
@@ -241,7 +236,8 @@ class Convolution2D(Layer):
             It defaults to the `image_dim_ordering` value found in your
             Keras config file at `~/.keras/keras.json`.
             If you never set it, then it will be "th".
-        bias: whether to include a bias (i.e. make the layer affine rather than linear).
+        bias: whether to include a bias
+            (i.e. make the layer affine rather than linear).
 
     # Input shape
         4D tensor with shape:
@@ -383,6 +379,373 @@ class Convolution2D(Layer):
         return dict(list(base_config.items()) + list(config.items()))
 
 
+class AtrousConvolution2D(Convolution2D):
+    '''Atrous Convolution operator for filtering windows of two-dimensional inputs.
+    A.k.a dilated convolution or convolution with holes.
+    When using this layer as the first layer in a model,
+    provide the keyword argument `input_shape`
+    (tuple of integers, does not include the sample axis),
+    e.g. `input_shape=(3, 128, 128)` for 128x128 RGB pictures.
+
+    # Examples
+
+    ```python
+        # apply a 3x3 convolution with atrous rate 2x2 and 64 output filters on a 256x256 image:
+        model = Sequential()
+        model.add(AtrousConvolution2D(64, 3, 3, atrous_rate=(2,2), border_mode='valid', input_shape=(3, 256, 256)))
+        # now the actual kernel size is dilated from 3x3 to 5x5 (3+(3-1)*(2-1)=5)
+        # thus model.output_shape == (None, 64, 252, 252)
+    ```
+
+    # Arguments
+        nb_filter: Number of convolution filters to use.
+        nb_row: Number of rows in the convolution kernel.
+        nb_col: Number of columns in the convolution kernel.
+        init: name of initialization function for the weights of the layer
+            (see [initializations](../initializations.md)), or alternatively,
+            Theano function to use for weights initialization.
+            This parameter is only relevant if you don't pass
+            a `weights` argument.
+        activation: name of activation function to use
+            (see [activations](../activations.md)),
+            or alternatively, elementwise Theano function.
+            If you don't specify anything, no activation is applied
+            (ie. "linear" activation: a(x) = x).
+        weights: list of numpy arrays to set as initial weights.
+        border_mode: 'valid' or 'same'.
+        subsample: tuple of length 2. Factor by which to subsample output.
+            Also called strides elsewhere.
+        atrous_rate: tuple of length 2. Factor for kernel dilation.
+            Also called filter_dilation elsewhere.
+        W_regularizer: instance of [WeightRegularizer](../regularizers.md)
+            (eg. L1 or L2 regularization), applied to the main weights matrix.
+        b_regularizer: instance of [WeightRegularizer](../regularizers.md),
+            applied to the bias.
+        activity_regularizer: instance of [ActivityRegularizer](../regularizers.md),
+            applied to the network output.
+        W_constraint: instance of the [constraints](../constraints.md) module
+            (eg. maxnorm, nonneg), applied to the main weights matrix.
+        b_constraint: instance of the [constraints](../constraints.md) module,
+            applied to the bias.
+        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
+            (the depth) is at index 1, in 'tf' mode is it at index 3.
+            It defaults to the `image_dim_ordering` value found in your
+            Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be "th".
+        bias: whether to include a bias (i.e. make the layer affine rather than linear).
+
+    # Input shape
+        4D tensor with shape:
+        `(samples, channels, rows, cols)` if dim_ordering='th'
+        or 4D tensor with shape:
+        `(samples, rows, cols, channels)` if dim_ordering='tf'.
+
+    # Output shape
+        4D tensor with shape:
+        `(samples, nb_filter, new_rows, new_cols)` if dim_ordering='th'
+        or 4D tensor with shape:
+        `(samples, new_rows, new_cols, nb_filter)` if dim_ordering='tf'.
+        `rows` and `cols` values might have changed due to padding.
+
+    # References
+        - [Multi-Scale Context Aggregation by Dilated Convolutions](https://arxiv.org/abs/1511.07122)
+    '''
+    def __init__(self, nb_filter, nb_row, nb_col,
+                 init='glorot_uniform', activation='linear', weights=None,
+                 border_mode='valid', subsample=(1, 1),
+                 atrous_rate=(1, 1), dim_ordering=K.image_dim_ordering(),
+                 W_regularizer=None, b_regularizer=None, activity_regularizer=None,
+                 W_constraint=None, b_constraint=None,
+                 bias=True, **kwargs):
+
+        if border_mode not in {'valid', 'same'}:
+            raise Exception('Invalid border mode for AtrousConv2D:', border_mode)
+
+        self.atrous_rate = tuple(atrous_rate)
+
+        super(AtrousConvolution2D, self).__init__(nb_filter, nb_row, nb_col,
+                                                  init=init, activation=activation,
+                                                  weights=weights, border_mode=border_mode,
+                                                  subsample=subsample, dim_ordering=dim_ordering,
+                                                  W_regularizer=W_regularizer, b_regularizer=b_regularizer,
+                                                  activity_regularizer=activity_regularizer,
+                                                  W_constraint=W_constraint, b_constraint=b_constraint,
+                                                  bias=bias, **kwargs)
+
+    def get_output_shape_for(self, input_shape):
+        if self.dim_ordering == 'th':
+            rows = input_shape[2]
+            cols = input_shape[3]
+        elif self.dim_ordering == 'tf':
+            rows = input_shape[1]
+            cols = input_shape[2]
+        else:
+            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+
+        rows = conv_output_length(rows, self.nb_row, self.border_mode,
+                                  self.subsample[0], dilation=self.atrous_rate[0])
+        cols = conv_output_length(cols, self.nb_col, self.border_mode,
+                                  self.subsample[1], dilation=self.atrous_rate[1])
+
+        if self.dim_ordering == 'th':
+            return (input_shape[0], self.nb_filter, rows, cols)
+        elif self.dim_ordering == 'tf':
+            return (input_shape[0], rows, cols, self.nb_filter)
+        else:
+            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+
+    def call(self, x, mask=None):
+        output = K.conv2d(x, self.W, strides=self.subsample,
+                          border_mode=self.border_mode,
+                          dim_ordering=self.dim_ordering,
+                          filter_shape=self.W_shape,
+                          filter_dilation=self.atrous_rate)
+        if self.bias:
+            if self.dim_ordering == 'th':
+                output += K.reshape(self.b, (1, self.nb_filter, 1, 1))
+            elif self.dim_ordering == 'tf':
+                output += K.reshape(self.b, (1, 1, 1, self.nb_filter))
+            else:
+                raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+        output = self.activation(output)
+        return output
+
+    def get_config(self):
+        config = {'atrous_rate': self.atrous_rate}
+        base_config = super(AtrousConvolution2D, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
+
+
+class SeparableConvolution2D(Layer):
+    '''Separable convolution operator for 2D inputs.
+
+    Separable convolutions consist in first performing
+    a depthwise spatial convolution
+    (which acts on each input channel separately)
+    followed by a pointwise convolution which mixes together the resulting
+    output channels. The `depth_multiplier` argument controls how many
+    output channels are generated per input channel in the depthwise step.
+
+    Intuitively, separable convolutions can be understood as
+    a way to factorize a convolution kernel into two smaller kernels,
+    or as an extreme version of an Inception block.
+
+    When using this layer as the first layer in a model,
+    provide the keyword argument `input_shape`
+    (tuple of integers, does not include the sample axis),
+    e.g. `input_shape=(3, 128, 128)` for 128x128 RGB pictures.
+
+    # Arguments
+        nb_filter: Number of convolution filters to use.
+        nb_row: Number of rows in the convolution kernel.
+        nb_col: Number of columns in the convolution kernel.
+        init: name of initialization function for the weights of the layer
+            (see [initializations](../initializations.md)), or alternatively,
+            Theano function to use for weights initialization.
+            This parameter is only relevant if you don't pass
+            a `weights` argument.
+        activation: name of activation function to use
+            (see [activations](../activations.md)),
+            or alternatively, elementwise Theano function.
+            If you don't specify anything, no activation is applied
+            (ie. "linear" activation: a(x) = x).
+        weights: list of numpy arrays to set as initial weights.
+        border_mode: 'valid' or 'same'.
+        subsample: tuple of length 2. Factor by which to subsample output.
+            Also called strides elsewhere.
+        depth_multiplier: how many output channel to use per input channel
+            for the depthwise convolution step.
+        atrous_rate: tuple of length 2. Factor for kernel dilation.
+            Also called filter_dilation elsewhere.
+        depthwise_regularizer: instance of [WeightRegularizer](../regularizers.md)
+            (eg. L1 or L2 regularization), applied to the depthwise weights matrix.
+        pointwise_regularizer: instance of [WeightRegularizer](../regularizers.md)
+            (eg. L1 or L2 regularization), applied to the pointwise weights matrix.
+        b_regularizer: instance of [WeightRegularizer](../regularizers.md),
+            applied to the bias.
+        activity_regularizer: instance of [ActivityRegularizer](../regularizers.md),
+            applied to the network output.
+        depthwise_constraint: instance of the [constraints](../constraints.md) module
+            (eg. maxnorm, nonneg), applied to the depthwise weights matrix.
+        pointwise_constraint: instance of the [constraints](../constraints.md) module
+            (eg. maxnorm, nonneg), applied to the pointwise weights matrix.
+        b_constraint: instance of the [constraints](../constraints.md) module,
+            applied to the bias.
+        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
+            (the depth) is at index 1, in 'tf' mode is it at index 3.
+            It defaults to the `image_dim_ordering` value found in your
+            Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be "th".
+        bias: whether to include a bias
+            (i.e. make the layer affine rather than linear).
+
+    # Input shape
+        4D tensor with shape:
+        `(samples, channels, rows, cols)` if dim_ordering='th'
+        or 4D tensor with shape:
+        `(samples, rows, cols, channels)` if dim_ordering='tf'.
+
+    # Output shape
+        4D tensor with shape:
+        `(samples, nb_filter, new_rows, new_cols)` if dim_ordering='th'
+        or 4D tensor with shape:
+        `(samples, new_rows, new_cols, nb_filter)` if dim_ordering='tf'.
+        `rows` and `cols` values might have changed due to padding.
+    '''
+    def __init__(self, nb_filter, nb_row, nb_col,
+                 init='glorot_uniform', activation='linear', weights=None,
+                 border_mode='valid', subsample=(1, 1),
+                 depth_multiplier=1, dim_ordering=K.image_dim_ordering(),
+                 depthwise_regularizer=None, pointwise_regularizer=None,
+                 b_regularizer=None, activity_regularizer=None,
+                 depthwise_constraint=None, pointwise_constraint=None,
+                 b_constraint=None,
+                 bias=True, **kwargs):
+
+        if K._BACKEND != 'tensorflow':
+            raise Exception('SeparableConv2D is only available '
+                            'with TensorFlow for the time being.')
+
+        if border_mode not in {'valid', 'same'}:
+            raise Exception('Invalid border mode for SeparableConv2D:', border_mode)
+
+        if border_mode not in {'valid', 'same'}:
+            raise Exception('Invalid border mode for SeparableConv2D:', border_mode)
+        self.nb_filter = nb_filter
+        self.nb_row = nb_row
+        self.nb_col = nb_col
+        self.init = initializations.get(init, dim_ordering=dim_ordering)
+        self.activation = activations.get(activation)
+        assert border_mode in {'valid', 'same'}, 'border_mode must be in {valid, same}'
+        self.border_mode = border_mode
+        self.subsample = tuple(subsample)
+        self.depth_multiplier = depth_multiplier
+        assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
+        self.dim_ordering = dim_ordering
+
+        self.depthwise_regularizer = regularizers.get(depthwise_regularizer)
+        self.pointwise_regularizer = regularizers.get(pointwise_regularizer)
+        self.b_regularizer = regularizers.get(b_regularizer)
+        self.activity_regularizer = regularizers.get(activity_regularizer)
+
+        self.depthwise_constraint = constraints.get(depthwise_constraint)
+        self.pointwise_constraint = constraints.get(pointwise_constraint)
+        self.b_constraint = constraints.get(b_constraint)
+
+        self.bias = bias
+        self.input_spec = [InputSpec(ndim=4)]
+        self.initial_weights = weights
+        super(SeparableConvolution2D, self).__init__(**kwargs)
+
+    def build(self, input_shape):
+        if self.dim_ordering == 'th':
+            stack_size = input_shape[1]
+            depthwise_shape = (self.depth_multiplier, stack_size, self.nb_row, self.nb_col)
+            pointwise_shape = (self.nb_filter, self.depth_multiplier * stack_size, 1, 1)
+        elif self.dim_ordering == 'tf':
+            stack_size = input_shape[3]
+            depthwise_shape = (self.nb_row, self.nb_col, stack_size, self.depth_multiplier)
+            pointwise_shape = (1, 1, self.depth_multiplier * stack_size, self.nb_filter)
+        else:
+            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+        self.depthwise_kernel = self.init(depthwise_shape,
+                                          name='{}_depthwise_kernel'.format(self.name))
+        self.pointwise_kernel = self.init(pointwise_shape,
+                                          name='{}_pointwise_kernel'.format(self.name))
+        if self.bias:
+            self.b = K.zeros((self.nb_filter,), name='{}_b'.format(self.name))
+            self.trainable_weights = [self.depthwise_kernel,
+                                      self.pointwise_kernel,
+                                      self.b]
+        else:
+            self.trainable_weights = [self.depthwise_kernel,
+                                      self.pointwise_kernel]
+        self.regularizers = []
+        if self.depthwise_regularizer:
+            self.depthwise_regularizer.set_param(self.depthwise_kernel)
+            self.regularizers.append(self.depthwise_regularizer)
+        if self.pointwise_regularizer:
+            self.pointwise_regularizer.set_param(self.pointwise_kernel)
+            self.regularizers.append(self.pointwise_regularizer)
+        if self.bias and self.b_regularizer:
+            self.b_regularizer.set_param(self.b)
+            self.regularizers.append(self.b_regularizer)
+        if self.activity_regularizer:
+            self.activity_regularizer.set_layer(self)
+            self.regularizers.append(self.activity_regularizer)
+
+        self.constraints = {}
+        if self.depthwise_constraint:
+            self.constraints[self.depthwise_kernel] = self.depthwise_constraint
+        if self.pointwise_constraint:
+            self.constraints[self.pointwise_kernel] = self.pointwise_constraint
+        if self.bias and self.b_constraint:
+            self.constraints[self.b] = self.b_constraint
+
+        if self.initial_weights is not None:
+            self.set_weights(self.initial_weights)
+            del self.initial_weights
+
+    def get_output_shape_for(self, input_shape):
+        if self.dim_ordering == 'th':
+            rows = input_shape[2]
+            cols = input_shape[3]
+        elif self.dim_ordering == 'tf':
+            rows = input_shape[1]
+            cols = input_shape[2]
+        else:
+            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+
+        rows = conv_output_length(rows, self.nb_row,
+                                  self.border_mode, self.subsample[0])
+        cols = conv_output_length(cols, self.nb_col,
+                                  self.border_mode, self.subsample[1])
+
+        if self.dim_ordering == 'th':
+            return (input_shape[0], self.nb_filter, rows, cols)
+        elif self.dim_ordering == 'tf':
+            return (input_shape[0], rows, cols, self.nb_filter)
+        else:
+            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+
+    def call(self, x, mask=None):
+        output = K.separable_conv2d(x, self.depthwise_kernel,
+                                    self.pointwise_kernel,
+                                    strides=self.subsample,
+                                    border_mode=self.border_mode,
+                                    dim_ordering=self.dim_ordering)
+        if self.bias:
+            if self.dim_ordering == 'th':
+                output += K.reshape(self.b, (1, self.nb_filter, 1, 1))
+            elif self.dim_ordering == 'tf':
+                output += K.reshape(self.b, (1, 1, 1, self.nb_filter))
+            else:
+                raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+        output = self.activation(output)
+        return output
+
+    def get_config(self):
+        config = {'nb_filter': self.nb_filter,
+                  'nb_row': self.nb_row,
+                  'nb_col': self.nb_col,
+                  'init': self.init.__name__,
+                  'activation': self.activation.__name__,
+                  'border_mode': self.border_mode,
+                  'subsample': self.subsample,
+                  'depth_multiplier': self.depth_multiplier,
+                  'dim_ordering': self.dim_ordering,
+                  'depthwise_regularizer': self.depthwise_regularizer.get_config() if self.depthwise_regularizer else None,
+                  'pointwise_regularizer': self.depthwise_regularizer.get_config() if self.depthwise_regularizer else None,
+                  'b_regularizer': self.b_regularizer.get_config() if self.b_regularizer else None,
+                  'activity_regularizer': self.activity_regularizer.get_config() if self.activity_regularizer else None,
+                  'depthwise_constraint': self.depthwise_constraint.get_config() if self.depthwise_constraint else None,
+                  'pointwise_constraint': self.pointwise_constraint.get_config() if self.pointwise_constraint else None,
+                  'b_constraint': self.b_constraint.get_config() if self.b_constraint else None,
+                  'bias': self.bias}
+        base_config = super(SeparableConvolution2D, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
+
+
 class Convolution3D(Layer):
     '''Convolution operator for filtering windows of three-dimensional inputs.
     When using this layer as the first layer in a model,
@@ -390,8 +753,6 @@ class Convolution3D(Layer):
     (tuple of integers, does not include the sample axis),
     e.g. `input_shape=(3, 10, 128, 128)` for 10 frames of 128x128 RGB pictures.
 
-    Note: this layer will only work with Theano for the time being.
-
     # Arguments
         nb_filter: Number of convolution filters to use.
         kernel_dim1: Length of the first dimension in the convolution kernel.
@@ -449,9 +810,6 @@ class Convolution3D(Layer):
                  W_regularizer=None, b_regularizer=None, activity_regularizer=None,
                  W_constraint=None, b_constraint=None,
                  bias=True, **kwargs):
-        if K._BACKEND != 'theano':
-            raise Exception(self.__class__.__name__ +
-                            ' is currently only working with Theano backend.')
         if border_mode not in {'valid', 'same'}:
             raise Exception('Invalid border mode for Convolution3D:', border_mode)
         self.nb_filter = nb_filter
@@ -586,401 +944,6 @@ class Convolution3D(Layer):
         return dict(list(base_config.items()) + list(config.items()))
 
 
-class _Pooling1D(Layer):
-    '''Abstract class for different pooling 1D layers.
-    '''
-    input_dim = 3
-
-    def __init__(self, pool_length=2, stride=None,
-                 border_mode='valid', **kwargs):
-        super(_Pooling1D, self).__init__(**kwargs)
-        if stride is None:
-            stride = pool_length
-        self.pool_length = pool_length
-        self.stride = stride
-        self.st = (self.stride, 1)
-        self.pool_size = (pool_length, 1)
-        assert border_mode in {'valid', 'same'}, 'border_mode must be in {valid, same}'
-        self.border_mode = border_mode
-        self.input_spec = [InputSpec(ndim=3)]
-
-    def get_output_shape_for(self, input_shape):
-        length = conv_output_length(input_shape[1], self.pool_length,
-                                    self.border_mode, self.stride)
-        return (input_shape[0], length, input_shape[2])
-
-    def _pooling_function(self, back_end, inputs, pool_size, strides,
-                          border_mode, dim_ordering):
-        raise NotImplementedError
-
-    def call(self, x, mask=None):
-        x = K.expand_dims(x, -1)   # add dummy last dimension
-        x = K.permute_dimensions(x, (0, 2, 1, 3))
-        output = self._pooling_function(inputs=x, pool_size=self.pool_size,
-                                        strides=self.st,
-                                        border_mode=self.border_mode,
-                                        dim_ordering='th')
-        output = K.permute_dimensions(output, (0, 2, 1, 3))
-        return K.squeeze(output, 3)  # remove dummy last dimension
-
-    def get_config(self):
-        config = {'stride': self.stride,
-                  'pool_length': self.pool_length,
-                  'border_mode': self.border_mode}
-        base_config = super(_Pooling1D, self).get_config()
-        return dict(list(base_config.items()) + list(config.items()))
-
-
-class MaxPooling1D(_Pooling1D):
-    '''Max pooling operation for temporal data.
-
-    # Input shape
-        3D tensor with shape: `(samples, steps, features)`.
-
-    # Output shape
-        3D tensor with shape: `(samples, downsampled_steps, features)`.
-
-    # Arguments
-        pool_length: factor by which to downscale. 2 will halve the input.
-        stride: integer or None. Stride value.
-        border_mode: 'valid' or 'same'.
-            Note: 'same' will only work with TensorFlow for the time being.
-    '''
-
-    def __init__(self, pool_length=2, stride=None,
-                 border_mode='valid', **kwargs):
-        super(MaxPooling1D, self).__init__(pool_length, stride,
-                                           border_mode, **kwargs)
-
-    def _pooling_function(self, inputs, pool_size, strides,
-                          border_mode, dim_ordering):
-        output = K.pool2d(inputs, pool_size, strides,
-                          border_mode, dim_ordering, pool_mode='max')
-        return output
-
-
-class AveragePooling1D(_Pooling1D):
-    '''Average pooling for temporal data.
-
-    # Arguments
-        pool_length: factor by which to downscale. 2 will halve the input.
-        stride: integer or None. Stride value.
-        border_mode: 'valid' or 'same'.
-            Note: 'same' will only work with TensorFlow for the time being.
-
-    # Input shape
-        3D tensor with shape: `(samples, steps, features)`.
-
-    # Output shape
-        3D tensor with shape: `(samples, downsampled_steps, features)`.
-    '''
-
-    def __init__(self, pool_length=2, stride=None,
-                 border_mode='valid', **kwargs):
-        super(AveragePooling1D, self).__init__(pool_length, stride,
-                                               border_mode, **kwargs)
-
-    def _pooling_function(self, inputs, pool_size, strides,
-                          border_mode, dim_ordering):
-        output = K.pool2d(inputs, pool_size, strides,
-                          border_mode, dim_ordering, pool_mode='avg')
-        return output
-
-
-class _Pooling2D(Layer):
-    '''Abstract class for different pooling 2D layers.
-    '''
-
-    def __init__(self, pool_size=(2, 2), strides=None, border_mode='valid',
-                 dim_ordering=K.image_dim_ordering(), **kwargs):
-        super(_Pooling2D, self).__init__(**kwargs)
-        self.pool_size = tuple(pool_size)
-        if strides is None:
-            strides = self.pool_size
-        self.strides = tuple(strides)
-        assert border_mode in {'valid', 'same'}, 'border_mode must be in {valid, same}'
-        self.border_mode = border_mode
-        assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
-        self.dim_ordering = dim_ordering
-        self.input_spec = [InputSpec(ndim=4)]
-
-    def get_output_shape_for(self, input_shape):
-        if self.dim_ordering == 'th':
-            rows = input_shape[2]
-            cols = input_shape[3]
-        elif self.dim_ordering == 'tf':
-            rows = input_shape[1]
-            cols = input_shape[2]
-        else:
-            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
-
-        rows = conv_output_length(rows, self.pool_size[0],
-                                  self.border_mode, self.strides[0])
-        cols = conv_output_length(cols, self.pool_size[1],
-                                  self.border_mode, self.strides[1])
-
-        if self.dim_ordering == 'th':
-            return (input_shape[0], input_shape[1], rows, cols)
-        elif self.dim_ordering == 'tf':
-            return (input_shape[0], rows, cols, input_shape[3])
-        else:
-            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
-
-    def _pooling_function(self, inputs, pool_size, strides,
-                          border_mode, dim_ordering):
-        raise NotImplementedError
-
-    def call(self, x, mask=None):
-        output = self._pooling_function(inputs=x, pool_size=self.pool_size,
-                                        strides=self.strides,
-                                        border_mode=self.border_mode,
-                                        dim_ordering=self.dim_ordering)
-        return output
-
-    def get_config(self):
-        config = {'pool_size': self.pool_size,
-                  'border_mode': self.border_mode,
-                  'strides': self.strides,
-                  'dim_ordering': self.dim_ordering}
-        base_config = super(_Pooling2D, self).get_config()
-        return dict(list(base_config.items()) + list(config.items()))
-
-
-class MaxPooling2D(_Pooling2D):
-    '''Max pooling operation for spatial data.
-
-    # Arguments
-        pool_size: tuple of 2 integers,
-            factors by which to downscale (vertical, horizontal).
-            (2, 2) will halve the image in each dimension.
-        strides: tuple of 2 integers, or None. Strides values.
-        border_mode: 'valid' or 'same'.
-            Note: 'same' will only work with TensorFlow for the time being.
-        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
-            (the depth) is at index 1, in 'tf' mode is it at index 3.
-            It defaults to the `image_dim_ordering` value found in your
-            Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
-
-    # Input shape
-        4D tensor with shape:
-        `(samples, channels, rows, cols)` if dim_ordering='th'
-        or 4D tensor with shape:
-        `(samples, rows, cols, channels)` if dim_ordering='tf'.
-
-    # Output shape
-        4D tensor with shape:
-        `(nb_samples, channels, pooled_rows, pooled_cols)` if dim_ordering='th'
-        or 4D tensor with shape:
-        `(samples, pooled_rows, pooled_cols, channels)` if dim_ordering='tf'.
-    '''
-
-    def __init__(self, pool_size=(2, 2), strides=None, border_mode='valid',
-                 dim_ordering=K.image_dim_ordering(), **kwargs):
-        super(MaxPooling2D, self).__init__(pool_size, strides, border_mode,
-                                           dim_ordering, **kwargs)
-
-    def _pooling_function(self, inputs, pool_size, strides,
-                          border_mode, dim_ordering):
-        output = K.pool2d(inputs, pool_size, strides,
-                          border_mode, dim_ordering, pool_mode='max')
-        return output
-
-
-class AveragePooling2D(_Pooling2D):
-    '''Average pooling operation for spatial data.
-
-    # Arguments
-        pool_size: tuple of 2 integers,
-            factors by which to downscale (vertical, horizontal).
-            (2, 2) will halve the image in each dimension.
-        strides: tuple of 2 integers, or None. Strides values.
-        border_mode: 'valid' or 'same'.
-            Note: 'same' will only work with TensorFlow for the time being.
-        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
-            (the depth) is at index 1, in 'tf' mode is it at index 3.
-            It defaults to the `image_dim_ordering` value found in your
-            Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
-
-    # Input shape
-        4D tensor with shape:
-        `(samples, channels, rows, cols)` if dim_ordering='th'
-        or 4D tensor with shape:
-        `(samples, rows, cols, channels)` if dim_ordering='tf'.
-
-    # Output shape
-        4D tensor with shape:
-        `(nb_samples, channels, pooled_rows, pooled_cols)` if dim_ordering='th'
-        or 4D tensor with shape:
-        `(samples, pooled_rows, pooled_cols, channels)` if dim_ordering='tf'.
-    '''
-
-    def __init__(self, pool_size=(2, 2), strides=None, border_mode='valid',
-                 dim_ordering=K.image_dim_ordering(), **kwargs):
-        super(AveragePooling2D, self).__init__(pool_size, strides, border_mode,
-                                               dim_ordering, **kwargs)
-
-    def _pooling_function(self, inputs, pool_size, strides,
-                          border_mode, dim_ordering):
-        output = K.pool2d(inputs, pool_size, strides,
-                          border_mode, dim_ordering, pool_mode='avg')
-        return output
-
-
-class _Pooling3D(Layer):
-    '''Abstract class for different pooling 3D layers.
-    '''
-
-    def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode='valid',
-                 dim_ordering=K.image_dim_ordering(), **kwargs):
-        super(_Pooling3D, self).__init__(**kwargs)
-        self.pool_size = tuple(pool_size)
-        if strides is None:
-            strides = self.pool_size
-        self.strides = tuple(strides)
-        assert border_mode in {'valid', 'same'}, 'border_mode must be in {valid, same}'
-        self.border_mode = border_mode
-        assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
-        self.dim_ordering = dim_ordering
-        self.input_spec = [InputSpec(ndim=5)]
-
-    def get_output_shape_for(self, input_shape):
-        if self.dim_ordering == 'th':
-            len_dim1 = input_shape[2]
-            len_dim2 = input_shape[3]
-            len_dim3 = input_shape[4]
-        elif self.dim_ordering == 'tf':
-            len_dim1 = input_shape[1]
-            len_dim2 = input_shape[2]
-            len_dim3 = input_shape[3]
-        else:
-            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
-
-        len_dim1 = conv_output_length(len_dim1, self.pool_size[0],
-                                      self.border_mode, self.strides[0])
-        len_dim2 = conv_output_length(len_dim2, self.pool_size[1],
-                                      self.border_mode, self.strides[1])
-        len_dim3 = conv_output_length(len_dim3, self.pool_size[2],
-                                      self.border_mode, self.strides[2])
-
-        if self.dim_ordering == 'th':
-            return (input_shape[0], input_shape[1], len_dim1, len_dim2, len_dim3)
-        elif self.dim_ordering == 'tf':
-            return (input_shape[0], len_dim1, len_dim2, len_dim3, input_shape[4])
-        else:
-            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
-
-    def _pooling_function(self, inputs, pool_size, strides,
-                          border_mode, dim_ordering):
-        raise NotImplementedError
-
-    def call(self, x, mask=None):
-        output = self._pooling_function(inputs=x, pool_size=self.pool_size,
-                                        strides=self.strides,
-                                        border_mode=self.border_mode,
-                                        dim_ordering=self.dim_ordering)
-        return output
-
-    def get_config(self):
-        config = {'pool_size': self.pool_size,
-                  'border_mode': self.border_mode,
-                  'strides': self.strides,
-                  'dim_ordering': self.dim_ordering}
-        base_config = super(_Pooling3D, self).get_config()
-        return dict(list(base_config.items()) + list(config.items()))
-
-
-class MaxPooling3D(_Pooling3D):
-    '''Max pooling operation for 3D data (spatial or spatio-temporal).
-
-    Note: this layer will only work with Theano for the time being.
-
-    # Arguments
-        pool_size: tuple of 3 integers,
-            factors by which to downscale (dim1, dim2, dim3).
-            (2, 2, 2) will halve the size of the 3D input in each dimension.
-        strides: tuple of 3 integers, or None. Strides values.
-        border_mode: 'valid' or 'same'.
-        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
-            (the depth) is at index 1, in 'tf' mode is it at index 4.
-            It defaults to the `image_dim_ordering` value found in your
-            Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
-
-    # Input shape
-        5D tensor with shape:
-        `(samples, channels, len_pool_dim1, len_pool_dim2, len_pool_dim3)` if dim_ordering='th'
-        or 5D tensor with shape:
-        `(samples, len_pool_dim1, len_pool_dim2, len_pool_dim3, channels)` if dim_ordering='tf'.
-
-    # Output shape
-        5D tensor with shape:
-        `(nb_samples, channels, pooled_dim1, pooled_dim2, pooled_dim3)` if dim_ordering='th'
-        or 5D tensor with shape:
-        `(samples, pooled_dim1, pooled_dim2, pooled_dim3, channels)` if dim_ordering='tf'.
-    '''
-
-    def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode='valid',
-                 dim_ordering=K.image_dim_ordering(), **kwargs):
-        if K._BACKEND != 'theano':
-            raise Exception(self.__class__.__name__ +
-                            ' is currently only working with Theano backend.')
-        super(MaxPooling3D, self).__init__(pool_size, strides, border_mode,
-                                           dim_ordering, **kwargs)
-
-    def _pooling_function(self, inputs, pool_size, strides,
-                          border_mode, dim_ordering):
-        output = K.pool3d(inputs, pool_size, strides,
-                          border_mode, dim_ordering, pool_mode='max')
-        return output
-
-
-class AveragePooling3D(_Pooling3D):
-    '''Average pooling operation for 3D data (spatial or spatio-temporal).
-
-    Note: this layer will only work with Theano for the time being.
-
-    # Arguments
-        pool_size: tuple of 3 integers,
-            factors by which to downscale (dim1, dim2, dim3).
-            (2, 2, 2) will halve the size of the 3D input in each dimension.
-        strides: tuple of 3 integers, or None. Strides values.
-        border_mode: 'valid' or 'same'.
-        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
-            (the depth) is at index 1, in 'tf' mode is it at index 4.
-            It defaults to the `image_dim_ordering` value found in your
-            Keras config file at `~/.keras/keras.json`.
-            If you never set it, then it will be "th".
-
-    # Input shape
-        5D tensor with shape:
-        `(samples, channels, len_pool_dim1, len_pool_dim2, len_pool_dim3)` if dim_ordering='th'
-        or 5D tensor with shape:
-        `(samples, len_pool_dim1, len_pool_dim2, len_pool_dim3, channels)` if dim_ordering='tf'.
-
-    # Output shape
-        5D tensor with shape:
-        `(nb_samples, channels, pooled_dim1, pooled_dim2, pooled_dim3)` if dim_ordering='th'
-        or 5D tensor with shape:
-        `(samples, pooled_dim1, pooled_dim2, pooled_dim3, channels)` if dim_ordering='tf'.
-    '''
-
-    def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode='valid',
-                 dim_ordering=K.image_dim_ordering(), **kwargs):
-        if K._BACKEND != 'theano':
-            raise Exception(self.__class__.__name__ +
-                            ' is currently only working with Theano backend.')
-        super(AveragePooling3D, self).__init__(pool_size, strides, border_mode,
-                                               dim_ordering, **kwargs)
-
-    def _pooling_function(self, inputs, pool_size, strides,
-                          border_mode, dim_ordering):
-        output = K.pool3d(inputs, pool_size, strides,
-                          border_mode, dim_ordering, pool_mode='avg')
-        return output
-
-
 class UpSampling1D(Layer):
     '''Repeat each temporal step `length` times along the time axis.
 
@@ -1291,3 +1254,12 @@ class ZeroPadding3D(Layer):
         config = {'padding': self.padding}
         base_config = super(ZeroPadding3D, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
+
+
+# Aliases
+
+Conv1D = Convolution1D
+Conv2D = Convolution2D
+Conv3D = Convolution3D
+AtrousConv2D = AtrousConvolution2D
+SeparableConv2D = SeparableConvolution2D

keras/layers/core.py

@@ -387,7 +387,14 @@ class Lambda(Layer):
         function: The function to be evaluated.
             Takes one argument: the output of previous layer
         output_shape: Expected output shape from function.
-            Could be a tuple or a function of the shape of the input
+            Can be a tuple or function.
+            If a tuple, it only specifies the first dimension onward; 
+                 sample dimension is assumed either the same as the input:
+                 `output_shape = (input_shape[0], ) + output_shape`
+                 or, the input is `None` and the sample dimension is also `None`:
+                 `output_shape = (None, ) + output_shape`
+            If a function, it specifies the entire shape as a function of 
+                 the input shape: `output_shape = f(input_shape)`
         arguments: optional dictionary of keyword arguments to be passed
             to the function.
 
@@ -402,7 +409,7 @@ class Lambda(Layer):
     def __init__(self, function, output_shape=None, arguments={}, **kwargs):
         self.function = function
         self.arguments = arguments
-        self.supports_masking = True
+        self.supports_masking = False
 
         if output_shape is None:
             self._output_shape = None

keras/layers/local.py

@@ -0,0 +1,421 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+from keras import backend as K
+from keras.layers import activations, initializations, regularizers, constraints
+from keras.engine import Layer, InputSpec
+from ..utils.np_utils import conv_output_length
+
+
+class LocallyConnected1D(Layer):
+    '''LocallyConnected1D layer works almost the same as Convolution1D layer,
+    except that weights are unshared, that is, a different set of filters is
+    applied at each different patch of the input. When using this layer as the
+    first layer in a model, either provide the keyword argument `input_dim`
+    (int, e.g. 128 for sequences of 128-dimensional vectors), or `input_shape`
+    (tuple of integers, e.g. (10, 128) for sequences of 10 vectors of
+    128-dimensional vectors). Also, you will need to fix shape of the previous
+    layer, since the weights can only be defined with determined output shape.
+
+    # Example
+    ```python
+        # apply a unshared weight convolution 1d of length 3 to a sequence with
+        # 10 timesteps, with 64 output filters
+        model = Sequential()
+        model.add(LocallyConnected1D(64, 3, input_shape=(10, 32)))
+        # now model.output_shape == (None, 8, 64)
+        # add a new conv1d on top
+        model.add(LocallyConnected1D(32, 3))
+        # now model.output_shape == (None, 6, 32)
+    ```
+    # Arguments
+        nb_filter: Dimensionality of the output.
+        filter_length: The extension (spatial or temporal) of each filter.
+        init: name of initialization function for the weights of the layer
+            (see [initializations](../initializations.md)),
+            or alternatively, Theano function to use for weights initialization.
+            This parameter is only relevant if you don't pass a `weights` argument.
+        activation: name of activation function to use
+            (see [activations](../activations.md)),
+            or alternatively, elementwise Theano function.
+            If you don't specify anything, no activation is applied
+            (ie. "linear" activation: a(x) = x).
+        weights: list of numpy arrays to set as initial weights.
+        border_mode: Only support 'valid'. Please make good use of
+            ZeroPadding1D to achieve same output length.
+        subsample_length: factor by which to subsample output.
+        W_regularizer: instance of [WeightRegularizer](../regularizers.md)
+            (eg. L1 or L2 regularization), applied to the main weights matrix.
+        b_regularizer: instance of [WeightRegularizer](../regularizers.md),
+            applied to the bias.
+        activity_regularizer: instance of [ActivityRegularizer](../regularizers.md),
+            applied to the network output.
+        W_constraint: instance of the [constraints](../constraints.md) module
+            (eg. maxnorm, nonneg), applied to the main weights matrix.
+        b_constraint: instance of the [constraints](../constraints.md) module,
+            applied to the bias.
+        bias: whether to include a bias (i.e. make the layer affine rather than linear).
+        input_dim: Number of channels/dimensions in the input.
+            Either this argument or the keyword argument `input_shape`must be
+            provided when using this layer as the first layer in a model.
+        input_length: Length of input sequences, when it is constant.
+            This argument is required if you are going to connect
+            `Flatten` then `Dense` layers upstream
+            (without it, the shape of the dense outputs cannot be computed).
+    # Input shape
+        3D tensor with shape: `(samples, steps, input_dim)`.
+    # Output shape
+        3D tensor with shape: `(samples, new_steps, nb_filter)`.
+        `steps` value might have changed due to padding.
+    '''
+    def __init__(self, nb_filter, filter_length,
+                 init='uniform', activation='linear', weights=None,
+                 border_mode='valid', subsample_length=1,
+                 W_regularizer=None, b_regularizer=None, activity_regularizer=None,
+                 W_constraint=None, b_constraint=None,
+                 bias=True, input_dim=None, input_length=None, **kwargs):
+        if border_mode != 'valid':
+            raise Exception('Invalid border mode for Convolution2D '
+                            '(only "valid" is supported):', border_mode)
+        self.nb_filter = nb_filter
+        self.filter_length = filter_length
+        self.init = initializations.get(init, dim_ordering='th')
+        self.activation = activations.get(activation)
+
+        self.border_mode = border_mode
+        self.subsample_length = subsample_length
+
+        self.W_regularizer = regularizers.get(W_regularizer)
+        self.b_regularizer = regularizers.get(b_regularizer)
+        self.activity_regularizer = regularizers.get(activity_regularizer)
+
+        self.W_constraint = constraints.get(W_constraint)
+        self.b_constraint = constraints.get(b_constraint)
+
+        self.bias = bias
+        self.input_spec = [InputSpec(ndim=3)]
+        self.initial_weights = weights
+        self.input_dim = input_dim
+        self.input_length = input_length
+        if self.input_dim:
+            kwargs['input_shape'] = (self.input_length, self.input_dim)
+        super(LocallyConnected1D, self).__init__(**kwargs)
+
+    def build(self, input_shape):
+        input_dim = input_shape[2]
+        _, output_length, nb_filter = self.get_output_shape_for(input_shape)
+
+        self.W_shape = (output_length, self.filter_length * input_dim, nb_filter)
+        self.W = self.init(self.W_shape, name='{}_W'.format(self.name))
+        if self.bias:
+            self.b = K.zeros((output_length, self.nb_filter), name='{}_b'.format(self.name))
+            self.trainable_weights = [self.W, self.b]
+        else:
+            self.trainable_weights = [self.W]
+
+        self.regularizers = []
+        if self.W_regularizer:
+            self.W_regularizer.set_param(self.W)
+            self.regularizers.append(self.W_regularizer)
+        if self.b_regularizer:
+            self.b_regularizer.set_param(self.b)
+            self.regularizers.append(self.b_regularizer)
+        if self.activity_regularizer:
+            self.activity_regularizer.set_layer(self)
+            self.regularizers.append(self.activity_regularizer)
+
+        self.constraints = {}
+        if self.W_constraint:
+            self.constraints[self.W] = self.W_constraint
+        if self.b_constraint:
+            self.constraints[self.b] = self.b_constraint
+
+        if self.initial_weights is not None:
+            self.set_weights(self.initial_weights)
+            del self.initial_weights
+
+    def get_output_shape_for(self, input_shape):
+        length = conv_output_length(input_shape[1],
+                                    self.filter_length,
+                                    self.border_mode,
+                                    self.subsample_length)
+        return (input_shape[0], length, self.nb_filter)
+
+    def call(self, x, mask=None):
+        stride = self.subsample_length
+        output_length, feature_dim, nb_filter = self.W_shape
+
+        xs = []
+        for i in range(output_length):
+            slice_length = slice(i * stride, i * stride + self.filter_length)
+            xs.append(K.reshape(x[:, slice_length, :], (1, -1, feature_dim)))
+        x_aggregate = K.concatenate(xs, axis=0)
+        # (output_length, batch_size, nb_filter)
+        output = K.batch_dot(x_aggregate, self.W)
+        output = K.permute_dimensions(output, (1, 0, 2))
+
+        if self.bias:
+            output += K.reshape(self.b, (1, output_length, nb_filter))
+
+        output = self.activation(output)
+        return output
+
+    def get_config(self):
+        config = {'nb_filter': self.nb_filter,
+                  'filter_length': self.filter_length,
+                  'init': self.init.__name__,
+                  'activation': self.activation.__name__,
+                  'border_mode': self.border_mode,
+                  'subsample_length': self.subsample_length,
+                  'W_regularizer': self.W_regularizer.get_config() if self.W_regularizer else None,
+                  'b_regularizer': self.b_regularizer.get_config() if self.b_regularizer else None,
+                  'activity_regularizer': self.activity_regularizer.get_config() if self.activity_regularizer else None,
+                  'W_constraint': self.W_constraint.get_config() if self.W_constraint else None,
+                  'b_constraint': self.b_constraint.get_config() if self.b_constraint else None,
+                  'bias': self.bias,
+                  'input_dim': self.input_dim,
+                  'input_length': self.input_length}
+        base_config = super(LocallyConnected1D, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
+
+
+class LocallyConnected2D(Layer):
+    '''LocallyConnected2D layer works almost the same as Convolution2D layer,
+    except that weights are unshared, that is, a different set of filters is
+    applied at each different patch of the input. When using this layer as the
+    first layer in a model, provide the keyword argument `input_shape` (tuple
+    of integers, does not include the sample axis), e.g.
+    `input_shape=(3, 128, 128)` for 128x128 RGB pictures. Also, you will need
+    to fix shape of the previous layer, since the weights can only be defined
+    with determined output shape.
+
+    # Examples
+    ```python
+        # apply a 3x3 unshared weights convolution with 64 output filters on a 32x32 image:
+        model = Sequential()
+        model.add(LocallyConnected2D(64, 3, 3, input_shape=(3, 32, 32)))
+        # now model.output_shape == (None, 64, 30, 30)
+        # notice that this layer will consume (30*30)*(3*3*3*64) + (30*30)*64 parameters
+
+        # add a 3x3 unshared weights convolution on top, with 32 output filters:
+        model.add(LocallyConnected2D(32, 3, 3))
+        # now model.output_shape == (None, 32, 28, 28)
+    ```
+
+    # Arguments
+        nb_filter: Number of convolution filters to use.
+        nb_row: Number of rows in the convolution kernel.
+        nb_col: Number of columns in the convolution kernel.
+        init: name of initialization function for the weights of the layer
+            (see [initializations](../initializations.md)), or alternatively,
+            Theano function to use for weights initialization.
+            This parameter is only relevant if you don't pass
+            a `weights` argument.
+        activation: name of activation function to use
+            (see [activations](../activations.md)),
+            or alternatively, elementwise Theano function.
+            If you don't specify anything, no activation is applied
+            (ie. "linear" activation: a(x) = x).
+        weights: list of numpy arrays to set as initial weights.
+        border_mode: Only support 'valid'. Please make good use of
+            ZeroPadding2D to achieve same output shape.
+        subsample: tuple of length 2. Factor by which to subsample output.
+            Also called strides elsewhere.
+        W_regularizer: instance of [WeightRegularizer](../regularizers.md)
+            (eg. L1 or L2 regularization), applied to the main weights matrix.
+        b_regularizer: instance of [WeightRegularizer](../regularizers.md),
+            applied to the bias.
+        activity_regularizer: instance of [ActivityRegularizer](../regularizers.md),
+            applied to the network output.
+        W_constraint: instance of the [constraints](../constraints.md) module
+            (eg. maxnorm, nonneg), applied to the main weights matrix.
+        b_constraint: instance of the [constraints](../constraints.md) module,
+            applied to the bias.
+        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
+            (the depth) is at index 1, in 'tf' mode is it at index 3.
+        bias: whether to include a bias (i.e. make the layer affine rather than linear).
+
+    # Input shape
+        4D tensor with shape:
+        `(samples, channels, rows, cols)` if dim_ordering='th'
+        or 4D tensor with shape:
+        `(samples, rows, cols, channels)` if dim_ordering='tf'.
+
+    # Output shape
+        4D tensor with shape:
+        `(samples, nb_filter, new_rows, new_cols)` if dim_ordering='th'
+        or 4D tensor with shape:
+        `(samples, new_rows, new_cols, nb_filter)` if dim_ordering='tf'.
+        `rows` and `cols` values might have changed due to padding.
+    '''
+    def __init__(self, nb_filter, nb_row, nb_col,
+                 init='glorot_uniform', activation='linear', weights=None,
+                 border_mode='valid', subsample=(1, 1),
+                 dim_ordering=K.image_dim_ordering(),
+                 W_regularizer=None, b_regularizer=None, activity_regularizer=None,
+                 W_constraint=None, b_constraint=None,
+                 bias=True, **kwargs):
+        if border_mode != 'valid':
+            raise Exception('Invalid border mode for Convolution2D '
+                            '(only "valid" is supported):', border_mode)
+        self.nb_filter = nb_filter
+        self.nb_row = nb_row
+        self.nb_col = nb_col
+        self.init = initializations.get(init, dim_ordering=dim_ordering)
+        self.activation = activations.get(activation)
+
+        self.border_mode = border_mode
+        self.subsample = tuple(subsample)
+        assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
+        self.dim_ordering = dim_ordering
+
+        self.W_regularizer = regularizers.get(W_regularizer)
+        self.b_regularizer = regularizers.get(b_regularizer)
+        self.activity_regularizer = regularizers.get(activity_regularizer)
+
+        self.W_constraint = constraints.get(W_constraint)
+        self.b_constraint = constraints.get(b_constraint)
+
+        self.bias = bias
+        self.input_spec = [InputSpec(ndim=4)]
+        self.initial_weights = weights
+        super(LocallyConnected2D, self).__init__(**kwargs)
+
+    def build(self, input_shape):
+        output_shape = self.get_output_shape_for(input_shape)
+        if self.dim_ordering == 'th':
+            _, nb_filter, output_row, output_col = output_shape
+            input_filter = input_shape[1]
+        elif self.dim_ordering == 'tf':
+            _, output_row, output_col, nb_filter = output_shape
+            input_filter = input_shape[3]
+        else:
+            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+
+        self.output_row = output_row
+        self.output_col = output_col
+        self.W_shape = (output_row * output_col, self.nb_row * self.nb_col * input_filter, nb_filter)
+        self.W = self.init(self.W_shape, name='{}_W'.format(self.name))
+
+        if self.bias:
+            self.b = K.zeros((output_row, output_col, nb_filter), name='{}_b'.format(self.name))
+            self.trainable_weights = [self.W, self.b]
+        else:
+            self.trainable_weights = [self.W]
+
+        self.regularizers = []
+        if self.W_regularizer:
+            self.W_regularizer.set_param(self.W)
+            self.regularizers.append(self.W_regularizer)
+        if self.bias and self.b_regularizer:
+            self.b_regularizer.set_param(self.b)
+            self.regularizers.append(self.b_regularizer)
+        if self.activity_regularizer:
+            self.activity_regularizer.set_layer(self)
+            self.regularizers.append(self.activity_regularizer)
+
+        self.constraints = {}
+        if self.W_constraint:
+            self.constraints[self.W] = self.W_constraint
+        if self.bias and self.b_constraint:
+            self.constraints[self.b] = self.b_constraint
+
+        if self.initial_weights is not None:
+            self.set_weights(self.initial_weights)
+            del self.initial_weights
+
+    def get_output_shape_for(self, input_shape):
+        if self.dim_ordering == 'th':
+            rows = input_shape[2]
+            cols = input_shape[3]
+        elif self.dim_ordering == 'tf':
+            rows = input_shape[1]
+            cols = input_shape[2]
+        else:
+            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+
+        rows = conv_output_length(rows, self.nb_row,
+                                  self.border_mode, self.subsample[0])
+        cols = conv_output_length(cols, self.nb_col,
+                                  self.border_mode, self.subsample[1])
+
+        if self.dim_ordering == 'th':
+            return (input_shape[0], self.nb_filter, rows, cols)
+        elif self.dim_ordering == 'tf':
+            return (input_shape[0], rows, cols, self.nb_filter)
+        else:
+            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+
+    def call(self, x, mask=None):
+        stride_row, stride_col = self.subsample
+        _, feature_dim, nb_filter = self.W_shape
+
+        if self.dim_ordering == 'th':
+            if K._backend == 'theano':
+                output = []
+                for i in range(self.output_row):
+                    for j in range(self.output_col):
+                        slice_row = slice(i * stride_row,
+                                          i * stride_row + self.nb_row)
+                        slice_col = slice(j * stride_col,
+                                          j * stride_col + self.nb_col)
+                        x_flatten = K.reshape(x[:, :, slice_row, slice_col], (1, -1, feature_dim))
+                        output.append(K.dot(x_flatten, self.W[i * self.output_col + j, :, :]))
+                output = K.concatenate(output, axis=0)
+            else:
+                xs = []
+                for i in range(self.output_row):
+                    for j in range(self.output_col):
+                        slice_row = slice(i * stride_row,
+                                          i * stride_row + self.nb_row)
+                        slice_col = slice(j * stride_col,
+                                          j * stride_col + self.nb_col)
+                        xs.append(K.reshape(x[:, :, slice_row, slice_col], (1, -1, feature_dim)))
+                x_aggregate = K.concatenate(xs, axis=0)
+                output = K.batch_dot(x_aggregate, self.W)
+            output = K.reshape(output, (self.output_row, self.output_col, -1, nb_filter))
+            output = K.permute_dimensions(output, (2, 3, 0, 1))
+        elif self.dim_ordering == 'tf':
+            xs = []
+            for i in range(self.output_row):
+                for j in range(self.output_col):
+                    slice_row = slice(i * stride_row,
+                                      i * stride_row + self.nb_row)
+                    slice_col = slice(j * stride_col,
+                                      j * stride_col + self.nb_col)
+                    xs.append(K.reshape(x[:, slice_row, slice_col, :], (1, -1, feature_dim)))
+            x_aggregate = K.concatenate(xs, axis=0)
+            output = K.batch_dot(x_aggregate, self.W)
+            output = K.reshape(output, (self.output_row, self.output_col, -1, nb_filter))
+            output = K.permute_dimensions(output, (2, 0, 1, 3))
+        else:
+            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+
+        if self.bias:
+            if self.dim_ordering == 'th':
+                output += K.reshape(self.b, (1, nb_filter, self.output_row, self.output_col))
+            elif self.dim_ordering == 'tf':
+                output += K.reshape(self.b, (1, self.output_row, self.output_col, nb_filter))
+            else:
+                raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+
+        output = self.activation(output)
+        return output
+
+    def get_config(self):
+        config = {'nb_filter': self.nb_filter,
+                  'nb_row': self.nb_row,
+                  'nb_col': self.nb_col,
+                  'init': self.init.__name__,
+                  'activation': self.activation.__name__,
+                  'border_mode': self.border_mode,
+                  'subsample': self.subsample,
+                  'dim_ordering': self.dim_ordering,
+                  'W_regularizer': self.W_regularizer.get_config() if self.W_regularizer else None,
+                  'b_regularizer': self.b_regularizer.get_config() if self.b_regularizer else None,
+                  'activity_regularizer': self.activity_regularizer.get_config() if self.activity_regularizer else None,
+                  'W_constraint': self.W_constraint.get_config() if self.W_constraint else None,
+                  'b_constraint': self.b_constraint.get_config() if self.b_constraint else None,
+                  'bias': self.bias}
+        base_config = super(LocallyConnected2D, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))

keras/layers/normalization.py

@@ -57,6 +57,7 @@ class BatchNormalization(Layer):
     '''
     def __init__(self, epsilon=1e-6, mode=0, axis=-1, momentum=0.9,
                  weights=None, beta_init='zero', gamma_init='one', **kwargs):
+        self.supports_masking = True
         self.beta_init = initializations.get(beta_init)
         self.gamma_init = initializations.get(gamma_init)
         self.epsilon = epsilon
@@ -98,18 +99,17 @@ class BatchNormalization(Layer):
             broadcast_shape = [1] * len(input_shape)
             broadcast_shape[self.axis] = input_shape[self.axis]
 
-            # case: train mode (uses stats of the current batch)
-            mean = K.mean(x, axis=reduction_axes)
-            brodcast_mean = K.reshape(mean, broadcast_shape)
-            std = K.mean(K.square(x - brodcast_mean) + self.epsilon, axis=reduction_axes)
-            std = K.sqrt(std)
-            brodcast_std = K.reshape(std, broadcast_shape)
-            mean_update = self.momentum * self.running_mean + (1 - self.momentum) * mean
-            std_update = self.momentum * self.running_std + (1 - self.momentum) * std
+            # # case: train mode (uses stats of the current batch)
+            # mean = K.mean(x, axis=reduction_axes)
+            # brodcast_mean = K.reshape(mean, broadcast_shape)
+            # std = K.mean(K.square(x - brodcast_mean) + self.epsilon, axis=reduction_axes)
+            # std = K.sqrt(std)
+            # brodcast_std = K.reshape(std, broadcast_shape)
 
             if self.mode == 2:
-                x_normed = (x - brodcast_mean) / (brodcast_std + self.epsilon)
-                out = K.reshape(self.gamma, broadcast_shape) * x_normed + K.reshape(self.beta, broadcast_shape)
+                x_normed, mean, std = K.normalize_batch_in_training(x, self.gamma, self.beta, reduction_axes, epsilon=self.epsilon)
+                mean_update = self.momentum * self.running_mean + (1 - self.momentum) * mean
+                std_update = self.momentum * self.running_std + (1 - self.momentum) * std
             else:
                 # mode 0
                 if self.called_with not in {None, x}:
@@ -123,26 +123,40 @@ class BatchNormalization(Layer):
                                     '(see docs for a description of '
                                     'the behavior).')
                 self.called_with = x
+                x_normed, mean, std = K.normalize_batch_in_training(x, self.gamma, self.beta, reduction_axes, epsilon=self.epsilon)
+                mean_update = self.momentum * self.running_mean + (1 - self.momentum) * mean
+                std_update = self.momentum * self.running_std + (1 - self.momentum) * std
                 self.updates = [(self.running_mean, mean_update),
                                 (self.running_std, std_update)]
-                x_normed = (x - brodcast_mean) / (brodcast_std + self.epsilon)
 
-                # case: test mode (uses running averages)
-                brodcast_running_mean = K.reshape(self.running_mean, broadcast_shape)
-                brodcast_running_std = K.reshape(self.running_std, broadcast_shape)
-                x_normed_running = ((x - brodcast_running_mean) / (brodcast_running_std + self.epsilon))
+                if sorted(reduction_axes) == range(K.ndim(x))[:-1]:
+                    x_normed_running = K.batch_normalization(x, self.running_mean,
+                                                             self.running_std,
+                                                             self.beta,
+                                                             self.gamma,
+                                                             epsilon=self.epsilon)
+                else:
+                    # need broadcasting
+                    broadcast_running_mean = K.reshape(self.running_mean, broadcast_shape)
+                    broadcast_running_std = K.reshape(self.running_std, broadcast_shape)
+                    broadcast_beta = K.reshape(self.beta, broadcast_shape)
+                    broadcast_gamma = K.reshape(self.gamma, broadcast_shape)
+                    x_normed_running = K.batch_normalization(x, broadcast_running_mean,
+                                                             broadcast_running_std,
+                                                             broadcast_beta,
+                                                             broadcast_gamma,
+                                                             epsilon=self.epsilon)
 
                 # pick the normalized form of x corresponding to the training phase
                 x_normed = K.in_train_phase(x_normed, x_normed_running)
-                out = K.reshape(self.gamma, broadcast_shape) * x_normed + K.reshape(self.beta, broadcast_shape)
 
         elif self.mode == 1:
             # sample-wise normalization
             m = K.mean(x, axis=-1, keepdims=True)
             std = K.sqrt(K.var(x, axis=-1, keepdims=True) + self.epsilon)
             x_normed = (x - m) / (std + self.epsilon)
-            out = self.gamma * x_normed + self.beta
-        return out
+            x_normed = self.gamma * x_normed + self.beta
+        return x_normed
 
     def get_config(self):
         config = {"epsilon": self.epsilon,

keras/layers/pooling.py

@@ -0,0 +1,395 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+from .. import backend as K
+from ..engine import Layer, InputSpec
+from ..utils.np_utils import conv_output_length
+
+
+class _Pooling1D(Layer):
+    '''Abstract class for different pooling 1D layers.
+    '''
+    input_dim = 3
+
+    def __init__(self, pool_length=2, stride=None,
+                 border_mode='valid', **kwargs):
+        super(_Pooling1D, self).__init__(**kwargs)
+        if stride is None:
+            stride = pool_length
+        self.pool_length = pool_length
+        self.stride = stride
+        self.st = (self.stride, 1)
+        self.pool_size = (pool_length, 1)
+        assert border_mode in {'valid', 'same'}, 'border_mode must be in {valid, same}'
+        self.border_mode = border_mode
+        self.input_spec = [InputSpec(ndim=3)]
+
+    def get_output_shape_for(self, input_shape):
+        length = conv_output_length(input_shape[1], self.pool_length,
+                                    self.border_mode, self.stride)
+        return (input_shape[0], length, input_shape[2])
+
+    def _pooling_function(self, back_end, inputs, pool_size, strides,
+                          border_mode, dim_ordering):
+        raise NotImplementedError
+
+    def call(self, x, mask=None):
+        x = K.expand_dims(x, -1)   # add dummy last dimension
+        x = K.permute_dimensions(x, (0, 2, 1, 3))
+        output = self._pooling_function(inputs=x, pool_size=self.pool_size,
+                                        strides=self.st,
+                                        border_mode=self.border_mode,
+                                        dim_ordering='th')
+        output = K.permute_dimensions(output, (0, 2, 1, 3))
+        return K.squeeze(output, 3)  # remove dummy last dimension
+
+    def get_config(self):
+        config = {'stride': self.stride,
+                  'pool_length': self.pool_length,
+                  'border_mode': self.border_mode}
+        base_config = super(_Pooling1D, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
+
+
+class MaxPooling1D(_Pooling1D):
+    '''Max pooling operation for temporal data.
+
+    # Input shape
+        3D tensor with shape: `(samples, steps, features)`.
+
+    # Output shape
+        3D tensor with shape: `(samples, downsampled_steps, features)`.
+
+    # Arguments
+        pool_length: factor by which to downscale. 2 will halve the input.
+        stride: integer, or None. Stride value.
+            If None, it will default to `pool_length`.
+        border_mode: 'valid' or 'same'.
+            Note: 'same' will only work with TensorFlow for the time being.
+    '''
+
+    def __init__(self, pool_length=2, stride=None,
+                 border_mode='valid', **kwargs):
+        super(MaxPooling1D, self).__init__(pool_length, stride,
+                                           border_mode, **kwargs)
+
+    def _pooling_function(self, inputs, pool_size, strides,
+                          border_mode, dim_ordering):
+        output = K.pool2d(inputs, pool_size, strides,
+                          border_mode, dim_ordering, pool_mode='max')
+        return output
+
+
+class AveragePooling1D(_Pooling1D):
+    '''Average pooling for temporal data.
+
+    # Arguments
+        pool_length: factor by which to downscale. 2 will halve the input.
+        stride: integer, or None. Stride value.
+            If None, it will default to `pool_length`.
+        border_mode: 'valid' or 'same'.
+            Note: 'same' will only work with TensorFlow for the time being.
+
+    # Input shape
+        3D tensor with shape: `(samples, steps, features)`.
+
+    # Output shape
+        3D tensor with shape: `(samples, downsampled_steps, features)`.
+    '''
+
+    def __init__(self, pool_length=2, stride=None,
+                 border_mode='valid', **kwargs):
+        super(AveragePooling1D, self).__init__(pool_length, stride,
+                                               border_mode, **kwargs)
+
+    def _pooling_function(self, inputs, pool_size, strides,
+                          border_mode, dim_ordering):
+        output = K.pool2d(inputs, pool_size, strides,
+                          border_mode, dim_ordering, pool_mode='avg')
+        return output
+
+
+class _Pooling2D(Layer):
+    '''Abstract class for different pooling 2D layers.
+    '''
+
+    def __init__(self, pool_size=(2, 2), strides=None, border_mode='valid',
+                 dim_ordering=K.image_dim_ordering(), **kwargs):
+        super(_Pooling2D, self).__init__(**kwargs)
+        self.pool_size = tuple(pool_size)
+        if strides is None:
+            strides = self.pool_size
+        self.strides = tuple(strides)
+        assert border_mode in {'valid', 'same'}, 'border_mode must be in {valid, same}'
+        self.border_mode = border_mode
+        assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
+        self.dim_ordering = dim_ordering
+        self.input_spec = [InputSpec(ndim=4)]
+
+    def get_output_shape_for(self, input_shape):
+        if self.dim_ordering == 'th':
+            rows = input_shape[2]
+            cols = input_shape[3]
+        elif self.dim_ordering == 'tf':
+            rows = input_shape[1]
+            cols = input_shape[2]
+        else:
+            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+
+        rows = conv_output_length(rows, self.pool_size[0],
+                                  self.border_mode, self.strides[0])
+        cols = conv_output_length(cols, self.pool_size[1],
+                                  self.border_mode, self.strides[1])
+
+        if self.dim_ordering == 'th':
+            return (input_shape[0], input_shape[1], rows, cols)
+        elif self.dim_ordering == 'tf':
+            return (input_shape[0], rows, cols, input_shape[3])
+        else:
+            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+
+    def _pooling_function(self, inputs, pool_size, strides,
+                          border_mode, dim_ordering):
+        raise NotImplementedError
+
+    def call(self, x, mask=None):
+        output = self._pooling_function(inputs=x, pool_size=self.pool_size,
+                                        strides=self.strides,
+                                        border_mode=self.border_mode,
+                                        dim_ordering=self.dim_ordering)
+        return output
+
+    def get_config(self):
+        config = {'pool_size': self.pool_size,
+                  'border_mode': self.border_mode,
+                  'strides': self.strides,
+                  'dim_ordering': self.dim_ordering}
+        base_config = super(_Pooling2D, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
+
+
+class MaxPooling2D(_Pooling2D):
+    '''Max pooling operation for spatial data.
+
+    # Arguments
+        pool_size: tuple of 2 integers,
+            factors by which to downscale (vertical, horizontal).
+            (2, 2) will halve the image in each dimension.
+        strides: tuple of 2 integers, or None. Strides values.
+            If None, it will default to `pool_size`.
+        border_mode: 'valid' or 'same'.
+            Note: 'same' will only work with TensorFlow for the time being.
+        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
+            (the depth) is at index 1, in 'tf' mode is it at index 3.
+            It defaults to the `image_dim_ordering` value found in your
+            Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be "th".
+
+    # Input shape
+        4D tensor with shape:
+        `(samples, channels, rows, cols)` if dim_ordering='th'
+        or 4D tensor with shape:
+        `(samples, rows, cols, channels)` if dim_ordering='tf'.
+
+    # Output shape
+        4D tensor with shape:
+        `(nb_samples, channels, pooled_rows, pooled_cols)` if dim_ordering='th'
+        or 4D tensor with shape:
+        `(samples, pooled_rows, pooled_cols, channels)` if dim_ordering='tf'.
+    '''
+
+    def __init__(self, pool_size=(2, 2), strides=None, border_mode='valid',
+                 dim_ordering=K.image_dim_ordering(), **kwargs):
+        super(MaxPooling2D, self).__init__(pool_size, strides, border_mode,
+                                           dim_ordering, **kwargs)
+
+    def _pooling_function(self, inputs, pool_size, strides,
+                          border_mode, dim_ordering):
+        output = K.pool2d(inputs, pool_size, strides,
+                          border_mode, dim_ordering, pool_mode='max')
+        return output
+
+
+class AveragePooling2D(_Pooling2D):
+    '''Average pooling operation for spatial data.
+
+    # Arguments
+        pool_size: tuple of 2 integers,
+            factors by which to downscale (vertical, horizontal).
+            (2, 2) will halve the image in each dimension.
+        strides: tuple of 2 integers, or None. Strides values.
+            If None, it will default to `pool_size`.
+        border_mode: 'valid' or 'same'.
+            Note: 'same' will only work with TensorFlow for the time being.
+        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
+            (the depth) is at index 1, in 'tf' mode is it at index 3.
+            It defaults to the `image_dim_ordering` value found in your
+            Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be "th".
+
+    # Input shape
+        4D tensor with shape:
+        `(samples, channels, rows, cols)` if dim_ordering='th'
+        or 4D tensor with shape:
+        `(samples, rows, cols, channels)` if dim_ordering='tf'.
+
+    # Output shape
+        4D tensor with shape:
+        `(nb_samples, channels, pooled_rows, pooled_cols)` if dim_ordering='th'
+        or 4D tensor with shape:
+        `(samples, pooled_rows, pooled_cols, channels)` if dim_ordering='tf'.
+    '''
+
+    def __init__(self, pool_size=(2, 2), strides=None, border_mode='valid',
+                 dim_ordering=K.image_dim_ordering(), **kwargs):
+        super(AveragePooling2D, self).__init__(pool_size, strides, border_mode,
+                                               dim_ordering, **kwargs)
+
+    def _pooling_function(self, inputs, pool_size, strides,
+                          border_mode, dim_ordering):
+        output = K.pool2d(inputs, pool_size, strides,
+                          border_mode, dim_ordering, pool_mode='avg')
+        return output
+
+
+class _Pooling3D(Layer):
+    '''Abstract class for different pooling 3D layers.
+    '''
+
+    def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode='valid',
+                 dim_ordering=K.image_dim_ordering(), **kwargs):
+        super(_Pooling3D, self).__init__(**kwargs)
+        self.pool_size = tuple(pool_size)
+        if strides is None:
+            strides = self.pool_size
+        self.strides = tuple(strides)
+        assert border_mode in {'valid', 'same'}, 'border_mode must be in {valid, same}'
+        self.border_mode = border_mode
+        assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
+        self.dim_ordering = dim_ordering
+        self.input_spec = [InputSpec(ndim=5)]
+
+    def get_output_shape_for(self, input_shape):
+        if self.dim_ordering == 'th':
+            len_dim1 = input_shape[2]
+            len_dim2 = input_shape[3]
+            len_dim3 = input_shape[4]
+        elif self.dim_ordering == 'tf':
+            len_dim1 = input_shape[1]
+            len_dim2 = input_shape[2]
+            len_dim3 = input_shape[3]
+        else:
+            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+
+        len_dim1 = conv_output_length(len_dim1, self.pool_size[0],
+                                      self.border_mode, self.strides[0])
+        len_dim2 = conv_output_length(len_dim2, self.pool_size[1],
+                                      self.border_mode, self.strides[1])
+        len_dim3 = conv_output_length(len_dim3, self.pool_size[2],
+                                      self.border_mode, self.strides[2])
+
+        if self.dim_ordering == 'th':
+            return (input_shape[0], input_shape[1], len_dim1, len_dim2, len_dim3)
+        elif self.dim_ordering == 'tf':
+            return (input_shape[0], len_dim1, len_dim2, len_dim3, input_shape[4])
+        else:
+            raise Exception('Invalid dim_ordering: ' + self.dim_ordering)
+
+    def _pooling_function(self, inputs, pool_size, strides,
+                          border_mode, dim_ordering):
+        raise NotImplementedError
+
+    def call(self, x, mask=None):
+        output = self._pooling_function(inputs=x, pool_size=self.pool_size,
+                                        strides=self.strides,
+                                        border_mode=self.border_mode,
+                                        dim_ordering=self.dim_ordering)
+        return output
+
+    def get_config(self):
+        config = {'pool_size': self.pool_size,
+                  'border_mode': self.border_mode,
+                  'strides': self.strides,
+                  'dim_ordering': self.dim_ordering}
+        base_config = super(_Pooling3D, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
+
+
+class MaxPooling3D(_Pooling3D):
+    '''Max pooling operation for 3D data (spatial or spatio-temporal).
+
+    # Arguments
+        pool_size: tuple of 3 integers,
+            factors by which to downscale (dim1, dim2, dim3).
+            (2, 2, 2) will halve the size of the 3D input in each dimension.
+        strides: tuple of 3 integers, or None. Strides values.
+        border_mode: 'valid' or 'same'.
+        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
+            (the depth) is at index 1, in 'tf' mode is it at index 4.
+            It defaults to the `image_dim_ordering` value found in your
+            Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be "th".
+
+    # Input shape
+        5D tensor with shape:
+        `(samples, channels, len_pool_dim1, len_pool_dim2, len_pool_dim3)` if dim_ordering='th'
+        or 5D tensor with shape:
+        `(samples, len_pool_dim1, len_pool_dim2, len_pool_dim3, channels)` if dim_ordering='tf'.
+
+    # Output shape
+        5D tensor with shape:
+        `(nb_samples, channels, pooled_dim1, pooled_dim2, pooled_dim3)` if dim_ordering='th'
+        or 5D tensor with shape:
+        `(samples, pooled_dim1, pooled_dim2, pooled_dim3, channels)` if dim_ordering='tf'.
+    '''
+
+    def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode='valid',
+                 dim_ordering=K.image_dim_ordering(), **kwargs):
+        super(MaxPooling3D, self).__init__(pool_size, strides, border_mode,
+                                           dim_ordering, **kwargs)
+
+    def _pooling_function(self, inputs, pool_size, strides,
+                          border_mode, dim_ordering):
+        output = K.pool3d(inputs, pool_size, strides,
+                          border_mode, dim_ordering, pool_mode='max')
+        return output
+
+
+class AveragePooling3D(_Pooling3D):
+    '''Average pooling operation for 3D data (spatial or spatio-temporal).
+
+    # Arguments
+        pool_size: tuple of 3 integers,
+            factors by which to downscale (dim1, dim2, dim3).
+            (2, 2, 2) will halve the size of the 3D input in each dimension.
+        strides: tuple of 3 integers, or None. Strides values.
+        border_mode: 'valid' or 'same'.
+        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
+            (the depth) is at index 1, in 'tf' mode is it at index 4.
+            It defaults to the `image_dim_ordering` value found in your
+            Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be "th".
+
+    # Input shape
+        5D tensor with shape:
+        `(samples, channels, len_pool_dim1, len_pool_dim2, len_pool_dim3)` if dim_ordering='th'
+        or 5D tensor with shape:
+        `(samples, len_pool_dim1, len_pool_dim2, len_pool_dim3, channels)` if dim_ordering='tf'.
+
+    # Output shape
+        5D tensor with shape:
+        `(nb_samples, channels, pooled_dim1, pooled_dim2, pooled_dim3)` if dim_ordering='th'
+        or 5D tensor with shape:
+        `(samples, pooled_dim1, pooled_dim2, pooled_dim3, channels)` if dim_ordering='tf'.
+    '''
+
+    def __init__(self, pool_size=(2, 2, 2), strides=None, border_mode='valid',
+                 dim_ordering=K.image_dim_ordering(), **kwargs):
+        super(AveragePooling3D, self).__init__(pool_size, strides, border_mode,
+                                               dim_ordering, **kwargs)
+
+    def _pooling_function(self, inputs, pool_size, strides,
+                          border_mode, dim_ordering):
+        output = K.pool3d(inputs, pool_size, strides,
+                          border_mode, dim_ordering, pool_mode='avg')
+        return output

keras/layers/recurrent.py

@@ -190,9 +190,9 @@ class Recurrent(Layer):
     def get_initial_states(self, x):
         # build an all-zero tensor of shape (samples, output_dim)
         initial_state = K.zeros_like(x)  # (samples, timesteps, input_dim)
-        initial_state = K.sum(initial_state, axis=1)  # (samples, input_dim)
-        reducer = K.zeros((self.input_dim, self.output_dim))
-        initial_state = K.dot(initial_state, reducer)  # (samples, output_dim)
+        initial_state = K.sum(initial_state, axis=(1, 2))  # (samples,)
+        initial_state = K.expand_dims(initial_state)  # (samples, 1)
+        initial_state = K.tile(initial_state, [1, self.output_dim])  # (samples, output_dim)
         initial_states = [initial_state for _ in range(len(self.states))]
         return initial_states
 
@@ -689,7 +689,7 @@ class LSTM(Recurrent):
                                      name='{}_U'.format(self.name))
 
             self.b = K.variable(np.hstack((np.zeros(self.output_dim),
-                                           K.get_value(self.forget_bias_init(self.output_dim)),
+                                           K.get_value(self.forget_bias_init((self.output_dim,))),
                                            np.zeros(self.output_dim),
                                            np.zeros(self.output_dim))),
                                 name='{}_b'.format(self.name))

keras/models.py

@@ -158,6 +158,22 @@ class Sequential(Model):
         self.built = False
         self._flattened_layers = None
 
+    def pop(self):
+        '''Removes the last layer in the model.
+        '''
+        if not self.layers:
+            raise Exception('There are no layers in the model.')
+
+        self.layers.pop()
+        if not self.layers:
+            self.outputs = []
+            self.inbound_nodes = []
+            self.outbound_nodes = []
+        else:
+            self.layers[-1].outbound_nodes = []
+            self.outputs = [self.layers[-1].output]
+        self.built = False
+
     def call(self, x, mask=None):
         if not self.built:
             self.build()
@@ -578,7 +594,7 @@ class Sequential(Model):
     def fit_generator(self, generator, samples_per_epoch, nb_epoch,
                       verbose=1, callbacks=[],
                       validation_data=None, nb_val_samples=None,
-                      class_weight=None, max_q_size=10, **kwargs):
+                      class_weight=None, max_q_size=10, nb_worker=1, pickle_safe=False, **kwargs):
         '''Fits the model on data generated batch-by-batch by
         a Python generator.
         The generator is run in parallel to the model, for efficiency.
@@ -609,6 +625,11 @@ class Sequential(Model):
             class_weight: dictionary mapping class indices to a weight
                 for the class.
             max_q_size: maximum size for the generator queue
+            nb_worker: maximum number of processes to spin up
+            pickle_safe: if True, use process based threading. Note that because
+                this implementation relies on multiprocessing, you should not pass non
+                non picklable arguments to the generator as they can't be passed
+                easily to children processes.
 
         # Returns
             A `History` object.
@@ -632,6 +653,9 @@ class Sequential(Model):
         '''
         if self.model is None:
             raise Exception('The model needs to be compiled before being used.')
+        if nb_worker > 1 and not pickle_safe:
+            warnings.warn('The "nb_worker" argument is deprecated when pickle_safe is False')
+            nb_worker = 1  # For backward compatibility
         if 'show_accuracy' in kwargs:
             kwargs.pop('show_accuracy')
             warnings.warn('The "show_accuracy" argument is deprecated, '
@@ -639,10 +663,6 @@ class Sequential(Model):
                           'the model at compile time:\n'
                           '`model.compile(optimizer, loss, '
                           'metrics=["accuracy"])`')
-        if 'nb_worker' in kwargs:
-            kwargs.pop('nb_worker')
-            warnings.warn('The "nb_worker" argument is deprecated, '
-                          'please remove it from your code.')
         if 'nb_val_worker' in kwargs:
             kwargs.pop('nb_val_worker')
             warnings.warn('The "nb_val_worker" argument is deprecated, '
@@ -658,9 +678,11 @@ class Sequential(Model):
                                         validation_data=validation_data,
                                         nb_val_samples=nb_val_samples,
                                         class_weight=class_weight,
-                                        max_q_size=max_q_size)
+                                        max_q_size=max_q_size,
+                                        nb_worker=nb_worker,
+                                        pickle_safe=pickle_safe)
 
-    def evaluate_generator(self, generator, val_samples, max_q_size=10, **kwargs):
+    def evaluate_generator(self, generator, val_samples, max_q_size=10, nb_worker=1, pickle_safe=False, **kwargs):
         '''Evaluates the model on a data generator. The generator should
         return the same kind of data as accepted by `test_on_batch`.
 
@@ -672,9 +694,17 @@ class Sequential(Model):
                 total number of samples to generate from `generator`
                 before returning.
             max_q_size: maximum size for the generator queue
+            nb_worker: maximum number of processes to spin up
+            pickle_safe: if True, use process based threading. Note that because
+                this implementation relies on multiprocessing, you should not pass non
+                non picklable arguments to the generator as they can't be passed
+                easily to children processes.
         '''
         if self.model is None:
             raise Exception('The model needs to be compiled before being used.')
+        if nb_worker > 1 and not pickle_safe:
+            warnings.warn('The "nb_worker" argument is deprecated when pickle_safe is False')
+            nb_worker = 1  # For backward compatibility
         if 'show_accuracy' in kwargs:
             kwargs.pop('show_accuracy')
             warnings.warn('The "show_accuracy" argument is deprecated, '
@@ -690,9 +720,11 @@ class Sequential(Model):
                             str(kwargs))
         return self.model.evaluate_generator(generator,
                                              val_samples,
-                                             max_q_size=max_q_size)
+                                             max_q_size=max_q_size,
+                                             nb_worker=nb_worker,
+                                             pickle_safe=pickle_safe)
 
-    def predict_generator(self, generator, val_samples, max_q_size=10):
+    def predict_generator(self, generator, val_samples, max_q_size=10, nb_worker=1, pickle_safe=False):
         '''Generates predictions for the input samples from a data generator.
         The generator should return the same kind of data as accepted by
         `predict_on_batch`.
@@ -702,14 +734,24 @@ class Sequential(Model):
             val_samples: total number of samples to generate from `generator`
                 before returning.
             max_q_size: maximum size for the generator queue
+            nb_worker: maximum number of processes to spin up
+            pickle_safe: if True, use process based threading. Note that because
+                this implementation relies on multiprocessing, you should not pass non
+                non picklable arguments to the generator as they can't be passed
+                easily to children processes.
 
         # Returns
             A Numpy array of predictions.
         '''
         if self.model is None:
             self.build()
+        if nb_worker > 1 and not pickle_safe:
+            warnings.warn('The "nb_worker" argument is deprecated when pickle_safe is False')
+            nb_worker = 1  # For backward compatibility
         return self.model.predict_generator(generator, val_samples,
-                                            max_q_size=max_q_size)
+                                            max_q_size=max_q_size,
+                                            nb_worker=nb_worker,
+                                            pickle_safe=pickle_safe)
 
     def get_config(self):
         '''Returns the model configuration

keras/regularizers.py

@@ -1,5 +1,4 @@
 from __future__ import absolute_import
-import numpy as np
 from . import backend as K
 
 
@@ -18,11 +17,11 @@ class Regularizer(object):
 
 
 class EigenvalueRegularizer(Regularizer):
-    '''This takes a constant that controls the 
-    regularization by Eigenvalue Decay on the 
-    current layer and outputs the regularized 
-    loss (evaluated on the training data) and 
-    the original loss (evaluated on the 
+    '''This takes a constant that controls
+    the regularization by Eigenvalue Decay on the
+    current layer and outputs the regularized
+    loss (evaluated on the training data) and
+    the original loss (evaluated on the
     validation data).
     '''
     def __init__(self, k):
@@ -41,16 +40,15 @@ class EigenvalueRegularizer(Regularizer):
                             'and embedding layers.')
         WW = K.dot(K.transpose(W), W)
         dim1, dim2 = K.eval(K.shape(WW))  # number of neurons in the layer
-        k = self.k
-        
+
         # power method for approximating the dominant eigenvector:
         o = K.ones([dim1, 1])  # initial values for the dominant eigenvector
         domin_eigenvect = K.dot(WW, o)
         for n in range(power - 1):
-            domin_eigenvect = K.dot(WW, domin_eigenvect)    
-        
+            domin_eigenvect = K.dot(WW, domin_eigenvect)
+
         WWd = K.dot(WW, domin_eigenvect)
-        
+
         # the corresponding dominant eigenvalue:
         domin_eigenval = K.dot(K.transpose(WWd), domin_eigenvect) / K.dot(K.transpose(domin_eigenvect), domin_eigenvect)
         regularized_loss = loss + (domin_eigenval ** 0.5) * self.k  # multiplied by the given regularization gain

keras/utils/np_utils.py

@@ -59,18 +59,64 @@ def convert_kernel(kernel, dim_ordering='th'):
     is its own inverse).
     '''
     new_kernel = np.copy(kernel)
-    if dim_ordering == 'th':
-        w = kernel.shape[2]
-        h = kernel.shape[3]
-        for i in range(w):
-            for j in range(h):
-                new_kernel[:, :, i, j] = kernel[:, :, w - i - 1, h - j - 1]
-    elif dim_ordering == 'tf':
-        w = kernel.shape[0]
-        h = kernel.shape[1]
-        for i in range(w):
-            for j in range(h):
-                new_kernel[i, j, :, :] = kernel[w - i - 1, h - j - 1, :, :]
+    if kernel.ndim == 4:
+        # conv 2d
+        # TH kernel shape: (depth, input_depth, rows, cols)
+        # TF kernel shape: (rows, cols, input_depth, depth)
+        if dim_ordering == 'th':
+            w = kernel.shape[2]
+            h = kernel.shape[3]
+            for i in range(w):
+                for j in range(h):
+                    new_kernel[:, :, i, j] = kernel[:, :, w - i - 1, h - j - 1]
+        elif dim_ordering == 'tf':
+            w = kernel.shape[0]
+            h = kernel.shape[1]
+            for i in range(w):
+                for j in range(h):
+                    new_kernel[i, j, :, :] = kernel[w - i - 1, h - j - 1, :, :]
+        else:
+            raise Exception('Invalid dim_ordering: ' + str(dim_ordering))
+    elif kernel.ndim == 5:
+        # conv 3d
+        # TH kernel shape: (out_depth, input_depth, kernel_dim1, kernel_dim2, kernel_dim3)
+        # TF kernel shape: (kernel_dim1, kernel_dim2, kernel_dim3, input_depth, out_depth)
+        if dim_ordering == 'th':
+            w = kernel.shape[2]
+            h = kernel.shape[3]
+            z = kernel.shape[4]
+            for i in range(w):
+                for j in range(h):
+                    for k in range(z):
+                        new_kernel[:, :, i, j, k] = kernel[:, :,
+                                                           w - i - 1,
+                                                           h - j - 1,
+                                                           z - k - 1]
+        elif dim_ordering == 'tf':
+            w = kernel.shape[0]
+            h = kernel.shape[1]
+            z = kernel.shape[2]
+            for i in range(w):
+                for j in range(h):
+                    for k in range(z):
+                        new_kernel[i, j, k, :, :] = kernel[w - i - 1,
+                                                           h - j - 1,
+                                                           z - k - 1,
+                                                           :, :]
+        else:
+            raise Exception('Invalid dim_ordering: ' + str(dim_ordering))
     else:
-        raise Exception('Invalid dim_ordering: ' + str(dim_ordering))
+        raise ValueError('Invalid kernel shape:', kernel.shape)
     return new_kernel
+
+
+def conv_output_length(input_length, filter_size, border_mode, stride, dilation=1):
+    if input_length is None:
+        return None
+    assert border_mode in {'same', 'valid'}
+    dilated_filter_size = filter_size + (filter_size - 1) * (dilation - 1)
+    if border_mode == 'same':
+        output_length = input_length
+    elif border_mode == 'valid':
+        output_length = input_length - dilated_filter_size + 1
+    return (output_length + stride - 1) // stride

setup.py

@@ -3,12 +3,12 @@ from setuptools import find_packages
 
 
 setup(name='Keras',
-      version='1.0.5',
+      version='1.0.6',
       description='Deep Learning for Python',
       author='Francois Chollet',
       author_email='francois.chollet@gmail.com',
       url='https://github.com/fchollet/keras',
-      download_url='https://github.com/fchollet/keras/tarball/1.0.5',
+      download_url='https://github.com/fchollet/keras/tarball/1.0.6',
       license='MIT',
       install_requires=['theano', 'pyyaml', 'six'],
       extras_require={

tests/keras/backend/test_backends.py

@@ -57,7 +57,7 @@ def check_composed_tensor_operations(first_function_name, first_function_args,
     ztf = KTF.eval(getattr(KTF, second_function_name)(ytf, **second_function_args))
 
     assert zth.shape == ztf.shape
-    assert_allclose(zth, ztf, atol=1e-05)   
+    assert_allclose(zth, ztf, atol=1e-05)
 
 class TestBackend(object):
 
@@ -90,8 +90,8 @@ class TestBackend(object):
         check_single_tensor_operation('expand_dims', (4, 3), dim=-1)
         check_single_tensor_operation('expand_dims', (4, 3, 2), dim=1)
         check_single_tensor_operation('squeeze', (4, 3, 1), axis=2)
-        check_composed_tensor_operations('reshape', {'shape':(4,3,1,1)}, 
-                                         'squeeze', {'axis':2}, 
+        check_composed_tensor_operations('reshape', {'shape':(4,3,1,1)},
+                                         'squeeze', {'axis':2},
                                          (4, 3, 1, 1))
 
     def test_repeat_elements(self):
@@ -208,14 +208,24 @@ class TestBackend(object):
         exptf = xtf * KTF.exp(xtf)
         lossth = KTH.sum(expth)
         losstf = KTF.sum(exptf)
+        zero_lossth = KTH.stop_gradient(lossth)
+        zero_losstf = KTF.stop_gradient(losstf)
 
         gradth = KTH.gradients(lossth, [expth])
         gradtf = KTF.gradients(losstf, [exptf])
+        zero_gradth = KTH.gradients(lossth + zero_lossth, [expth])
+        zero_gradtf = KTF.gradients(losstf + zero_losstf, [exptf])
 
         zth = KTH.eval(gradth[0])
         ztf = KTF.eval(gradtf[0])
+        zero_zth = KTH.eval(zero_gradth[0])
+        zero_ztf = KTF.eval(zero_gradtf[0])
         assert zth.shape == ztf.shape
+        assert zero_zth.shape == zero_ztf.shape
         assert_allclose(zth, ztf, atol=1e-05)
+        assert_allclose(zero_zth, zero_ztf, atol=1e-05)
+        assert_allclose(zero_zth, zth, atol=1e-05)
+        assert_allclose(zero_ztf, ztf, atol=1e-05)
 
     def test_function(self):
         val = np.random.random((4, 2))
@@ -450,6 +460,51 @@ class TestBackend(object):
         assert zth.shape == ztf.shape
         assert_allclose(zth, ztf, atol=1e-05)
 
+    def test_conv3d(self):
+        # TH input shape: (samples, input_depth, conv_dim1, conv_dim2, conv_dim3)
+        # TF input shape: (samples, conv_dim1, conv_dim2, conv_dim3, input_depth)
+        # TH kernel shape: (depth, input_depth, x, y, z)
+        # TF kernel shape: (x, y, z, input_depth, depth)
+
+        # test in dim_ordering = th
+        for input_shape in [(2, 3, 4, 5, 4), (2, 3, 5, 4, 6)]:
+            for kernel_shape in [(4, 3, 2, 2, 2), (4, 3, 3, 2, 4)]:
+                xval = np.random.random(input_shape)
+
+                xth = KTH.variable(xval)
+                xtf = KTF.variable(xval)
+
+                kernel_val = np.random.random(kernel_shape) - 0.5
+
+                kernel_th = KTH.variable(convert_kernel(kernel_val))
+                kernel_tf = KTF.variable(kernel_val)
+
+                zth = KTH.eval(KTH.conv3d(xth, kernel_th))
+                ztf = KTF.eval(KTF.conv3d(xtf, kernel_tf))
+
+                assert zth.shape == ztf.shape
+                assert_allclose(zth, ztf, atol=1e-05)
+
+        # test in dim_ordering = tf
+        input_shape = (1, 2, 2, 2, 1)
+        kernel_shape = (2, 2, 2, 1, 1)
+
+        xval = np.random.random(input_shape)
+
+        xth = KTH.variable(xval)
+        xtf = KTF.variable(xval)
+
+        kernel_val = np.random.random(kernel_shape) - 0.5
+
+        kernel_th = KTH.variable(convert_kernel(kernel_val, dim_ordering='tf'))
+        kernel_tf = KTF.variable(kernel_val)
+
+        zth = KTH.eval(KTH.conv3d(xth, kernel_th, dim_ordering='tf'))
+        ztf = KTF.eval(KTF.conv3d(xtf, kernel_tf, dim_ordering='tf'))
+
+        assert zth.shape == ztf.shape
+        assert_allclose(zth, ztf, atol=1e-05)
+
     def test_pool2d(self):
         check_single_tensor_operation('pool2d', (5, 3, 10, 12), pool_size=(2, 2),
                                       strides=(1, 1), border_mode='valid')
@@ -460,6 +515,16 @@ class TestBackend(object):
         check_single_tensor_operation('pool2d', (5, 3, 9, 11), pool_size=(2, 3),
                                       strides=(1, 1), border_mode='valid')
 
+    def test_pool3d(self):
+        check_single_tensor_operation('pool3d', (5, 3, 10, 12, 5), pool_size=(2, 2, 2),
+                                      strides=(1, 1, 1), border_mode='valid')
+
+        check_single_tensor_operation('pool3d', (5, 3, 9, 11, 5), pool_size=(2, 2, 2),
+                                      strides=(1, 1, 1), border_mode='valid')
+
+        check_single_tensor_operation('pool3d', (5, 3, 9, 11, 5), pool_size=(2, 3, 2),
+                                      strides=(1, 1, 1), border_mode='valid')
+
     def test_random_normal(self):
         mean = 0.
         std = 1.

tests/keras/datasets/test_datasets.py

@@ -1,26 +1,44 @@
 from __future__ import print_function
 import pytest
+import time
+import random
 from keras.datasets import cifar10, cifar100, reuters, imdb, mnist
 
 
 def test_cifar():
-    (X_train, y_train), (X_test, y_test) = cifar10.load_data()
-    (X_train, y_train), (X_test, y_test) = cifar100.load_data('fine')
-    (X_train, y_train), (X_test, y_test) = cifar100.load_data('coarse')
+    # only run data download tests 20% of the time
+    # to speed up frequent testing
+    random.seed(time.time())
+    if random.random() > 0.8:
+        (X_train, y_train), (X_test, y_test) = cifar10.load_data()
+        (X_train, y_train), (X_test, y_test) = cifar100.load_data('fine')
+        (X_train, y_train), (X_test, y_test) = cifar100.load_data('coarse')
 
 
 def test_reuters():
-    (X_train, y_train), (X_test, y_test) = reuters.load_data()
-    (X_train, y_train), (X_test, y_test) = reuters.load_data(maxlen=10)
+    # only run data download tests 20% of the time
+    # to speed up frequent testing
+    random.seed(time.time())
+    if random.random() > 0.8:
+        (X_train, y_train), (X_test, y_test) = reuters.load_data()
+        (X_train, y_train), (X_test, y_test) = reuters.load_data(maxlen=10)
 
 
 def test_mnist():
-    (X_train, y_train), (X_test, y_test) = mnist.load_data()
+    # only run data download tests 20% of the time
+    # to speed up frequent testing
+    random.seed(time.time())
+    if random.random() > 0.8:
+        (X_train, y_train), (X_test, y_test) = mnist.load_data()
 
 
 def test_imdb():
-    (X_train, y_train), (X_test, y_test) = imdb.load_data()
-    (X_train, y_train), (X_test, y_test) = imdb.load_data(maxlen=40)
+    # only run data download tests 20% of the time
+    # to speed up frequent testing
+    random.seed(time.time())
+    if random.random() > 0.8:
+        (X_train, y_train), (X_test, y_test) = imdb.load_data()
+        (X_train, y_train), (X_test, y_test) = imdb.load_data(maxlen=40)
 
 
 if __name__ == '__main__':

tests/keras/engine/test_topology.py

@@ -2,7 +2,7 @@ import pytest
 import json
 import numpy as np
 
-from keras.layers import Dense, Dropout
+from keras.layers import Dense, Dropout, InputLayer
 from keras.engine import merge, Input, get_source_inputs
 from keras.models import Model
 from keras import backend as K
@@ -392,6 +392,13 @@ def test_recursion():
         # test merge
         o_tf = merge([j_tf, k_tf], mode='concat', concat_axis=1)
 
+        # test tensor input
+        x = tf.placeholder(shape=(None, 2), dtype=K.floatx())
+        input_layer = InputLayer(input_tensor=x)
+
+        x = Input(tensor=x)
+        y = Dense(2)(x)
+
 
 def test_functional_guide():
     # MNIST

tests/keras/layers/test_convolutional.py

@@ -53,7 +53,7 @@ def test_averagepooling_1d():
 
 
 def test_convolution_2d():
-    nb_samples = 8
+    nb_samples = 2
     nb_filter = 3
     stack_size = 4
     nb_row = 10
@@ -84,6 +84,82 @@ def test_convolution_2d():
                        input_shape=(nb_samples, stack_size, nb_row, nb_col))
 
 
+def test_atrous_conv_2d():
+    nb_samples = 2
+    nb_filter = 3
+    stack_size = 4
+    nb_row = 10
+    nb_col = 6
+
+    for border_mode in ['valid', 'same']:
+        for subsample in [(1, 1), (2, 2)]:
+            for atrous_rate in [(1, 1), (2, 2)]:
+                if border_mode == 'same' and subsample != (1, 1):
+                    continue
+                if subsample != (1, 1) and atrous_rate != (1, 1):
+                    continue
+
+                layer_test(convolutional.AtrousConv2D,
+                           kwargs={'nb_filter': nb_filter,
+                                   'nb_row': 3,
+                                   'nb_col': 3,
+                                   'border_mode': border_mode,
+                                   'subsample': subsample,
+                                   'atrous_rate': atrous_rate},
+                           input_shape=(nb_samples, stack_size, nb_row, nb_col))
+
+                layer_test(convolutional.AtrousConv2D,
+                           kwargs={'nb_filter': nb_filter,
+                                   'nb_row': 3,
+                                   'nb_col': 3,
+                                   'border_mode': border_mode,
+                                   'W_regularizer': 'l2',
+                                   'b_regularizer': 'l2',
+                                   'activity_regularizer': 'activity_l2',
+                                   'subsample': subsample,
+                                   'atrous_rate': atrous_rate},
+                           input_shape=(nb_samples, stack_size, nb_row, nb_col))
+
+
+@pytest.mark.skipif(K._BACKEND != 'tensorflow', reason="Requires TF backend")
+def test_separable_conv_2d():
+    nb_samples = 2
+    nb_filter = 8
+    stack_size = 4
+    nb_row = 10
+    nb_col = 6
+
+    for border_mode in ['valid', 'same']:
+        for subsample in [(1, 1), (2, 2)]:
+            for multiplier in [1, 2]:
+                if border_mode == 'same' and subsample != (1, 1):
+                    continue
+
+                layer_test(convolutional.SeparableConv2D,
+                           kwargs={'nb_filter': nb_filter,
+                                   'nb_row': 3,
+                                   'nb_col': 3,
+                                   'border_mode': border_mode,
+                                   'subsample': subsample,
+                                   'depth_multiplier': multiplier},
+                           input_shape=(nb_samples, stack_size, nb_row, nb_col))
+
+                layer_test(convolutional.SeparableConv2D,
+                           kwargs={'nb_filter': nb_filter,
+                                   'nb_row': 3,
+                                   'nb_col': 3,
+                                   'border_mode': border_mode,
+                                   'depthwise_regularizer': 'l2',
+                                   'pointwise_regularizer': 'l2',
+                                   'b_regularizer': 'l2',
+                                   'activity_regularizer': 'activity_l2',
+                                   'pointwise_constraint': 'unitnorm',
+                                   'depthwise_constraint': 'unitnorm',
+                                   'subsample': subsample,
+                                   'depth_multiplier': multiplier},
+                           input_shape=(nb_samples, stack_size, nb_row, nb_col))
+
+
 def test_maxpooling_2d():
     pool_size = (3, 3)
 
@@ -108,7 +184,6 @@ def test_averagepooling_2d():
                            input_shape=(3, 4, 11, 12))
 
 
-@pytest.mark.skipif(K._BACKEND != 'theano', reason="Requires Theano backend")
 def test_convolution_3d():
     nb_samples = 2
     nb_filter = 5
@@ -150,7 +225,6 @@ def test_convolution_3d():
                                     input_len_dim1, input_len_dim2, input_len_dim3))
 
 
-@pytest.mark.skipif(K._BACKEND != 'theano', reason="Requires Theano backend")
 def test_maxpooling_3d():
     pool_size = (3, 3, 3)
 
@@ -162,7 +236,6 @@ def test_maxpooling_3d():
                    input_shape=(3, 4, 11, 12, 10))
 
 
-@pytest.mark.skipif(K._BACKEND != 'theano', reason="Requires Theano backend")
 def test_averagepooling_3d():
     pool_size = (3, 3, 3)
 
@@ -175,7 +248,7 @@ def test_averagepooling_3d():
 
 
 def test_zero_padding_2d():
-    nb_samples = 9
+    nb_samples = 2
     stack_size = 7
     input_nb_row = 11
     input_nb_col = 12
@@ -201,7 +274,7 @@ def test_zero_padding_2d():
 
 @pytest.mark.skipif(K._BACKEND != 'theano', reason="Requires Theano backend")
 def test_zero_padding_3d():
-    nb_samples = 9
+    nb_samples = 2
     stack_size = 7
     input_len_dim1 = 10
     input_len_dim2 = 11
@@ -234,7 +307,7 @@ def test_upsampling_1d():
 
 
 def test_upsampling_2d():
-    nb_samples = 9
+    nb_samples = 2
     stack_size = 7
     input_nb_row = 11
     input_nb_col = 12
@@ -275,7 +348,7 @@ def test_upsampling_2d():
 
 @pytest.mark.skipif(K._BACKEND != 'theano', reason="Requires Theano backend")
 def test_upsampling_3d():
-    nb_samples = 9
+    nb_samples = 2
     stack_size = 7
     input_len_dim1 = 10
     input_len_dim2 = 11
@@ -320,5 +393,4 @@ def test_upsampling_3d():
 
 
 if __name__ == '__main__':
-    # pytest.main([__file__])
-    test_convolution_3d()
+    pytest.main([__file__])

tests/keras/layers/test_core.py

@@ -21,7 +21,7 @@ def test_merge():
     inputs = [np.random.random(shape) for shape in input_shapes]
 
     # test functional API
-    for mode in ['sum', 'mul', 'concat', 'ave']:
+    for mode in ['sum', 'mul', 'concat', 'ave', 'max']:
         print(mode)
         input_a = Input(shape=input_shapes[0][1:])
         input_b = Input(shape=input_shapes[1][1:])

tests/keras/layers/test_local.py

@@ -0,0 +1,74 @@
+import pytest
+
+from keras.utils.test_utils import layer_test
+from keras.layers import local
+
+
+def test_locallyconnected_1d():
+    nb_samples = 2
+    nb_steps = 8
+    input_dim = 5
+    filter_length = 3
+    nb_filter = 4
+
+    for border_mode in ['valid']:
+        for subsample_length in [1]:
+            if border_mode == 'same' and subsample_length != 1:
+                continue
+            layer_test(local.LocallyConnected1D,
+                       kwargs={'nb_filter': nb_filter,
+                               'filter_length': filter_length,
+                               'border_mode': border_mode,
+                               'subsample_length': subsample_length},
+                       input_shape=(nb_samples, nb_steps, input_dim))
+
+            layer_test(local.LocallyConnected1D,
+                       kwargs={'nb_filter': nb_filter,
+                               'filter_length': filter_length,
+                               'border_mode': border_mode,
+                               'W_regularizer': 'l2',
+                               'b_regularizer': 'l2',
+                               'activity_regularizer': 'activity_l2',
+                               'subsample_length': subsample_length},
+                       input_shape=(nb_samples, nb_steps, input_dim))
+
+
+def test_locallyconnected_2d():
+    nb_samples = 8
+    nb_filter = 3
+    stack_size = 4
+    nb_row = 6
+    nb_col = 10
+
+    for border_mode in ['valid']:
+        for subsample in [(1, 1), (2, 2)]:
+            if border_mode == 'same' and subsample != (1, 1):
+                continue
+
+            layer_test(local.LocallyConnected2D,
+                       kwargs={'nb_filter': nb_filter,
+                               'nb_row': 3,
+                               'nb_col': 3,
+                               'border_mode': border_mode,
+                               'W_regularizer': 'l2',
+                               'b_regularizer': 'l2',
+                               'activity_regularizer': 'activity_l2',
+                               'subsample': subsample,
+                               'dim_ordering': 'tf'},
+                       input_shape=(nb_samples, nb_row, nb_col, stack_size))
+
+            layer_test(local.LocallyConnected2D,
+                       kwargs={'nb_filter': nb_filter,
+                               'nb_row': 3,
+                               'nb_col': 3,
+                               'border_mode': border_mode,
+                               'W_regularizer': 'l2',
+                               'b_regularizer': 'l2',
+                               'activity_regularizer': 'activity_l2',
+                               'subsample': subsample,
+                               'dim_ordering': 'th'},
+                       input_shape=(nb_samples, stack_size, nb_row, nb_col))
+
+
+if __name__ == '__main__':
+    pytest.main([__file__])

tests/keras/test_graph_model.py

@@ -209,7 +209,7 @@ def test_siamese_1():
     loss = graph.test_on_batch({'input1': X_test_graph, 'input2': X2_test_graph, 'output1': y_test_graph})
     loss = graph.train_on_batch({'input1': X_test_graph, 'input2': X2_test_graph, 'output1': y_test_graph})
     loss = graph.evaluate({'input1': X_test_graph, 'input2': X2_test_graph, 'output1': y_test_graph})
-    assert(loss < 4.0)
+    assert(loss < 5.0)
 
     # test serialization
     config = graph.get_config()

tests/keras/test_multiprocessing.py

@@ -0,0 +1,189 @@
+from __future__ import print_function
+import pytest
+import numpy as np
+from keras.models import Sequential
+from keras.layers.core import Dense, Activation
+
+
+def test_multiprocessing_training():
+
+    reached_end = False
+
+    arr_data = np.random.randint(0,256, (500, 200))
+    arr_labels = np.random.randint(0, 2, 500)
+
+    def myGenerator():
+
+        batch_size = 32
+        n_samples = 500
+
+        while True:
+            batch_index = np.random.randint(0, n_samples - batch_size)
+            start = batch_index
+            end = start + batch_size
+            X = arr_data[start: end]
+            y = arr_labels[start: end]
+            yield X, y
+
+    # Build a NN
+    model = Sequential()
+    model.add(Dense(10, input_shape=(200, )))
+    model.add(Activation('relu'))
+    model.add(Dense(1))
+    model.add(Activation('linear'))
+    model.compile(loss='mse', optimizer='adadelta')
+
+    model.fit_generator(myGenerator(),
+                        samples_per_epoch=320,
+                        nb_epoch=1,
+                        verbose=1,
+                        max_q_size=10,
+                        nb_worker=4,
+                        pickle_safe=True)
+
+    model.fit_generator(myGenerator(),
+                        samples_per_epoch=320,
+                        nb_epoch=1,
+                        verbose=1,
+                        max_q_size=10,
+                        pickle_safe=False)
+
+    reached_end = True
+
+    assert reached_end
+
+
+def test_multiprocessing_training_fromfile():
+
+    reached_end = False
+
+    arr_data = np.random.randint(0,256, (500, 200))
+    arr_labels = np.random.randint(0, 2, 500)
+    np.savez("data.npz", **{"data": arr_data, "labels": arr_labels})
+
+    def myGenerator():
+
+        batch_size = 32
+        n_samples = 500
+
+        arr = np.load("data.npz")
+
+        while True:
+            batch_index = np.random.randint(0, n_samples - batch_size)
+            start = batch_index
+            end = start + batch_size
+            X = arr["data"][start: end]
+            y = arr["labels"][start: end]
+            yield X, y
+
+    # Build a NN
+    model = Sequential()
+    model.add(Dense(10, input_shape=(200, )))
+    model.add(Activation('relu'))
+    model.add(Dense(1))
+    model.add(Activation('linear'))
+    model.compile(loss='mse', optimizer='adadelta')
+
+    model.fit_generator(myGenerator(),
+                        samples_per_epoch=320,
+                        nb_epoch=1,
+                        verbose=1,
+                        max_q_size=10,
+                        nb_worker=2,
+                        pickle_safe=True)
+
+    model.fit_generator(myGenerator(),
+                        samples_per_epoch=320,
+                        nb_epoch=1,
+                        verbose=1,
+                        max_q_size=10,
+                        pickle_safe=False)
+    reached_end = True
+
+    assert reached_end
+
+
+def test_multiprocessing_predicting():
+
+    reached_end = False
+
+    arr_data = np.random.randint(0,256, (500, 200))
+
+    def myGenerator():
+
+        batch_size = 32
+        n_samples = 500
+
+        while True:
+            batch_index = np.random.randint(0, n_samples - batch_size)
+            start = batch_index
+            end = start + batch_size
+            X = arr_data[start: end]
+            yield X
+
+    # Build a NN
+    model = Sequential()
+    model.add(Dense(10, input_shape=(200, )))
+    model.add(Activation('relu'))
+    model.add(Dense(1))
+    model.add(Activation('linear'))
+    model.compile(loss='mse', optimizer='adadelta')
+    model.predict_generator(myGenerator(),
+                            val_samples=320,
+                            max_q_size=10,
+                            nb_worker=2,
+                            pickle_safe=True)
+    model.predict_generator(myGenerator(),
+                            val_samples=320,
+                            max_q_size=10,
+                            pickle_safe=False)
+    reached_end = True
+
+    assert reached_end
+
+
+def test_multiprocessing_evaluating():
+
+    reached_end = False
+
+    arr_data = np.random.randint(0,256, (500, 200))
+    arr_labels = np.random.randint(0, 2, 500)
+
+    def myGenerator():
+
+        batch_size = 32
+        n_samples = 500
+
+        while True:
+            batch_index = np.random.randint(0, n_samples - batch_size)
+            start = batch_index
+            end = start + batch_size
+            X = arr_data[start: end]
+            y = arr_labels[start: end]
+            yield X, y
+
+    # Build a NN
+    model = Sequential()
+    model.add(Dense(10, input_shape=(200, )))
+    model.add(Activation('relu'))
+    model.add(Dense(1))
+    model.add(Activation('linear'))
+    model.compile(loss='mse', optimizer='adadelta')
+
+    model.evaluate_generator(myGenerator(),
+                             val_samples=320,
+                             max_q_size=10,
+                             nb_worker=2,
+                             pickle_safe=True)
+    model.evaluate_generator(myGenerator(),
+                             val_samples=320,
+                             max_q_size=10,
+                             pickle_safe=False)
+    reached_end = True
+
+    assert reached_end
+
+
+if __name__ == '__main__':
+
+    pytest.main([__file__])

tests/keras/test_sequential_model.py

@@ -59,6 +59,8 @@ def test_sequential_fit_generator():
     model.add(Dense(nb_hidden, input_shape=(input_dim,)))
     model.add(Activation('relu'))
     model.add(Dense(nb_class))
+    model.pop()
+    model.add(Dense(nb_class))
     model.add(Activation('softmax'))
     model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
 

tests/test_loss_masking.py

@@ -10,16 +10,16 @@ from keras import backend as K
 
 def test_masking():
     np.random.seed(1337)
-    X = np.array(
-        [[[1, 1], [2, 1], [3, 1], [5, 5]],
-         [[1, 5], [5, 0], [0, 0], [0, 0]]], dtype=np.int32)
+    X = np.array([[[1], [1]],
+                  [[0], [0]]])
     model = Sequential()
-    model.add(Masking(mask_value=0, input_shape=(4, 2)))
+    model.add(Masking(mask_value=0, input_shape=(2, 1)))
     model.add(TimeDistributedDense(1, init='one'))
     model.compile(loss='mse', optimizer='sgd')
-    y = model.predict(X)
-    history = model.fit(X, 4 * y, nb_epoch=1, batch_size=2, verbose=1)
-    assert history.history['loss'][0] == 285.
+    y = np.array([[[1], [1]],
+                  [[1], [1]]])
+    loss = model.train_on_batch(X, y)
+    assert loss == 0
 
 
 def test_loss_masking():